agentfootprint/dist/esm/rag/loaders/text.js

The explainable agent framework — backtrack a wrong answer to the exact context that caused it (evidence, not guesses). Built on footprintjs.

/** Decode UTF-8 and normalise line endings. Shared by every text-shaped loader. */
export function decodeText(bytes) {
    // `fatal: false` — a corpus is other people's files, and one bad byte in a
    // 200-page document should degrade that character, not lose the document.
    const text = new TextDecoder('utf-8', { fatal: false }).decode(bytes);
    // Strip a UTF-8 BOM: it is invisible, it is not content, and left in place it
    // becomes the first character of the first chunk of the document.
    const withoutBom = text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
    return withoutBom.replace(/\r\n/g, '\n');
}
export function textLoader() {
    return {
        name: 'text',
        extensions: ['.txt', '.text', '.log', '.csv', '.tsv', '.json', '.yaml', '.yml'],
        // eslint-disable-next-line @typescript-eslint/require-await
        async load(input) {
            return { text: decodeText(input.bytes) };
        },
    };
}
//# sourceMappingURL=text.js.map