agentfootprint/dist/esm/rag/splitters/fixedWithOverlap.js

The explainable agent framework — backtrack a wrong answer to the exact context that caused it (evidence, not guesses). Built on footprintjs.

import { DEFAULT_MAX_CHARS, DEFAULT_OVERLAP_CHARS } from './constants.js';
import { applyOverlap, foldRunts, toPieces } from './shared.js';
export function fixedWithOverlap(options = {}) {
    const chars = options.chars ?? DEFAULT_MAX_CHARS;
    const overlapChars = options.overlapChars ?? DEFAULT_OVERLAP_CHARS;
    const wordWindow = options.wordWindow ?? 40;
    return {
        name: 'fixedWithOverlap',
        split(doc) {
            const text = doc.text;
            const spans = [];
            let start = 0;
            while (start < text.length) {
                let end = Math.min(text.length, start + chars);
                // Nudge back to whitespace, but never past the window and never so far
                // that the chunk collapses.
                if (end < text.length && wordWindow > 0) {
                    const floor = Math.max(start + 1, end - wordWindow);
                    let candidate = end;
                    while (candidate > floor && !/\s/.test(text[candidate] ?? ''))
                        candidate -= 1;
                    if (candidate > floor)
                        end = candidate;
                }
                spans.push({ start, end });
                if (end >= text.length)
                    break;
                start = end;
            }
            return toPieces(doc, applyOverlap(foldRunts(spans), overlapChars));
        },
    };
}
//# sourceMappingURL=fixedWithOverlap.js.map