agentfootprint/dist/rag/splitters/fixedWithOverlap.js

The explainable agent framework — backtrack a wrong answer to the exact context that caused it (evidence, not guesses). Built on footprintjs.

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.fixedWithOverlap = void 0;
const constants_js_1 = require("./constants.js");
const shared_js_1 = require("./shared.js");
function fixedWithOverlap(options = {}) {
    const chars = options.chars ?? constants_js_1.DEFAULT_MAX_CHARS;
    const overlapChars = options.overlapChars ?? constants_js_1.DEFAULT_OVERLAP_CHARS;
    const wordWindow = options.wordWindow ?? 40;
    return {
        name: 'fixedWithOverlap',
        split(doc) {
            const text = doc.text;
            const spans = [];
            let start = 0;
            while (start < text.length) {
                let end = Math.min(text.length, start + chars);
                // Nudge back to whitespace, but never past the window and never so far
                // that the chunk collapses.
                if (end < text.length && wordWindow > 0) {
                    const floor = Math.max(start + 1, end - wordWindow);
                    let candidate = end;
                    while (candidate > floor && !/\s/.test(text[candidate] ?? ''))
                        candidate -= 1;
                    if (candidate > floor)
                        end = candidate;
                }
                spans.push({ start, end });
                if (end >= text.length)
                    break;
                start = end;
            }
            return (0, shared_js_1.toPieces)(doc, (0, shared_js_1.applyOverlap)((0, shared_js_1.foldRunts)(spans), overlapChars));
        },
    };
}
exports.fixedWithOverlap = fixedWithOverlap;
//# sourceMappingURL=fixedWithOverlap.js.map