agentfootprint/dist/esm/rag/splitters/byHeading.js

The explainable agent framework — backtrack a wrong answer to the exact context that caused it (evidence, not guesses). Built on footprintjs.

import { DEFAULT_MAX_CHARS, DEFAULT_OVERLAP_CHARS } from './constants.js';
import { applyOverlap, foldRunts, hardCut, mergeShortSpansForward, packSpans, paragraphSpans, resolveMinChars, toPieces, } from './shared.js';
import { isLineBreak } from '../../lib/linearText.js';
/**
 * ATX headings only (`# Title`). Setext (`Title\n=====`) is rare in docs corpora.
 *
 * Exactly the lines `/^(#{1,6})[ \t]+(.+?)[ \t]*$/gm` matches, with the same
 * title, found in one pass. The regex is not used because a document is input
 * a deployer may not trust: on `# a` followed by a long run of blanks and one
 * more letter, its lazy title stops at every blank and re-reads the rest of the
 * run to see whether the line ends there — quadratic in the run (16,000 blanks
 * take ~0.2 s in V8).
 */
export function atxHeadings(text) {
    const found = [];
    let lineStart = 0;
    for (;;) {
        let lineEnd = lineStart;
        while (lineEnd < text.length && !isLineBreak(text.charCodeAt(lineEnd)))
            lineEnd++;
        const heading = atxHeadingOn(text, lineStart, lineEnd);
        if (heading !== undefined)
            found.push(heading);
        if (lineEnd >= text.length)
            return found;
        lineStart = lineEnd + 1;
    }
}
/** The heading on the line `[start, end)`, if it is one. */
function atxHeadingOn(text, start, end) {
    let level = 0;
    while (start + level < end && text.charCodeAt(start + level) === 0x23)
        level++;
    if (level === 0 || level > 6)
        return undefined;
    const afterHashes = start + level;
    if (afterHashes >= end || !isBlank(text.charCodeAt(afterHashes)))
        return undefined;
    let titleStart = afterHashes;
    while (titleStart < end && isBlank(text.charCodeAt(titleStart)))
        titleStart++;
    let titleEnd = end;
    while (titleEnd > titleStart && isBlank(text.charCodeAt(titleEnd - 1)))
        titleEnd--;
    if (titleEnd > titleStart) {
        return { offset: start, end, level, title: text.slice(titleStart, titleEnd) };
    }
    // Hashes and blanks only. The pattern still matches once there are two
    // blanks: `[ \t]+` gives one back and the title is the last blank.
    if (end - afterHashes >= 2) {
        return { offset: start, end, level, title: text.slice(end - 1, end) };
    }
    return undefined;
}
/** `[ \t]` */
function isBlank(code) {
    return code === 0x20 || code === 0x09;
}
export function byHeading(options = {}) {
    const maxChars = options.maxChars ?? DEFAULT_MAX_CHARS;
    const overlapChars = options.overlapChars ?? DEFAULT_OVERLAP_CHARS;
    const maxLevel = options.maxLevel ?? 6;
    const minChars = resolveMinChars(options.minChars, maxChars);
    return {
        name: 'byHeading',
        split(doc) {
            const text = doc.text;
            // Where each section starts, and what it is called.
            const starts = [];
            for (const found of atxHeadings(text)) {
                if (found.level > maxLevel)
                    continue;
                starts.push({ offset: found.offset, heading: found.title, headingEnd: found.end });
            }
            // A document with no headings at all is a paragraph document; say so by
            // doing the paragraph thing rather than returning one giant chunk.
            if (starts.length === 0) {
                const packed = packSpans(paragraphSpans(text), maxChars);
                const cut = packed.flatMap((s) => s.end - s.start > maxChars ? hardCut(s, maxChars, overlapChars) : [s]);
                const floored = mergeShortSpansForward(foldRunts(cut), minChars);
                return toPieces(doc, applyOverlap(floored, overlapChars));
            }
            const sections = [];
            // Preamble before the first heading — its own section, unnamed. Its
            // whole span IS its body.
            const firstStart = starts[0]?.offset ?? 0;
            if (firstStart > 0)
                sections.push({ start: 0, end: firstStart, bodyStart: 0 });
            for (let i = 0; i < starts.length; i++) {
                const start = starts[i]?.offset ?? 0;
                const end = starts[i + 1]?.offset ?? text.length;
                const heading = starts[i]?.heading;
                const bodyStart = Math.min(end, starts[i]?.headingEnd ?? start);
                sections.push({ start, end, bodyStart, ...(heading !== undefined && { heading }) });
            }
            // Sections that fit stay whole; the rest are packed by paragraph WITHIN
            // the section, so a boundary never merges two headings' content. Kept
            // GROUPED per section here, because the floor pass below must reach the
            // FIRST chunk of a section and only that one.
            const groups = sections.map((section) => {
                if (section.end - section.start <= maxChars) {
                    return [
                        {
                            start: section.start,
                            end: section.end,
                            ...(section.heading !== undefined && { heading: section.heading }),
                        },
                    ];
                }
                // Paragraphs of the BODY, with the heading line glued onto the first
                // one. Splitting the section's own text used to make the heading line
                // its own paragraph unit — and when the first body paragraph was too
                // big to pack with it, the heading shipped as a chunk of its own. That
                // is the exact heading-only failure this release closes; the heading
                // is a label, never a unit.
                const inner = paragraphSpans(text.slice(section.bodyStart, section.end)).map((s) => ({
                    start: section.bodyStart + s.start,
                    end: section.bodyStart + s.end,
                    ...(section.heading !== undefined && { heading: section.heading }),
                }));
                const firstInner = inner[0];
                if (firstInner !== undefined && section.start < section.bodyStart) {
                    inner[0] = {
                        start: section.start,
                        end: firstInner.end,
                        ...(section.heading !== undefined && { heading: section.heading }),
                    };
                }
                const spans = [];
                for (const packedSpan of packSpans(inner, maxChars)) {
                    if (packedSpan.end - packedSpan.start > maxChars) {
                        spans.push(...hardCut(packedSpan, maxChars, overlapChars));
                    }
                    else {
                        spans.push(packedSpan);
                    }
                }
                return spans;
            });
            // ── The floor pass (8.20.0). ─────────────────────────────────────────
            // A short section is by construction a single span (its whole span is
            // under the floor, which is under maxChars), so merging it forward means
            // extending the FIRST span of the next full section back over it — the
            // rest of that section's spans keep their own heading. Consecutive short
            // sections accumulate; an accumulation that clears the floor and has a
            // real body is emitted as its own chunk rather than diluting a full
            // neighbour.
            const out = [];
            let pending;
            for (let i = 0; i < sections.length; i++) {
                // sections and groups are built index-for-index above.
                // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
                const section = sections[i];
                // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
                const group = groups[i];
                const bodyLength = text.slice(section.bodyStart, section.end).trim().length;
                const bare = bodyLength === 0; // heading-plus-whitespace: merged UNCONDITIONALLY
                const short = !bare && bodyLength < minChars;
                if (bare || short) {
                    if (pending === undefined) {
                        pending = {
                            start: section.start,
                            bodyChars: bodyLength,
                            ...(section.heading !== undefined && { heading: section.heading }),
                        };
                    }
                    else {
                        pending.bodyChars += bodyLength;
                    }
                    // Adjacent shorts whose BODIES together clear the floor become their
                    // own chunk — the same measure shortness is judged by, so a single
                    // short section can never self-emit on the strength of its markup.
                    if (minChars > 0 && pending.bodyChars >= minChars) {
                        out.push({
                            start: pending.start,
                            end: section.end,
                            ...(pending.heading !== undefined && { heading: pending.heading }),
                        });
                        pending = undefined;
                    }
                    continue;
                }
                if (pending !== undefined) {
                    const first = group[0];
                    if (first !== undefined) {
                        // The merged chunk STARTS at the short section's heading, so the
                        // short section's heading is the honest label. A heading-less
                        // preamble keeps the chunk heading-less: a chunk is never labelled
                        // with a heading its text only reaches later.
                        group[0] = {
                            start: pending.start,
                            end: first.end,
                            ...(pending.heading !== undefined && { heading: pending.heading }),
                        };
                    }
                    pending = undefined;
                }
                out.push(...group);
            }
            if (pending !== undefined) {
                const lastSectionEnd = sections[sections.length - 1]?.end ?? text.length;
                const previous = out.pop();
                if (previous !== undefined) {
                    // Trailing shorts have no next chunk — the one edge that merges
                    // BACKWARD, extending the previous chunk under its own heading.
                    out.push({
                        start: previous.start,
                        end: lastSectionEnd,
                        ...(previous.heading !== undefined && { heading: previous.heading }),
                    });
                }
                else if (pending.bodyChars > 0) {
                    // The whole document is short sections. No neighbour exists, and
                    // "never dropped" wins: one merged chunk.
                    out.push({
                        start: pending.start,
                        end: lastSectionEnd,
                        ...(pending.heading !== undefined && { heading: pending.heading }),
                    });
                }
                // else: nothing but headings and whitespace — coordinates without a
                // passage anywhere. No chunk is emitted; `indexDocuments` makes the
                // same refusal one layer up for a passage-less document.
            }
            // Within-section tail runts still fold BACKWARD (same heading — the
            // direction that cannot mislabel), now up to the same floor.
            const runtFloor = minChars > 0 ? minChars : undefined;
            return toPieces(doc, applyOverlap(foldRunts(out, runtFloor), overlapChars));
        },
    };
}
//# sourceMappingURL=byHeading.js.map