agentfootprint/dist/esm/rag/splitters/shared.js

The explainable agent framework — backtrack a wrong answer to the exact context that caused it (evidence, not guesses). Built on footprintjs.

import { DEFAULT_MIN_CHARS, MIN_CHUNK_CHARS } from './constants.js';
/**
 * Which page an offset falls on, 1-based, or `undefined` when the document is
 * not paginated.
 *
 * Pages were joined with `'\n\n'` by the loader, so the boundaries are
 * reconstructible exactly. A chunk that straddles a page break is attributed
 * to the page it STARTS on — one number, the one you would turn to first.
 */
export function pageLocator(doc) {
    const pages = doc.pages;
    if (!pages || pages.length === 0)
        return () => undefined;
    const bounds = [];
    let cursor = 0;
    for (const page of pages) {
        bounds.push(cursor);
        cursor += page.length + 2; // the '\n\n' the loader joined with
    }
    return (offset) => {
        let page = 1;
        for (let i = 0; i < bounds.length; i++) {
            if (offset >= (bounds[i] ?? 0))
                page = i + 1;
            else
                break;
        }
        return page;
    };
}
/**
 * Trim a span inward past leading/trailing whitespace, keeping the offsets
 * honest.
 *
 * The naive version — `slice().trim()` — produces text that no longer matches
 * its own offsets. This moves the offsets instead, so the invariant holds.
 * Returns `undefined` when the span is entirely whitespace.
 */
export function trimSpan(text, span) {
    let start = span.start;
    let end = span.end;
    while (start < end && /\s/.test(text[start] ?? ''))
        start += 1;
    while (end > start && /\s/.test(text[end - 1] ?? ''))
        end -= 1;
    if (start >= end)
        return undefined;
    return {
        start,
        end,
        ...(span.heading !== undefined && { heading: span.heading }),
        ...(span.contentStart !== undefined && { contentStart: span.contentStart }),
    };
}
/**
 * Pack consecutive units (paragraphs, lines) into spans of at most `maxChars`,
 * then apply `overlapChars` of backward overlap.
 *
 * Units are never split by this function — a single unit longer than `maxChars`
 * becomes its own oversized span, and the caller decides whether to hard-cut
 * it. That separation matters: a 4,000-character paragraph is a real thing in
 * real documents, and silently slicing it mid-word is a different decision from
 * packing.
 */
export function packSpans(units, maxChars) {
    const out = [];
    let current;
    for (const unit of units) {
        if (current === undefined) {
            current = unit;
            continue;
        }
        // Same heading, and the merged span still fits.
        const fits = unit.end - current.start <= maxChars;
        const sameSection = current.heading === unit.heading;
        if (fits && sameSection) {
            current = {
                start: current.start,
                end: unit.end,
                ...(current.heading !== undefined && { heading: current.heading }),
            };
            continue;
        }
        out.push(current);
        current = unit;
    }
    if (current !== undefined)
        out.push(current);
    return out;
}
/**
 * Extend each span backward by `overlapChars`, without crossing the previous
 * span's start.
 *
 * Backward rather than forward so a chunk always ENDS on the boundary its
 * strategy chose — a heading section that ends mid-sentence because of overlap
 * would be a section that lies about where it ends.
 */
export function applyOverlap(spans, overlapChars) {
    if (overlapChars <= 0)
        return [...spans];
    return spans.map((span, i) => {
        if (i === 0)
            return span;
        const floor = spans[i - 1]?.start ?? 0;
        const start = Math.max(floor, span.start - overlapChars);
        // Remember where the chunk's own content starts, so the page it is
        // attributed to is the page it is ABOUT, not the page its run-up borrowed
        // from.
        return start === span.start ? span : { ...span, start, contentStart: span.start };
    });
}
/**
 * Fold a too-small span back into its predecessor.
 *
 * A fragment of a few characters — a stray heading, a signature line — embeds
 * to noise and then competes for a top-K slot with real passages.
 *
 * **It never folds across a heading boundary.** A merged chunk carries the
 * FIRST span's heading, so folding a short section into the one before it
 * would produce a chunk labelled `heading="Alpha"` whose text is Beta's — a
 * citation that names the wrong section, which is worse than a small chunk.
 * Two spans merge only when they are already the same section.
 */
export function foldRunts(spans, minChars = MIN_CHUNK_CHARS) {
    const out = [];
    for (const span of spans) {
        const previous = out[out.length - 1];
        const sameSection = previous !== undefined && previous.heading === span.heading;
        if (previous !== undefined && sameSection && span.end - span.start < minChars) {
            out[out.length - 1] = {
                start: previous.start,
                end: span.end,
                ...(previous.heading !== undefined && { heading: previous.heading }),
            };
            continue;
        }
        out.push(span);
    }
    return out;
}
/**
 * Resolve the effective `minChars` floor for a splitter (8.20.0).
 *
 * Unset → `min(DEFAULT_MIN_CHARS, maxChars / 4)`: 250 at the default target,
 * shrinking proportionally when the caller asked for small chunks, so a
 * defaulted floor can never swallow the chunk size the caller chose.
 *
 * An EXPLICIT floor is validated instead of clamped: a floor at or above the
 * target would merge every chunk into its neighbour, which is a configuration
 * contradiction, not a preference — refused by name rather than rounded away.
 */
export function resolveMinChars(minChars, maxChars) {
    if (minChars === undefined) {
        return Math.min(DEFAULT_MIN_CHARS, Math.floor(maxChars / 4));
    }
    if (!Number.isFinite(minChars) || minChars < 0) {
        throw new Error(`minChars must be a non-negative number of characters — received ${String(minChars)}. ` +
            `Pass 0 to disable the floor (heading-only chunks are still never emitted).`);
    }
    if (minChars >= maxChars) {
        throw new Error(`minChars (${minChars}) must be smaller than maxChars (${maxChars}) — a floor at or ` +
            `above the target chunk size would merge every chunk into its neighbour. Lower ` +
            `minChars, or raise maxChars.`);
    }
    return minChars;
}
/**
 * Merge sub-floor spans FORWARD into their next neighbour (8.20.0).
 *
 * For heading-less span families (paragraphs, and `byHeading`'s no-headings
 * fallback). A span under the floor is never dropped and never shipped alone:
 *
 *   - adjacent short spans accumulate; the moment the accumulation reaches the
 *     floor it is emitted as its own span (two 150-char paragraphs make a fine
 *     300-char chunk — better than pushing both into a full-sized neighbour);
 *   - an accumulation still under the floor joins the NEXT full span, so the
 *     short text leads the merged chunk;
 *   - at the end of the document there is no next, so a trailing accumulation
 *     merges BACKWARD into the last emitted span;
 *   - a document that is nothing but short spans stays one merged span —
 *     there is no neighbour, and "never dropped" wins.
 *
 * Merged spans slice the SOURCE (start of the first constituent to end of the
 * last), so the offset invariant holds by construction, as everywhere else.
 */
export function mergeShortSpansForward(spans, minChars) {
    if (minChars <= 0 || spans.length <= 1)
        return [...spans];
    const out = [];
    let pendingStart;
    let lastEnd = 0;
    for (const span of spans) {
        lastEnd = span.end;
        if (span.end - span.start < minChars && pendingStart === undefined) {
            pendingStart = span.start;
            continue;
        }
        if (pendingStart !== undefined) {
            // Accumulating: emit as soon as the accumulation clears the floor,
            // whether the closer was itself short or full-sized.
            if (span.end - pendingStart >= minChars) {
                out.push({ start: pendingStart, end: span.end });
                pendingStart = undefined;
            }
            continue;
        }
        out.push(span);
    }
    if (pendingStart !== undefined) {
        const previous = out.pop();
        out.push(previous === undefined
            ? { start: pendingStart, end: lastEnd }
            : { start: previous.start, end: lastEnd });
    }
    return out;
}
/** Turn spans into pieces, slicing the text so the invariant holds by construction. */
export function toPieces(doc, spans) {
    const pageAt = pageLocator(doc);
    const pieces = [];
    for (const span of spans) {
        const trimmed = trimSpan(doc.text, span);
        if (trimmed === undefined)
            continue;
        // See `Span.contentStart`: the page is the one this chunk is about.
        const page = pageAt(span.contentStart ?? trimmed.start);
        pieces.push({
            // Sliced, never concatenated. This is the invariant.
            text: doc.text.slice(trimmed.start, trimmed.end),
            charStart: trimmed.start,
            charEnd: trimmed.end,
            ...(trimmed.heading !== undefined && { heading: trimmed.heading }),
            ...(page !== undefined && { page }),
        });
    }
    return pieces;
}
/** Hard-cut a span that no packing could fit, at `maxChars` with overlap. */
export function hardCut(span, maxChars, overlapChars) {
    const out = [];
    const stride = Math.max(1, maxChars - overlapChars);
    for (let start = span.start; start < span.end; start += stride) {
        const end = Math.min(span.end, start + maxChars);
        out.push({ start, end, ...(span.heading !== undefined && { heading: span.heading }) });
        if (end >= span.end)
            break;
    }
    return out;
}
/** Split into paragraph spans on blank lines, keeping offsets. */
export function paragraphSpans(text) {
    const spans = [];
    const separator = /\n[ \t]*\n/g;
    let cursor = 0;
    let match;
    while ((match = separator.exec(text)) !== null) {
        spans.push({ start: cursor, end: match.index });
        cursor = match.index + match[0].length;
    }
    spans.push({ start: cursor, end: text.length });
    return spans.filter((s) => s.end > s.start);
}
//# sourceMappingURL=shared.js.map