agentfootprint/dist/esm/rag/splitters/byHeading.d.ts

The explainable agent framework — backtrack a wrong answer to the exact context that caused it (evidence, not guesses). Built on footprintjs.

/**
 * byHeading — cut where the author already said a section starts.
 *
 * Pattern: Strategy (one of `Splitter`).
 * Role:    rag/ layer. The default for Markdown, and the only splitter in this
 *          library that is **not a heuristic**.
 * Emits:   N/A.
 *
 * Every other strategy infers boundaries from typography. This one reads them:
 * a Markdown `#` line is the document telling you where a section begins, and
 * ignoring that in favour of a character count is throwing away the best
 * information in the file.
 *
 * Each chunk carries its heading, which flows through to the citation the
 * model sees — `<source id="refunds.md#3" heading="Refund timing">` — so a
 * reader can find the passage by name rather than by counting characters.
 *
 * A section longer than `maxChars` is packed by paragraph within the section
 * and, failing that, hard-cut; every piece keeps the heading it came from.
 * Text before the first heading (a preamble, a title block) is its own
 * heading-less section rather than being attached to a heading it precedes.
 *
 * ── The floor, and why merging goes FORWARD (8.20.0) ────────────────────────
 * A section whose own body is tiny does not retrieve badly — it retrieves TOO
 * WELL. Similarity is a density measure: a heading plus one preamble sentence
 * concentrates its topic's vocabulary with none of its substance. Measured in
 * a production corpus, a 180-character heading-and-preamble chunk outranked
 * the 1,032-character body of its own section at 0.430 — and the model,
 * handed a passage that PROMISES findings and contains none, fabricated a
 * plausible file path to fill the gap.
 *
 * So a section whose body is under `minChars` is merged into the chunk that
 * FOLLOWS it, under its own heading — the preamble sentence survives, leading
 * the chunk it introduces, and the citation still names the section a reader
 * would look up. Merging it BACKWARD would append it to the previous section's
 * chunk, where it introduces nothing and its heading would be lost; merging
 * forward is the direction the author's own document flows. The last section
 * has no next, so a trailing short section merges backward — the one edge
 * where that is the only honest option. Nothing is ever dropped.
 *
 * Two special cases are unconditional, independent of `minChars`:
 *   - a section that is heading-plus-whitespace is NEVER emitted alone. It is
 *     a coordinate, not a passage — the same distinction `indexDocuments`
 *     enforces one layer up when it refuses a passage-less document.
 *   - a document that is NOTHING but headings and whitespace yields no chunks
 *     at all: there is no passage anywhere in it to retrieve.
 */
import type { Splitter } from '../types.js';
export interface ByHeadingOptions {
    /** Target chunk size in characters. Default 1000. Sections longer than this are packed within. */
    readonly maxChars?: number;
    /** Backward overlap. Default 150. */
    readonly overlapChars?: number;
    /**
     * Deepest heading level that starts a new section (1 = `#` only, 6 = all).
     * Default 6 — every heading is a boundary. Lower it when a document uses
     * `####` for emphasis rather than structure.
     */
    readonly maxLevel?: number;
    /**
     * The floor under a section's own body, in characters (8.20.0). A section
     * whose body is shorter is merged FORWARD into the next chunk under its own
     * heading — never dropped, never shipped alone. Default
     * `min(250, maxChars / 4)`; see `DEFAULT_MIN_CHARS` for the field
     * measurement behind the number (a 180-char heading-and-preamble chunk that
     * outranked its own section's 1,032-char body and drove a fabricated
     * citation).
     *
     * `0` disables the floor. A heading-plus-whitespace section is still never
     * emitted alone — that refusal is unconditional, because a chunk with no
     * body at all is a coordinate, not a passage.
     */
    readonly minChars?: number;
}
/** One ATX heading line (`## Title`): where it starts and ends, its level, its title. */
export interface AtxHeading {
    /** Offset of the line's first `#`. */
    readonly offset: number;
    /** Offset just past the line's last character (before its line break). */
    readonly end: number;
    /** How many `#` open it, 1–6. */
    readonly level: number;
    readonly title: string;
}
/**
 * ATX headings only (`# Title`). Setext (`Title\n=====`) is rare in docs corpora.
 *
 * Exactly the lines `/^(#{1,6})[ \t]+(.+?)[ \t]*$/gm` matches, with the same
 * title, found in one pass. The regex is not used because a document is input
 * a deployer may not trust: on `# a` followed by a long run of blanks and one
 * more letter, its lazy title stops at every blank and re-reads the rest of the
 * run to see whether the line ends there — quadratic in the run (16,000 blanks
 * take ~0.2 s in V8).
 */
export declare function atxHeadings(text: string): AtxHeading[];
export declare function byHeading(options?: ByHeadingOptions): Splitter;