agentfootprint/dist/esm/rag/indexCorpus.d.ts

The explainable agent framework — backtrack a wrong answer to the exact context that caused it (evidence, not guesses). Built on footprintjs.

import type { TypedScope } from 'footprintjs';
import type { Embedder } from '../memory/embedding/index.js';
import type { MemoryStore } from '../memory/store/index.js';
import type { MemoryIdentity } from '../memory/identity/index.js';
import type { Chunk, DocumentLoader, DocumentSource, FailedDocument, IndexReport, LoadedDocument, Splitter, TruncatedChunk } from './types.js';
export interface IndexCorpusConfig {
    /** Where the documents come from. */
    readonly source: DocumentSource;
    /** Where the vectors go. Any `MemoryStore`; `sqliteVectorStore` to keep them. */
    readonly store: MemoryStore;
    /** What turns chunk text into vectors. */
    readonly embedder: Embedder;
    /**
     * The namespace to index into. Defaults to the same
     * `{ conversationId: '_global' }` that `defineRAG` reads from, so index with
     * no options and retrieve with no options and the two meet.
     */
    readonly corpus?: MemoryIdentity;
    /**
     * How to cut the documents. Default `byHeading()` — the one strategy that
     * reads the document's own structure instead of inferring it.
     */
    readonly splitter?: Splitter;
    /** Loaders consulted before the built-ins. */
    readonly loaders?: readonly DocumentLoader[];
    /**
     * The embedder's id, stored per vector and refused on when it changes.
     * Defaults to `embedder.id`.
     */
    readonly embedderId?: string;
    /** Chunks per embed batch. Default 64. */
    readonly batchSize?: number;
    /**
     * How many batches embed at once. Default 4.
     *
     * A hard ceiling on PARALLELISM, not on total work: batches beyond it wait
     * for the next window rather than being dropped.
     */
    readonly maxConcurrentBatches?: number;
    /** Attempts per batch, including the first. Default 3. */
    readonly attempts?: number;
    /**
     * Delete chunks whose document is no longer in the source. Default true —
     * an index that answers from a document you deleted is worse than one that
     * does not answer.
     */
    readonly removeMissing?: boolean;
    /**
     * The embedder's input ceiling in characters. Chunks longer than this are
     * embedded anyway (the embedder clips them) and RECORDED in
     * `report.truncated` / `report.truncatedCount` — so silent half-embedding
     * becomes a number you can see, and a run that clipped anything says so once
     * on `console.warn`.
     *
     * **Default (9.1.0): the embedder's own declared `maxInputChars`**, falling
     * back to 2,000 — the measured `localEmbedder` cliff — for an embedder that
     * declares none. Passing a number here always wins over both: you are
     * allowed to know your corpus is denser than the embedder's own arithmetic
     * assumes.
     *
     * Before 9.1.0 this defaulted to 2,000 for EVERY embedder, which is the
     * on-device cliff applied to hosted models that read sixteen times as much.
     * Combined with a splitter told to cut larger chunks — `byHeading({ maxChars:
     * 2500 })` — the result was a corpus indexed by the OPENING of each chunk
     * while the whole chunk was served as the passage, so retrieval could not
     * find wording plainly visible in the block the model was shown.
     */
    readonly maxChunkChars?: number;
}
/** The chart's scope. Every field here is in the commit log. */
interface IndexState {
    discoveredCount: number;
    documents: readonly LoadedDocument[];
    failed: readonly FailedDocument[];
    chunks: readonly Chunk[];
    toEmbed: readonly Chunk[];
    toSkip: readonly string[];
    toRemove: readonly string[];
    /** The queue of batches still to embed. Drained one window at a time. */
    pendingBatches: readonly (readonly Chunk[])[];
    /** The window being fanned out over — never longer than `maxConcurrentBatches`. */
    window: readonly (readonly Chunk[])[];
    batchResults: readonly ({
        written?: number;
    } | undefined)[];
    embeddedCount: number;
    removedCount: number;
    truncated: readonly TruncatedChunk[];
    report?: IndexReport;
    [key: string]: unknown;
}
/**
 * Build and run the indexing chart.
 *
 * @returns the report — which is also committed to the run's own log, so it is
 *   evidence rather than only a return value.
 *
 * @example
 * ```ts
 * const report = await indexCorpus({
 *   source: { dir: './docs' },
 *   store: sqliteVectorStore({ file: './corpus.db' }),
 *   embedder: staticEmbedder(),
 * });
 * // { discovered: 3, loaded: 3, chunks: 14, embedded: 14, skipped: 0, removed: 0, … }
 * ```
 */
export declare function indexCorpus(config: IndexCorpusConfig): Promise<IndexReport>;
/**
 * The chart itself, exposed so a caller can mount it, attach recorders to it,
 * or run it under their own executor — the same freedom every other chart in
 * this library gives.
 */
export declare function buildIndexChart(config: IndexCorpusConfig): ReturnType<typeof compile>;
declare function compile(config: IndexCorpusConfig): import("footprintjs").RunnableFlowChart<any, TypedScope<IndexState>>;
export {};