agentfootprint/dist/rag/indexCorpus.js

The explainable agent framework — backtrack a wrong answer to the exact context that caused it (evidence, not guesses). Built on footprintjs.

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.buildIndexChart = exports.indexCorpus = void 0;
/**
 * indexCorpus — building the index is itself a flowchart, so the run explains
 * itself.
 *
 * Pattern: a footprintjs chart. Not "a function that logs" — the stages, the
 *          decision, the fan-out and the report are the real thing, and the
 *          commit log IS the indexing report.
 * Role:    rag/ layer. Step three, and the one that ties the other two to a
 *          store.
 * Emits:   `agentfootprint.embedding.generated` (`inputKind: 'document'`) per
 *          embedded batch, through the emit channel.
 *
 * ── Why a chart and not a loop ──────────────────────────────────────────────
 * Indexing is where a corpus acquires the properties retrieval later depends
 * on, and every one of those is a question someone eventually asks at an
 * awkward moment: *why is this chunk in the index? why was that one skipped?
 * what did this run cost? which document went missing?* A `for` loop answers
 * none of them after it has finished. A chart answers all of them from its own
 * commit log, months later, without the caller having saved anything.
 *
 *   discover → load → split → plan (DECIDER)
 *            → take-window → embed (FAN-OUT + retry) → tally → more? ⟲
 *            → remove → report
 *
 * The **plan** stage is a real decider rather than an `if`, because "skip this
 * chunk" is a decision with evidence: the stored content hash equals the new
 * one AND the stored embedder fingerprint equals the current one. Both
 * predicates read from scope, so `decide()` captures the values that drove the
 * choice and `FlowRecorder.onDecision` carries them — which is what makes
 * "why was that one skipped?" answerable rather than merely plausible.
 *
 * The **embed** stage fans out with `addParallelForEach`, one branch per batch,
 * each with a declarative `retry`. A hand-rolled retry inside the stage would
 * be invisible: the narrative would show one stage and the two rate-limited
 * attempts before the successful one would leave no mark. Declared, every
 * attempt is in the record.
 *
 * It fans out over a WINDOW rather than over every batch, and loops. That is
 * not an optimisation: `maxBranches` truncates the extra items rather than
 * queueing them, so a single fan-out over 200 batches with a ceiling of 4
 * would embed 4 and report success. The window can never exceed the ceiling,
 * and the loop comes back for the rest.
 *
 * ── Pausing, and the honest limit on it ─────────────────────────────────────
 * A large corpus can exceed a budget mid-run, so the window loop is where a
 * long run can stop. **It can only stop at a WINDOW BOUNDARY**, and that is a
 * real constraint rather than a preference: `interrupt()` re-runs its stage
 * from the top on resume, so a pause inside an embedder call would re-embed
 * everything that call had already done. Every completed batch is committed
 * before the loop comes round, so a resume re-embeds at most the window that
 * was in flight.
 */
const footprintjs_1 = require("footprintjs");
const footprintjs_2 = require("footprintjs");
const inputCeiling_js_1 = require("../memory/embedding/inputCeiling.js");
const capability_js_1 = require("../memory/store/capability.js");
const emitEmbedding_js_1 = require("../memory/embedding/emitEmbedding.js");
const defineRAG_js_1 = require("./retriever/defineRAG.js");
const loadDocuments_js_1 = require("./loadDocuments.js");
const splitDocuments_js_1 = require("./splitDocuments.js");
const byHeading_js_1 = require("./splitters/byHeading.js");
/**
 * Build and run the indexing chart.
 *
 * @returns the report — which is also committed to the run's own log, so it is
 *   evidence rather than only a return value.
 *
 * @example
 * ```ts
 * const report = await indexCorpus({
 *   source: { dir: './docs' },
 *   store: sqliteVectorStore({ file: './corpus.db' }),
 *   embedder: staticEmbedder(),
 * });
 * // { discovered: 3, loaded: 3, chunks: 14, embedded: 14, skipped: 0, removed: 0, … }
 * ```
 */
async function indexCorpus(config) {
    const chart = buildIndexChart(config);
    // `run()` hands back `{ state, output, executionTree, commitLog }`. The
    // report is read off the final state — and it is ALSO in `commitLog`, which
    // is the copy that survives without the caller keeping this return value.
    const result = (await chart.run({}));
    const report = result?.state?.report;
    if (!report) {
        throw new Error('indexCorpus: the chart finished without producing a report.');
    }
    return report;
}
exports.indexCorpus = indexCorpus;
/**
 * The chart itself, exposed so a caller can mount it, attach recorders to it,
 * or run it under their own executor — the same freedom every other chart in
 * this library gives.
 */
function buildIndexChart(config) {
    return compile(config);
}
exports.buildIndexChart = buildIndexChart;
function compile(config) {
    // Before the chart exists, before a byte is embedded: a store that has
    // declared it cannot serve vectors back would make this whole run a
    // successful report about an unreachable corpus. Refused at BUILD, so
    // `buildIndexChart` refuses it too rather than handing back a chart that
    // is guaranteed to lie.
    (0, capability_js_1.assertServesVectors)(config.store, 'indexCorpus');
    const corpus = config.corpus ?? defineRAG_js_1.DEFAULT_CORPUS_IDENTITY;
    const splitter = config.splitter ?? (0, byHeading_js_1.byHeading)();
    const embedderId = config.embedderId ?? config.embedder.id ?? 'default-embedder';
    const batchSize = Math.max(1, Math.floor(config.batchSize ?? 64));
    const maxBatches = Math.max(1, Math.floor(config.maxConcurrentBatches ?? 4));
    const attempts = Math.max(1, Math.floor(config.attempts ?? 3));
    // The ceiling comes from the embedder unless the caller stated one. The
    // number lives where the knowledge is: an indexer's own default is a guess
    // about a backend it has never met, and the safe guess (the smallest cliff
    // any embedder might have) cuts every larger embedder short.
    const ceiling = (0, inputCeiling_js_1.resolveChunkCeiling)(config.maxChunkChars, config.embedder);
    const maxChunkChars = ceiling.chars;
    const removeMissing = config.removeMissing ?? true;
    const startedAt = Date.now();
    const fingerprint = `${embedderId}@${config.embedder.dimensions}`;
    return ((0, footprintjs_1.flowChart)('Discover + load', async (scope) => {
        const { documents, failed, discovered } = await (0, loadDocuments_js_1.loadDocuments)(config.source, {
            ...(config.loaders && { loaders: config.loaders }),
        });
        scope.discoveredCount = discovered;
        scope.documents = documents;
        scope.failed = failed;
    }, 'discover-load', { description: 'Walk the source and read every document a loader claims' })
        .addFunction('Split', (scope) => {
        const chunks = (0, splitDocuments_js_1.splitDocuments)(scope.documents ?? [], splitter);
        scope.chunks = chunks;
        scope.truncated = chunks
            .filter((c) => c.text.length > maxChunkChars)
            .map((c) => ({ id: c.id, chars: c.text.length }));
    }, 'split', `Cut every document with '${splitter.name}'`)
        // ── The plan. A decider, because "skip" is a decision with evidence. ──
        .addDeciderFunction('Plan', (async (scope) => {
        const chunks = scope.chunks ?? [];
        const wanted = new Map(chunks.map((c) => [c.id, c]));
        // What is already there, and is it still current? Two predicates,
        // both recorded: same content, same embedding space.
        const toEmbed = [];
        const toSkip = [];
        for (const chunk of chunks) {
            const existing = await config.store.get(corpus, chunk.id);
            const storedHash = existing?.value?.contentHash;
            const storedFingerprint = existing?.embeddingModel === undefined
                ? undefined
                : `${existing.embeddingModel}@${(existing.embedding ?? []).length}`;
            const sameContent = storedHash === chunk.contentHash;
            const sameEmbedder = storedFingerprint === fingerprint;
            if (existing && sameContent && sameEmbedder)
                toSkip.push(chunk.id);
            else
                toEmbed.push(chunk);
        }
        // Chunks in the index whose document is no longer in the source, or
        // which this document no longer produces (it got shorter).
        const toRemove = [];
        if (removeMissing) {
            const liveDocs = new Set((scope.documents ?? []).map((d) => d.uri));
            let cursor;
            do {
                const page = await config.store.list(corpus, {
                    limit: 500,
                    ...(cursor !== undefined && { cursor }),
                });
                for (const entry of page.entries) {
                    if (wanted.has(entry.id))
                        continue;
                    const docUri = entry.value?.docUri;
                    // Only ever remove chunks of documents this run actually saw
                    // the source for. A namespace shared with something else is
                    // not ours to prune.
                    if (docUri !== undefined && liveDocs.has(docUri))
                        toRemove.push(entry.id);
                    else if (docUri !== undefined && isFromThisCorpus(docUri, scope)) {
                        toRemove.push(entry.id);
                    }
                }
                cursor = page.cursor;
            } while (cursor !== undefined);
        }
        scope.toEmbed = toEmbed;
        scope.toSkip = toSkip;
        scope.toRemove = toRemove;
        scope.pendingBatches = chunk(toEmbed, batchSize);
        scope.embeddedCount = 0;
        return (0, footprintjs_2.decide)(scope, [
            {
                when: (s) => (s.toEmbed ?? []).length === 0 && (s.toRemove ?? []).length === 0,
                then: 'nothing-to-do',
                label: 'every chunk is already indexed with this embedder, and nothing was removed',
            },
            {
                when: (s) => (s.toSkip ?? []).length > 0,
                then: 'incremental',
                label: 'some chunks are unchanged — same content hash, same embedder fingerprint',
            },
        ], 'full-index');
    }), 'plan', 'Decide what to embed, what to skip, and what to remove')
        .addFunctionBranch('nothing-to-do', 'NothingToDo', noop, 'No work — index is current')
        .addFunctionBranch('incremental', 'Incremental', noop, 'Embed only what changed')
        .addFunctionBranch('full-index', 'FullIndex', noop, 'Embed everything discovered')
        // Back to the trunk: the branch above is the RECORD of which plan was
        // taken, and the work after it is the same whichever one fired — an
        // empty `toEmbed` fans out over nothing, which costs nothing and still
        // files a report.
        .end()
        // ── Embed, one WINDOW of batches at a time. ──────────────────────────
        // `maxBranches` is a hard ceiling, and footprintjs does not queue the
        // extra items — it TRUNCATES them. Fanning out over every batch at once
        // would therefore embed only the first `maxConcurrentBatches` of them
        // and report success: an index silently containing a fraction of the
        // corpus, which is the worst thing this release could ship.
        //
        // So the fan-out runs over a WINDOW that can never exceed the ceiling,
        // and the loop after it comes back for the next one. Bounded
        // parallelism, unbounded total work, nothing dropped.
        .addFunction('Take window', (scope) => {
        const pending = scope.pendingBatches ?? [];
        scope.window = pending.slice(0, maxBatches);
        scope.pendingBatches = pending.slice(maxBatches);
    }, 'take-window', `Take up to ${maxBatches} batches for this pass`)
        .addParallelForEach('Embed batches', 'embed', {
        items: (scope) => scope.window ?? [],
        branch: (batch, index) => (0, footprintjs_1.flowChart)(`Embed batch ${index + 1}`, async (scope) => {
            const batchChunks = (scope.item ?? batch);
            if (batchChunks.length === 0) {
                scope.written = 0;
                return;
            }
            const texts = batchChunks.map((c) => c.text);
            const startedEmbedAt = Date.now();
            const vectors = config.embedder.embedBatch
                ? await config.embedder.embedBatch({ texts })
                : await Promise.all(texts.map((text) => config.embedder.embed({ text })));
            (0, emitEmbedding_js_1.emitEmbedding)(scope, {
                model: embedderId,
                provider: 'custom',
                inputKind: 'document',
                dimension: config.embedder.dimensions,
                count: vectors.length,
                durationMs: Date.now() - startedEmbedAt,
            });
            const now = Date.now();
            const entries = batchChunks.map((c, i) => ({
                id: c.id,
                value: c,
                version: 1,
                createdAt: now,
                updatedAt: now,
                lastAccessedAt: now,
                accessCount: 0,
                ...(vectors[i] && { embedding: [...vectors[i]] }),
                embeddingModel: embedderId,
            }));
            // One transactional write per batch — so a batch is all-in or
            // all-out, and a resumed run re-embeds at most this batch.
            await config.store.putMany(corpus, entries);
            scope.written = entries.length;
        }, `embed-batch-${index}`, { description: `Embed + store ${batch.length} chunks` })
            // Declared, not hand-rolled: every attempt is in the record.
            .retry({
            attempts,
            backoffMs: (n) => Math.min(8000, 250 * 2 ** (n - 1)),
        })
            .build(),
        maxBranches: maxBatches,
        into: 'batchResults',
        // A failed batch must fail the RUN. Best-effort (the default) leaves a
        // half-indexed corpus that keeps answering and quietly cannot see the
        // documents that did not land — which reads as "the model does not know
        // that" rather than as a failure, and is the single most expensive way
        // for this to go wrong.
        failFast: true,
    })
        .addFunction('Tally window', (scope) => {
        // Counted from what the batches actually WROTE, never from what the
        // plan intended: a report that says "embedded 3" because three were
        // queued cannot be trusted about the one number anybody checks.
        //
        // A fan-out branch's slot holds that branch's final STATE, so the
        // count comes off `written` — the field the branch sets after the
        // store call returned.
        const written = (scope.batchResults ?? []).reduce((total, result) => {
            const n = result?.written;
            return total + (typeof n === 'number' ? n : 0);
        }, 0);
        scope.embeddedCount = (scope.embeddedCount ?? 0) + written;
    }, 'tally-window', 'Add this window to the running total')
        // More batches? Go round again. This loop is also the ONLY point at
        // which a long indexing run can be interrupted safely: every completed
        // batch is already committed, so a resume re-embeds at most the window
        // that was in flight.
        .addDeciderFunction('More batches?', ((scope) => (0, footprintjs_2.decide)(scope, [
        {
            when: (s) => (s.pendingBatches ?? []).length > 0,
            then: 'more-batches',
            label: 'batches remain — take another window',
        },
    ], 'batches-done')), 'more-batches-decider', 'Is there another window of batches to embed?')
        .addFunctionBranch('more-batches', 'MoreBatches', noop, 'Another window remains')
        .loopTo('take-window')
        .addFunctionBranch('batches-done', 'BatchesDone', noop, 'Every batch is embedded')
        .end()
        .addFunction('Store removals', async (scope) => {
        let removed = 0;
        for (const id of scope.toRemove ?? []) {
            await config.store.delete(corpus, id);
            removed += 1;
        }
        scope.removedCount = removed;
    }, 'store-removals', 'Delete chunks whose document changed or disappeared')
        .addFunction('Report', (scope) => {
        // The report is a COMMIT, not just a return value. A later reader
        // asks the trace what this run did; nothing had to be saved.
        const truncated = scope.truncated ?? [];
        scope.report = {
            discovered: scope.discoveredCount ?? 0,
            loaded: (scope.documents ?? []).length,
            chunks: (scope.chunks ?? []).length,
            embedded: scope.embeddedCount ?? 0,
            skipped: (scope.toSkip ?? []).length,
            removed: scope.removedCount ?? 0,
            failed: scope.failed ?? [],
            truncated,
            truncatedCount: truncated.length,
            embedderFingerprint: fingerprint,
            splitter: splitter.name,
            elapsedMs: Date.now() - startedAt,
        };
        // Said OUT LOUD, once, at the end of the run — because an invisible
        // failure is indistinguishable from success. The list has been in
        // the report since 8.10.0 and nobody reads a report that says
        // everything went fine; the corpus that is quietly missing a
        // paragraph is discovered months later as "the model does not know
        // that". One line, at the one moment the count is final.
        if (truncated.length > 0) {
            console.warn((0, inputCeiling_js_1.truncationWarning)({
                caller: 'indexCorpus',
                count: truncated.length,
                total: (scope.chunks ?? []).length,
                noun: 'chunk',
                ceiling,
                embedderId,
                idsIn: "the returned report's `truncated` list",
            }));
        }
    }, 'report', 'File what this run did — the commit log IS the report')
        .build());
}
const noop = () => {
    /* The branch itself is the record; the work is downstream. */
};
/** Was this document uri produced by the source this run walked? */
function isFromThisCorpus(docUri, scope) {
    // A chunk whose document was in the index and is NOT in this run's document
    // list came from a document that disappeared — provided this run walked the
    // same source. `documents` empty means the source turned up nothing, and
    // pruning the whole namespace on an empty walk is how a typo in a path
    // deletes a corpus, so it does not.
    return (scope.documents ?? []).length > 0 && !docUri.startsWith('');
}
function chunk(items, size) {
    const out = [];
    for (let i = 0; i < items.length; i += size)
        out.push([...items.slice(i, i + size)]);
    return out;
}
//# sourceMappingURL=indexCorpus.js.map