agentfootprint
Version:
The explainable agent framework — backtrack a wrong answer to the exact context that caused it (evidence, not guesses). Built on footprintjs.
197 lines • 8.78 kB
JavaScript
/**
* indexDocuments — seed a vector-capable MemoryStore with documents.
*
* Embeds each document, builds a `MemoryEntry<{content, metadata?}>`,
* batches into `store.putMany()`. Used at application startup to
* populate a RAG store before the first agent run.
*
* Pattern: Bulk-write helper. Not a flowchart stage — it runs once
* at boot, not per-iteration.
* Role: Layer-3 RAG pipeline starter. Pairs with `defineRAG()`
* which only does the read side.
* Emits: N/A — startup-time batch write, not part of the agent run.
*
* @example
* ```ts
* import { Agent, indexDocuments, defineRAG } from 'agentfootprint'
import { InMemoryStore, mockEmbedder } from 'agentfootprint/memory';
*
* const store = new InMemoryStore();
* const embedder = mockEmbedder();
*
* await indexDocuments(store, embedder, [
* { id: 'doc1', content: 'Refunds processed within 3 business days.' },
* { id: 'doc2', content: 'Pro plan: $20/mo, includes priority support.', metadata: { tier: 'pro' } },
* { id: 'doc3', content: 'Free plan: limited to 100 calls/month.' },
* ]);
*
* const docs = defineRAG({ id: 'product-docs', store, embedder });
* const agent = Agent.create({ provider }).rag(docs).build();
* ```
*/
import { resolveChunkCeiling, truncationWarning } from '../../memory/embedding/inputCeiling.js';
import { assertServesVectors } from '../../memory/store/capability.js';
import { chunkText } from '../../memory/retrieval/provenance.js';
const DEFAULT_IDENTITY = { conversationId: '_global' };
/**
* Embed + persist documents. Returns the count actually indexed
* (skips duplicates if the store rejects them). Throws on embedder
* failure or store error — fail loud at startup is desirable.
*
* **Re-indexing semantics:** entries are written with `version: 1` and
* `putMany` (most adapters: last-write-wins). Re-running this helper
* after the store has been mutated by other writers may stomp their
* versions. For idempotent corpus refresh, either delete-then-index
* or use a custom upsert via `store.putIfVersion()` per document. A
* first-class `mode: 'upsert' | 'replace'` API is planned for a
* future release.
*/
export async function indexDocuments(store, embedder, documents, options = {}) {
if (!store)
throw new Error('indexDocuments: `store` is required.');
if (!embedder)
throw new Error('indexDocuments: `embedder` is required.');
if (!Array.isArray(documents) || documents.length === 0)
return 0;
// A store that cannot serve vectors back is a corpus that indexes,
// reports success, and retrieves nothing — refused before anything is
// embedded, and therefore before anything is billed.
assertServesVectors(store, 'indexDocuments');
assertPassages(documents);
const identity = options.identity ?? DEFAULT_IDENTITY;
// The embedder's own name first: it is what a durable store fingerprints
// the namespace with, and an anonymous index cannot refuse a later swap.
const embedderId = options.embedderId ?? embedder.id ?? 'default-embedder';
const now = Date.now();
const embedStartedAt = Date.now();
const ttl = options.ttlMs ? now + options.ttlMs : undefined;
// Embed in batch when supported, else fall back to capped-concurrency
// single calls. Unlimited concurrency on a large corpus would
// saturate embedder rate limits; cap defaults to 8.
// Read through the SAME accessor the formatter reads with, so the bytes
// that get embedded and the bytes the model is later shown can never be
// two different things.
const texts = documents.map((d) => chunkText(d));
warnIfClipped(texts, embedder, options.maxChunkChars, embedderId);
let vectors;
if (embedder.embedBatch) {
vectors = await embedder.embedBatch({
texts,
...(options.signal && { signal: options.signal }),
});
}
else {
const limit = Math.max(1, options.maxConcurrency ?? 8);
vectors = await embedWithConcurrency(embedder, texts, limit, options.signal);
}
options.onEmbedding?.({
model: embedderId,
provider: 'custom',
inputKind: 'document',
dimension: embedder.dimensions,
count: vectors.length,
durationMs: Date.now() - embedStartedAt,
});
const entries = documents.map((doc, i) => {
const vec = vectors[i];
return {
id: doc.id,
value: doc,
version: 1,
createdAt: now,
updatedAt: now,
lastAccessedAt: now,
accessCount: 0,
...(vec && vec.length > 0 && { embedding: [...vec] }),
embeddingModel: embedderId,
...(ttl !== undefined && { ttl }),
...(options.tier && { tier: options.tier }),
source: { turn: 0, identity },
};
});
await store.putMany(identity, entries);
return entries.length;
}
/**
* Refuse a document that carries no passage at all.
*
* The formatter reads `content` OR `text` (8.19.0), so a corpus entry that
* spells it either way renders. One that spells it NEITHER has nothing to
* render, and until now that was discovered at read time as a
* perfectly-formed citation wrapped around an empty body — right document,
* right heading, right score, no passage, no error. An unrenderable passage
* and an absent one are different facts, and only one of them can be fixed
* by the person who wrote the corpus.
*
* It fires before anything is embedded, so a typo in a field name costs a
* message rather than an embedding bill. It names the ids (bounded — the
* first few are enough to find the bug) and never the values: a refused
* document is still the caller's data.
*/
function assertPassages(documents) {
const empty = [];
for (const doc of documents) {
if (chunkText(doc).trim().length === 0)
empty.push(doc.id || '(no id)');
}
if (empty.length === 0)
return;
const shown = empty.slice(0, 5).join(', ');
const more = empty.length > 5 ? `, and ${empty.length - 5} more` : '';
throw new Error(`indexDocuments: ${empty.length} document(s) carry no passage — neither \`content\` nor ` +
`\`text\` holds a non-empty string: ${shown}${more}.\n` +
` Indexing them would embed the empty string and store a citable id with nothing to ` +
`cite: retrieval would return the chunk, the prompt would show ` +
`<source id="..." doc="..." score="...">\\n\\n</source>, and the model would be handed ` +
`a passage's coordinates without the passage.\n` +
` Fix: put the passage on \`content\` (or \`text\` — both are read), or drop the ` +
`document from the batch.`);
}
/**
* Say out loud, once, that some of these documents will be embedded CLIPPED.
*
* The sibling door files the same fact in `report.truncated`; this one returns
* a count and has no report to file anything in, so the console is the only
* channel it has — and the alternative is what shipped before 9.1.0: nothing
* at all. A vector built from the opening of a document, stored against the
* whole document as its passage, is retrieval that cannot find text the model
* can see. It fires BEFORE the embedding call, so the warning arrives even
* when the backend then rejects the batch for the same reason.
*/
function warnIfClipped(texts, embedder, explicit, embedderId) {
const ceiling = resolveChunkCeiling(explicit, embedder);
const over = texts.filter((t) => t.length > ceiling.chars).length;
if (over === 0)
return;
console.warn(truncationWarning({
caller: 'indexDocuments',
count: over,
total: texts.length,
noun: 'document',
ceiling,
embedderId,
resplitHint: 'cut them into chunks before indexing (the `agentfootprint/rag` door splits with ' +
'coordinates, so each piece can still cite where it came from)',
}));
}
async function embedWithConcurrency(embedder, texts, limit, signal) {
const results = new Array(texts.length);
let next = 0;
const workers = Array.from({ length: Math.min(limit, texts.length) }, async () => {
for (;;) {
const i = next++;
if (i >= texts.length)
return;
// i bounded by texts.length above; texts[i] is defined.
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
const text = texts[i];
results[i] = await embedder.embed({
text,
...(signal && { signal }),
});
}
});
await Promise.all(workers);
return results;
}
//# sourceMappingURL=indexDocuments.js.map