agentfootprint
Version:
The explainable agent framework — backtrack a wrong answer to the exact context that caused it (evidence, not guesses). Built on footprintjs.
168 lines • 7.81 kB
TypeScript
/**
* indexDocuments — seed a vector-capable MemoryStore with documents.
*
* Embeds each document, builds a `MemoryEntry<{content, metadata?}>`,
* batches into `store.putMany()`. Used at application startup to
* populate a RAG store before the first agent run.
*
* Pattern: Bulk-write helper. Not a flowchart stage — it runs once
* at boot, not per-iteration.
* Role: Layer-3 RAG pipeline starter. Pairs with `defineRAG()`
* which only does the read side.
* Emits: N/A — startup-time batch write, not part of the agent run.
*
* @example
* ```ts
* import { Agent, indexDocuments, defineRAG } from 'agentfootprint'
import { InMemoryStore, mockEmbedder } from 'agentfootprint/memory';
*
* const store = new InMemoryStore();
* const embedder = mockEmbedder();
*
* await indexDocuments(store, embedder, [
* { id: 'doc1', content: 'Refunds processed within 3 business days.' },
* { id: 'doc2', content: 'Pro plan: $20/mo, includes priority support.', metadata: { tier: 'pro' } },
* { id: 'doc3', content: 'Free plan: limited to 100 calls/month.' },
* ]);
*
* const docs = defineRAG({ id: 'product-docs', store, embedder });
* const agent = Agent.create({ provider }).rag(docs).build();
* ```
*/
import type { Embedder } from '../../memory/embedding/index.js';
import type { MemoryStore } from '../../memory/store/index.js';
import type { MemoryIdentity } from '../../memory/identity/index.js';
/**
* A document to index. `id` must be unique within the store + identity.
*
* The passage rides on `content`. Since 8.19.0 `text` is accepted as the
* same thing, because the retrieval formatter reads both keys and a
* hand-built entry that spelled it `text` used to index and retrieve
* perfectly while rendering an EMPTY passage. One sane meaning, two
* spellings, and `indexDocuments` refuses a document carrying NEITHER —
* an unrenderable passage and an absent one are different facts.
*/
export type RagDocument = {
readonly id: string;
/** The passage. Use this one; `text` is the accepted alias. */
readonly content: string;
readonly text?: string;
readonly metadata?: Readonly<Record<string, unknown>>;
} | {
readonly id: string;
readonly content?: string;
/**
* The passage, spelled the way a `Chunk` spells it. Accepted so a
* corpus assembled by hand from `rag`-door chunks indexes without a
* rename. `content` wins when both are present.
*/
readonly text: string;
readonly metadata?: Readonly<Record<string, unknown>>;
};
export interface IndexDocumentsOptions {
/**
* Identity scope to write under. Default: a single shared
* `{ conversationId: '_global' }` namespace, suitable for app-wide
* corpora.
*
* **Multi-tenant footgun:** the read side (`agent.run({ identity })`)
* queries within whichever identity is passed at request time.
* If you index here under `_global` but query under
* `{ tenant: 'acme' }`, you'll get ZERO results — silently. Either:
* 1. Index every document under each tenant's identity (duplicated
* storage, but isolated), or
* 2. Index under `_global` AND query under `_global` (shared
* corpus across tenants — fine for product docs, NOT for
* tenant-private data), or
* 3. Use a vector store adapter that supports multi-namespace
* reads at query time (Pinecone, Qdrant — outside this helper's
* scope).
*/
readonly identity?: MemoryIdentity;
/**
* Stable id of the embedder. Stored on each entry so a future embedder swap
* doesn't silently mix similarity scores.
*
* Defaults to the embedder's own `id` (8.9.0 — every shipped embedder sets
* one), and to `'default-embedder'` for a hand-written `Embedder` that has
* none. That default matters more with a durable store: it is half of the
* `'<id>@<dims>'` fingerprint `sqliteVectorStore` records per vector and
* refuses on, so an index built by `staticEmbedder()` will not silently
* accept vectors from `openaiEmbedder()`.
*/
readonly embedderId?: string;
/**
* Optional tier tag to attach to indexed entries (`'hot'` /
* `'warm'` / `'cold'`). Useful when read-side `defineRAG` should
* filter to a subset of the corpus.
*/
readonly tier?: 'hot' | 'warm' | 'cold';
/**
* Optional TTL in milliseconds from indexing time. Useful for
* compliance retention windows (e.g., re-index quarterly).
*/
readonly ttlMs?: number;
/**
* Optional abort signal — embedders making network calls thread
* this through to abort batch indexing on shutdown / timeout.
*/
readonly signal?: AbortSignal;
/**
* Called once with the cost of the embedding work this call did (8.9.0).
*
* `indexDocuments` runs at STARTUP, outside any agent run, so it has no
* emit channel to ride — there is no scope, no dispatcher and no
* `runtimeStageId` to correlate against. Rather than pretend otherwise, it
* hands the same payload the in-run stages emit as
* `agentfootprint.embedding.generated` straight to you, so the index-time
* half of the cost model is reportable from a boot script:
*
* ```ts
* await indexDocuments(store, embedder, docs, {
* onEmbedding: (e) => console.log(`embedded ${e.count} documents in ${e.durationMs}ms`),
* });
* ```
*/
readonly onEmbedding?: (payload: import('../../events/payloads.js').EmbeddingGeneratedPayload) => void;
/**
* Max number of concurrent embed calls when the embedder doesn't
* implement `embedBatch`. Default `8`. Without this cap, a 10K-doc
* corpus would fire 10K parallel embed calls and trigger rate limits.
* Ignored when `embedBatch` is available (the embedder controls
* its own batching).
*/
readonly maxConcurrency?: number;
/**
* The embedder's input ceiling in characters (9.1.0). Documents longer than
* it are embedded anyway — the backend CLIPS them rather than refusing — and
* the run says so once on `console.warn`.
*
* Default: the embedder's own declared `maxInputChars`, falling back to
* 2,000 (the measured `localEmbedder` cliff) for one that declares none.
* An explicit number here wins over both.
*
* This helper does not split — a document you hand it is stored whole and
* served whole. So a document past the ceiling is indexed by its OPENING
* while the model is later shown all of it, and retrieval cannot find
* wording that is plainly there. Nothing throws; the corpus is quietly
* partially indexed. Cut such documents into chunks first — the
* `agentfootprint/rag` door does exactly that, and its chunks keep the
* coordinates a citation is checked against.
*/
readonly maxChunkChars?: number;
}
/**
* Embed + persist documents. Returns the count actually indexed
* (skips duplicates if the store rejects them). Throws on embedder
* failure or store error — fail loud at startup is desirable.
*
* **Re-indexing semantics:** entries are written with `version: 1` and
* `putMany` (most adapters: last-write-wins). Re-running this helper
* after the store has been mutated by other writers may stomp their
* versions. For idempotent corpus refresh, either delete-then-index
* or use a custom upsert via `store.putIfVersion()` per document. A
* first-class `mode: 'upsert' | 'replace'` API is planned for a
* future release.
*/
export declare function indexDocuments(store: MemoryStore, embedder: Embedder, documents: readonly RagDocument[], options?: IndexDocumentsOptions): Promise<number>;
//# sourceMappingURL=indexDocuments.d.ts.map