@adonis-agora/agent/dist/src/rag/ingest.js

Governed, durable-backed AI agent (chat + tool-calling + governance) for AdonisJS — part of the Agora ecosystem.

import { embedCountingUsage, } from '../spi/embedding-provider.js';
import { chunkText } from './chunk.js';
/**
 * Split documents into chunk records. Chunk ids are `${doc.id}#${index}`, so re-chunking the same
 * document produces the same ids (upsert overwrites in place rather than duplicating).
 */
export function chunkDocuments(documents, options = {}) {
    const chunks = [];
    for (const document of documents) {
        chunkText(document.text, options).forEach((text, index) => {
            chunks.push({
                id: `${document.id}#${index}`,
                text,
                ...(document.source !== undefined ? { source: document.source } : {}),
                ...(document.metadata !== undefined ? { metadata: document.metadata } : {}),
            });
        });
    }
    return chunks;
}
/** Embed pre-chunked records (one batched `embed`) and upsert them. Returns the record count. */
export async function ingestChunks(chunks, options) {
    if (chunks.length === 0) {
        return 0;
    }
    const { vectors: embeddings, usage } = await embedCountingUsage(options.embedder, chunks.map((chunk) => chunk.text));
    if (usage !== undefined && options.onUsage !== undefined) {
        await options.onUsage(usage);
    }
    const records = chunks.map((chunk, index) => ({
        ...chunk,
        embedding: embeddings[index] ?? [],
    }));
    await options.store.upsert(records);
    return records.length;
}
/** Chunk → embed → upsert in one call. Returns the number of chunks indexed. */
export async function ingestDocuments(documents, options) {
    return ingestChunks(chunkDocuments(documents, options), options);
}
//# sourceMappingURL=ingest.js.map