@adonis-agora/agent/dist/src/rag/chunk.js

Governed, durable-backed AI agent (chat + tool-calling + governance) for AdonisJS — part of the Agora ecosystem.

/**
 * Splits text into overlapping chunks for embedding. Greedy up to `chunkSize`, but prefers to break on
 * a paragraph → sentence → word boundary in the back half of the window (so a chunk rarely cuts
 * mid-sentence), then carries `overlap` characters into the next chunk to preserve context across the
 * seam. Pure and deterministic.
 *
 * Pass {@link ChunkOptions.separator} for text whose record boundaries are known rather than guessed;
 * that path cuts only on the separator, and its two deviations (`chunkSize` becomes a target,
 * `overlap` is spent on whole records) are documented there.
 */
export function chunkText(text, options = {}) {
    const chunkSize = options.chunkSize ?? 800;
    const overlap = options.overlap ?? 100;
    const separator = options.separator;
    if (separator !== undefined && separator.length > 0) {
        return chunkRecords(text, separator, chunkSize, overlap);
    }
    const normalized = text.trim();
    if (normalized.length === 0) {
        return [];
    }
    if (normalized.length <= chunkSize) {
        return [normalized];
    }
    const chunks = [];
    let start = 0;
    while (start < normalized.length) {
        const end = Math.min(start + chunkSize, normalized.length);
        let breakAt = end;
        if (end < normalized.length) {
            const window = normalized.slice(start, end);
            const boundary = Math.max(window.lastIndexOf('\n\n'), window.lastIndexOf('. '), window.lastIndexOf(' '));
            // Only honour a boundary in the back half, else a tiny chunk cascades.
            if (boundary > chunkSize / 2) {
                breakAt = start + boundary + 1;
            }
        }
        const chunk = normalized.slice(start, breakAt).trim();
        if (chunk.length > 0) {
            chunks.push(chunk);
        }
        if (breakAt >= normalized.length) {
            break;
        }
        // Step forward with overlap, but always make progress (guards against a stuck boundary).
        start = Math.max(breakAt - overlap, start + 1);
    }
    return chunks;
}
/**
 * The {@link ChunkOptions.separator} path: pack WHOLE records into chunks. Records are the atoms here —
 * no chunk ever contains a fragment of one — so `chunkSize` is honoured by taking fewer records rather
 * than by cutting one, and `overlap` is honoured by repeating trailing records rather than by repeating
 * characters. Pure and deterministic; see {@link ChunkOptions.separator} for the two trades.
 */
function chunkRecords(text, separator, chunkSize, overlap) {
    const records = text
        .split(separator)
        .map((record) => record.trim())
        .filter((record) => record.length > 0);
    if (records.length === 0) {
        return [];
    }
    const chunks = [];
    let start = 0;
    while (start < records.length) {
        // Pack forward while the joined length fits. The FIRST record is taken unconditionally: that is
        // what emits an over-long record whole instead of cutting it (trade 1).
        let end = start;
        let length = 0;
        while (end < records.length) {
            const record = records[end] ?? '';
            const cost = end === start ? record.length : separator.length + record.length;
            if (end > start && length + cost > chunkSize) {
                break;
            }
            length += cost;
            end += 1;
        }
        chunks.push(records.slice(start, end).join(separator));
        if (end >= records.length) {
            break;
        }
        // Overlap as a whole-record budget: walk back over trailing records while they fit in `overlap`.
        // `back > start + 1` keeps at least one record consumed by this chunk, so `start` strictly
        // increases — an over-long record (which always chunks alone) is therefore never carried forward,
        // and no `overlap`, however large, can stall the loop.
        let back = end;
        let carried = 0;
        while (back > start + 1) {
            const cost = (records[back - 1] ?? '').length + separator.length;
            if (carried + cost > overlap) {
                break;
            }
            carried += cost;
            back -= 1;
        }
        start = back;
    }
    return chunks;
}
//# sourceMappingURL=chunk.js.map