@cherrystudio/embedjs-utils/src/util/splitter.cjs

Useful util functions when extending the embedjs ecosystem

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.splitTextWithOffsets = splitTextWithOffsets;
const textsplitters_1 = require("@langchain/textsplitters");
async function splitTextWithOffsets(textToSplit, chunkSize, chunkOverlap) {
    const textSplitter = new textsplitters_1.RecursiveCharacterTextSplitter({
        chunkSize: chunkSize,
        chunkOverlap: chunkOverlap,
    });
    const textChunks = await textSplitter.splitText(textToSplit);
    const chunksWithOffsets = [];
    let currentSearchOffset = 0;
    for (const chunkText of textChunks) {
        const startOffset = textToSplit.indexOf(chunkText, currentSearchOffset);
        if (startOffset !== -1) {
            const endOffset = startOffset + chunkText.length;
            chunksWithOffsets.push({ text: chunkText, startOffset, endOffset });
            currentSearchOffset = startOffset + 1;
        }
    }
    return chunksWithOffsets;
}
//# sourceMappingURL=splitter.js.map