@cherrystudio/embedjs-utils/src/util/splitter.js

Useful util functions when extending the embedjs ecosystem

import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
export async function splitTextWithOffsets(textToSplit, chunkSize, chunkOverlap) {
    const textSplitter = new RecursiveCharacterTextSplitter({
        chunkSize: chunkSize,
        chunkOverlap: chunkOverlap,
    });
    const textChunks = await textSplitter.splitText(textToSplit);
    const chunksWithOffsets = [];
    let currentSearchOffset = 0;
    for (const chunkText of textChunks) {
        const startOffset = textToSplit.indexOf(chunkText, currentSearchOffset);
        if (startOffset !== -1) {
            const endOffset = startOffset + chunkText.length;
            chunksWithOffsets.push({ text: chunkText, startOffset, endOffset });
            currentSearchOffset = startOffset + 1;
        }
    }
    return chunksWithOffsets;
}
//# sourceMappingURL=splitter.js.map