@yogesh0333/yogiway-prompt/dist/chunker.js

Free & Open Source Prompt Optimization Library - Save 30-50% on AI API costs. Multi-language, multi-platform support.

"use strict";
/**
 * Smart Chunker - Intelligent text chunking for large prompts
 */
Object.defineProperty(exports, "__esModule", { value: true });
exports.chunkText = chunkText;
const tokenizer_1 = require("./tokenizer");
/**
 * Chunk text intelligently
 */
function chunkText(text, options) {
    const { maxTokens, overlap = 0, strategy = 'smart', preserveStructure = true, } = options;
    switch (strategy) {
        case 'paragraph':
            return chunkByParagraph(text, maxTokens, overlap);
        case 'sentence':
            return chunkBySentence(text, maxTokens, overlap);
        case 'word':
            return chunkByWord(text, maxTokens, overlap);
        case 'smart':
        default:
            return smartChunk(text, maxTokens, overlap, preserveStructure);
    }
}
/**
 * Smart chunking (preserves structure)
 */
function smartChunk(text, maxTokens, overlap, preserveStructure) {
    const chunks = [];
    let currentChunk = '';
    let currentStart = 0;
    let chunkIndex = 0;
    // Split by paragraphs first
    const paragraphs = text.split(/\n\s*\n/);
    for (const paragraph of paragraphs) {
        const testChunk = currentChunk + (currentChunk ? '\n\n' : '') + paragraph;
        const testTokens = (0, tokenizer_1.countTokens)(testChunk);
        if (testTokens > maxTokens) {
            // Current chunk + paragraph exceeds limit
            if (currentChunk) {
                // Save current chunk
                chunks.push({
                    text: currentChunk,
                    tokens: (0, tokenizer_1.countTokens)(currentChunk),
                    index: chunkIndex++,
                    start: currentStart,
                    end: currentStart + currentChunk.length,
                });
                // Start new chunk with overlap
                if (overlap > 0 && chunks.length > 0) {
                    const overlapText = getOverlap(currentChunk, overlap);
                    currentChunk = overlapText + '\n\n' + paragraph;
                    currentStart = currentStart + currentChunk.length - overlapText.length;
                }
                else {
                    currentChunk = paragraph;
                    currentStart = text.indexOf(paragraph, currentStart);
                }
            }
            else {
                // Paragraph itself is too large, split it
                const paraChunks = chunkBySentence(paragraph, maxTokens, 0);
                paraChunks.forEach((chunk) => {
                    chunks.push({
                        ...chunk,
                        index: chunkIndex++,
                        start: currentStart + chunk.start,
                        end: currentStart + chunk.end,
                    });
                });
                currentChunk = '';
            }
        }
        else {
            currentChunk = testChunk;
        }
    }
    // Add remaining chunk
    if (currentChunk) {
        chunks.push({
            text: currentChunk,
            tokens: (0, tokenizer_1.countTokens)(currentChunk),
            index: chunkIndex,
            start: currentStart,
            end: currentStart + currentChunk.length,
        });
    }
    return chunks;
}
/**
 * Chunk by paragraphs
 */
function chunkByParagraph(text, maxTokens, overlap) {
    const paragraphs = text.split(/\n\s*\n/);
    const chunks = [];
    let currentChunk = '';
    let currentStart = 0;
    let chunkIndex = 0;
    for (const paragraph of paragraphs) {
        const testChunk = currentChunk + (currentChunk ? '\n\n' : '') + paragraph;
        const testTokens = (0, tokenizer_1.countTokens)(testChunk);
        if (testTokens > maxTokens) {
            if (currentChunk) {
                chunks.push({
                    text: currentChunk,
                    tokens: (0, tokenizer_1.countTokens)(currentChunk),
                    index: chunkIndex++,
                    start: currentStart,
                    end: currentStart + currentChunk.length,
                });
                currentChunk = paragraph;
                currentStart = text.indexOf(paragraph, currentStart);
            }
            else {
                // Paragraph too large, use sentence chunking
                return chunkBySentence(text, maxTokens, overlap);
            }
        }
        else {
            currentChunk = testChunk;
        }
    }
    if (currentChunk) {
        chunks.push({
            text: currentChunk,
            tokens: (0, tokenizer_1.countTokens)(currentChunk),
            index: chunkIndex,
            start: currentStart,
            end: currentStart + currentChunk.length,
        });
    }
    return chunks;
}
/**
 * Chunk by sentences
 */
function chunkBySentence(text, maxTokens, overlap) {
    const sentences = text.split(/([.!?]+\s+)/);
    const chunks = [];
    let currentChunk = '';
    let currentStart = 0;
    let chunkIndex = 0;
    for (let i = 0; i < sentences.length; i += 2) {
        const sentence = sentences[i] + (sentences[i + 1] || '');
        const testChunk = currentChunk + (currentChunk ? ' ' : '') + sentence;
        const testTokens = (0, tokenizer_1.countTokens)(testChunk);
        if (testTokens > maxTokens) {
            if (currentChunk) {
                chunks.push({
                    text: currentChunk,
                    tokens: (0, tokenizer_1.countTokens)(currentChunk),
                    index: chunkIndex++,
                    start: currentStart,
                    end: currentStart + currentChunk.length,
                });
                currentChunk = sentence;
                currentStart = text.indexOf(sentence, currentStart);
            }
            else {
                // Sentence too large, use word chunking
                return chunkByWord(text, maxTokens, overlap);
            }
        }
        else {
            currentChunk = testChunk;
        }
    }
    if (currentChunk) {
        chunks.push({
            text: currentChunk,
            tokens: (0, tokenizer_1.countTokens)(currentChunk),
            index: chunkIndex,
            start: currentStart,
            end: currentStart + currentChunk.length,
        });
    }
    return chunks;
}
/**
 * Chunk by words (last resort)
 */
function chunkByWord(text, maxTokens, overlap) {
    const words = text.split(/\s+/);
    const chunks = [];
    let currentChunk = '';
    let currentStart = 0;
    let chunkIndex = 0;
    for (const word of words) {
        const testChunk = currentChunk + (currentChunk ? ' ' : '') + word;
        const testTokens = (0, tokenizer_1.countTokens)(testChunk);
        if (testTokens > maxTokens) {
            if (currentChunk) {
                chunks.push({
                    text: currentChunk,
                    tokens: (0, tokenizer_1.countTokens)(currentChunk),
                    index: chunkIndex++,
                    start: currentStart,
                    end: currentStart + currentChunk.length,
                });
                currentChunk = word;
                currentStart = text.indexOf(word, currentStart);
            }
            else {
                // Single word exceeds limit (very rare)
                chunks.push({
                    text: word,
                    tokens: (0, tokenizer_1.countTokens)(word),
                    index: chunkIndex++,
                    start: currentStart,
                    end: currentStart + word.length,
                });
                currentStart += word.length;
            }
        }
        else {
            currentChunk = testChunk;
        }
    }
    if (currentChunk) {
        chunks.push({
            text: currentChunk,
            tokens: (0, tokenizer_1.countTokens)(currentChunk),
            index: chunkIndex,
            start: currentStart,
            end: currentStart + currentChunk.length,
        });
    }
    return chunks;
}
/**
 * Get overlap text from end
 */
function getOverlap(text, overlapTokens) {
    const words = text.split(/\s+/);
    let overlapText = '';
    let tokens = 0;
    for (let i = words.length - 1; i >= 0 && tokens < overlapTokens; i--) {
        const word = words[i];
        const wordTokens = (0, tokenizer_1.countTokens)(word);
        if (tokens + wordTokens <= overlapTokens) {
            overlapText = word + (overlapText ? ' ' : '') + overlapText;
            tokens += wordTokens;
        }
        else {
            break;
        }
    }
    return overlapText;
}