@callmedayz/ai-prompt-toolkit
Version:
Professional AI prompt engineering toolkit with advanced template features, real-time dashboards, conditional logic, template inheritance, live monitoring, OpenRouter integration, and 310+ model support
161 lines • 6.07 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.TextChunker = void 0;
const token_counter_1 = require("./token-counter");
/**
* Text chunking utility for processing large texts with AI models
*/
class TextChunker {
/**
* Split text into chunks that fit within token limits
*/
static chunkText(text, options) {
const { maxTokens, overlap = 0, preserveWords = true, preserveSentences = false } = options;
// Handle empty text
if (!text || text.trim().length === 0) {
return text ? [text] : [];
}
if (preserveSentences) {
return this.chunkBySentences(text, maxTokens, overlap);
}
else if (preserveWords) {
return this.chunkByWords(text, maxTokens, overlap);
}
else {
return this.chunkByCharacters(text, maxTokens, overlap);
}
}
/**
* Chunk text by sentences, preserving sentence boundaries
*/
static chunkBySentences(text, maxTokens, overlap) {
const sentences = this.splitIntoSentences(text);
const chunks = [];
let currentChunk = [];
let currentTokens = 0;
for (const sentence of sentences) {
const sentenceTokens = token_counter_1.TokenCounter.estimateTokens(sentence).tokens;
if (currentTokens + sentenceTokens > maxTokens && currentChunk.length > 0) {
// Finalize current chunk
chunks.push(currentChunk.join(' '));
// Start new chunk with overlap
if (overlap > 0) {
currentChunk = this.getOverlapSentences(currentChunk, overlap);
currentTokens = token_counter_1.TokenCounter.estimateTokens(currentChunk.join(' ')).tokens;
}
else {
currentChunk = [];
currentTokens = 0;
}
}
currentChunk.push(sentence);
currentTokens += sentenceTokens;
}
if (currentChunk.length > 0) {
chunks.push(currentChunk.join(' '));
}
return chunks;
}
/**
* Chunk text by words, preserving word boundaries
*/
static chunkByWords(text, maxTokens, overlap) {
const words = text.split(/\s+/).filter(word => word.length > 0);
const chunks = [];
let currentChunk = [];
let currentTokens = 0;
for (const word of words) {
const wordTokens = token_counter_1.TokenCounter.estimateTokens(word).tokens;
if (currentTokens + wordTokens > maxTokens && currentChunk.length > 0) {
// Finalize current chunk
chunks.push(currentChunk.join(' '));
// Start new chunk with overlap
if (overlap > 0) {
const overlapWords = Math.min(overlap, currentChunk.length);
currentChunk = currentChunk.slice(-overlapWords);
currentTokens = token_counter_1.TokenCounter.estimateTokens(currentChunk.join(' ')).tokens;
}
else {
currentChunk = [];
currentTokens = 0;
}
}
currentChunk.push(word);
currentTokens += wordTokens;
}
if (currentChunk.length > 0) {
chunks.push(currentChunk.join(' '));
}
return chunks;
}
/**
* Chunk text by characters (least precise but fastest)
*/
static chunkByCharacters(text, maxTokens, overlap) {
const maxChars = maxTokens * 4; // Rough estimation: 1 token ≈ 4 characters
const overlapChars = overlap * 4;
const chunks = [];
for (let i = 0; i < text.length; i += maxChars - overlapChars) {
const chunk = text.slice(i, i + maxChars);
chunks.push(chunk);
if (i + maxChars >= text.length)
break;
}
return chunks;
}
/**
* Split text into sentences using common sentence endings
*/
static splitIntoSentences(text) {
// Simple sentence splitting - could be enhanced with more sophisticated NLP
return text
.split(/[.!?]+/)
.map(sentence => sentence.trim())
.filter(sentence => sentence.length > 0)
.map(sentence => sentence + '.');
}
/**
* Get overlap sentences for chunking
*/
static getOverlapSentences(sentences, overlapTokens) {
let tokens = 0;
const overlap = [];
for (let i = sentences.length - 1; i >= 0; i--) {
const sentenceTokens = token_counter_1.TokenCounter.estimateTokens(sentences[i]).tokens;
if (tokens + sentenceTokens > overlapTokens)
break;
overlap.unshift(sentences[i]);
tokens += sentenceTokens;
}
return overlap;
}
/**
* Chunk text for a specific model
*/
static chunkForModel(text, model, overlapPercent = 10) {
const config = token_counter_1.TokenCounter.getModelConfig(model);
const maxTokens = Math.floor(config.maxTokens * 0.9); // Leave 10% buffer
const overlap = Math.floor(maxTokens * (overlapPercent / 100));
return this.chunkText(text, {
maxTokens,
overlap,
preserveWords: true,
preserveSentences: true
});
}
/**
* Get chunk statistics
*/
static getChunkStats(chunks) {
const tokenCounts = chunks.map(chunk => token_counter_1.TokenCounter.estimateTokens(chunk).tokens);
return {
totalChunks: chunks.length,
averageTokens: Math.round(tokenCounts.reduce((a, b) => a + b, 0) / chunks.length),
minTokens: Math.min(...tokenCounts),
maxTokens: Math.max(...tokenCounts),
totalTokens: tokenCounts.reduce((a, b) => a + b, 0)
};
}
}
exports.TextChunker = TextChunker;
//# sourceMappingURL=text-chunker.js.map