context-crystallizer
Version:
AI Crystallization Engineering for Large Repositories - Transform massive repositories into crystallized, AI-consumable knowledge through systematic analysis and optimization. Crystallization extracts meaningful context from any readable files.
59 lines • 2.72 kB
JavaScript
export class TokenCounter {
static CHARS_PER_TOKEN = 4; // Rough estimate for English text
static CODE_CHARS_PER_TOKEN = 3; // Code is typically more dense
static countTokens(text, isCode = false) {
const charsPerToken = isCode ? this.CODE_CHARS_PER_TOKEN : this.CHARS_PER_TOKEN;
return Math.ceil(text.length / charsPerToken);
}
static countMarkdownTokens(markdown) {
// Remove markdown syntax for more accurate token counting
const cleaned = markdown
.replace(/^#{1,6}\s+/gm, '') // Headers
.replace(/\*\*(.*?)\*\*/g, '$1') // Bold
.replace(/\*(.*?)\*/g, '$1') // Italic
.replace(/`([^`]+)`/g, '$1') // Inline code
.replace(/```[\s\S]*?```/g, (match) => {
// Count code blocks separately
const codeContent = match.replace(/```\w*\n?/, '').replace(/```$/, '');
return ' '.repeat(this.countTokens(codeContent, true) * this.CHARS_PER_TOKEN);
})
.replace(/^\s*[-*+]\s+/gm, '') // List items
.replace(/^\s*\d+\.\s+/gm, '') // Numbered lists
.replace(/\[([^\]]+)\]\([^)]+\)/g, '$1'); // Links
return this.countTokens(cleaned);
}
static truncateToTokenLimit(text, maxTokens, isCode = false) {
const estimatedTokens = this.countTokens(text, isCode);
if (estimatedTokens <= maxTokens) {
return text;
}
const charsPerToken = isCode ? this.CODE_CHARS_PER_TOKEN : this.CHARS_PER_TOKEN;
const maxChars = maxTokens * charsPerToken;
if (text.length <= maxChars) {
return text;
}
// Try to truncate at sentence or line boundaries
const truncated = text.substring(0, maxChars);
const lastSentence = truncated.lastIndexOf('.');
const lastNewline = truncated.lastIndexOf('\n');
if (lastSentence > maxChars * 0.8) {
return `${truncated.substring(0, lastSentence + 1)}...`;
}
else if (lastNewline > maxChars * 0.8) {
return `${truncated.substring(0, lastNewline)}\n...`;
}
else {
const lastSpace = truncated.lastIndexOf(' ');
return `${truncated.substring(0, lastSpace > maxChars * 0.8 ? lastSpace : maxChars)}...`;
}
}
static optimizeContent(content, maxTokens) {
// Remove excessive whitespace
content = content.replace(/\n\s*\n\s*\n/g, '\n\n');
content = content.replace(/[ \t]+/g, ' ');
content = content.trim();
// If still too long, truncate intelligently
return this.truncateToTokenLimit(content, maxTokens);
}
}
//# sourceMappingURL=token-counter.js.map