vibe-coder-mcp
Version:
Production-ready MCP server with complete agent integration, multi-transport support, and comprehensive development automation tools for AI-assisted workflows.
167 lines (166 loc) • 5.95 kB
JavaScript
export class TokenEstimator {
static CHARS_PER_TOKEN = 4;
static WORDS_PER_TOKEN = 0.75;
static OVERHEAD_RATIOS = {
xml: 1.15,
json: 1.10,
markdown: 1.05,
code: 1.20,
plain: 1.0
};
static estimateTokens(text) {
if (!text || text.length === 0)
return 0;
const normalizedText = text.replace(/\s+/g, ' ').trim();
return Math.ceil(normalizedText.length / this.CHARS_PER_TOKEN);
}
static estimateTokensByWords(text) {
if (!text || text.length === 0)
return 0;
const trimmed = text.trim();
if (trimmed.length === 0)
return 0;
const words = trimmed.split(/\s+/).length;
return Math.ceil(words / this.WORDS_PER_TOKEN);
}
static estimateTokensAdvanced(text, contentType = 'plain') {
if (!text || text.length === 0) {
return {
estimatedTokens: 0,
confidence: 'high',
method: 'character_ratio',
breakdown: { contentTokens: 0 }
};
}
const charBasedTokens = this.estimateTokens(text);
const wordBasedTokens = this.estimateTokensByWords(text);
const baseTokens = Math.ceil((charBasedTokens + wordBasedTokens) / 2);
const overhead = this.OVERHEAD_RATIOS[contentType];
const finalTokens = Math.ceil(baseTokens * overhead);
const confidence = this.determineConfidence(text, charBasedTokens, wordBasedTokens);
return {
estimatedTokens: finalTokens,
confidence,
method: 'hybrid',
breakdown: {
contentTokens: baseTokens,
formattingTokens: finalTokens - baseTokens
}
};
}
static estimateFileTokens(filePath, content, contentType) {
const pathTokens = this.estimateTokens(filePath);
const detectedType = contentType || this.detectContentType(filePath);
const contentEstimation = this.estimateTokensAdvanced(content, detectedType);
const totalTokens = contentEstimation.estimatedTokens + pathTokens;
return {
filePath,
contentTokens: contentEstimation.estimatedTokens,
pathTokens,
totalTokens,
confidence: contentEstimation.confidence,
estimationMethod: `${contentEstimation.method}_with_path`
};
}
static validateTokenBudget(estimatedTokens, maxBudget) {
const utilizationPercentage = (estimatedTokens / maxBudget) * 100;
const remainingTokens = maxBudget - estimatedTokens;
const isValid = estimatedTokens <= maxBudget;
let warningLevel = 'none';
let recommendedAction = 'proceed';
if (utilizationPercentage >= 100) {
warningLevel = 'critical';
recommendedAction = 'reduce_scope';
}
else if (utilizationPercentage >= 90) {
warningLevel = 'high';
recommendedAction = 'optimize';
}
else if (utilizationPercentage >= 75) {
warningLevel = 'medium';
recommendedAction = 'optimize';
}
else if (utilizationPercentage >= 60) {
warningLevel = 'low';
recommendedAction = 'proceed';
}
return {
isValid,
utilizationPercentage: Math.round(utilizationPercentage * 100) / 100,
remainingTokens,
recommendedAction,
warningLevel
};
}
static estimateMultipleFiles(files) {
const fileEstimates = files.map(file => this.estimateFileTokens(file.path, file.content));
const totalTokens = fileEstimates.reduce((sum, estimate) => sum + estimate.totalTokens, 0);
let budgetRecommendation = 'suitable_for_standard_budget';
if (totalTokens > 100000) {
budgetRecommendation = 'requires_large_budget';
}
else if (totalTokens > 50000) {
budgetRecommendation = 'requires_medium_budget';
}
return {
totalTokens,
fileEstimates,
budgetRecommendation
};
}
static determineConfidence(text, charTokens, wordTokens) {
const variance = Math.abs(charTokens - wordTokens) / Math.max(charTokens, wordTokens);
if (variance < 0.1)
return 'high';
if (variance < 0.3)
return 'medium';
return 'low';
}
static detectContentType(filePath) {
const extension = filePath.split('.').pop()?.toLowerCase();
switch (extension) {
case 'xml':
case 'html':
case 'xhtml':
return 'xml';
case 'json':
case 'jsonl':
return 'json';
case 'md':
case 'markdown':
case 'rst':
return 'markdown';
case 'js':
case 'ts':
case 'jsx':
case 'tsx':
case 'py':
case 'java':
case 'cpp':
case 'c':
case 'cs':
case 'php':
case 'rb':
case 'go':
case 'rs':
case 'swift':
case 'kt':
return 'code';
default:
return 'plain';
}
}
static getEstimationStats(text) {
const lines = text.split('\n');
const words = text.trim().split(/\s+/);
return {
characterCount: text.length,
wordCount: words.length,
lineCount: lines.length,
charBasedTokens: this.estimateTokens(text),
wordBasedTokens: this.estimateTokensByWords(text),
averageWordsPerLine: Math.round((words.length / lines.length) * 100) / 100,
averageCharsPerWord: Math.round((text.length / words.length) * 100) / 100
};
}
}