@callmedayz/ai-prompt-toolkit
Version:
Professional AI prompt engineering toolkit with advanced template features, real-time dashboards, conditional logic, template inheritance, live monitoring, OpenRouter integration, and 310+ model support
141 lines • 5.33 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.TokenCounter = void 0;
const openrouter_models_1 = require("./openrouter-models");
const openrouter_types_1 = require("./openrouter-types");
const openrouter_client_1 = require("./openrouter-client");
const tokenization_service_1 = require("./tokenization-service");
/**
* Token counting utility for various AI models
*/
class TokenCounter {
/**
* Set the OpenRouter client for real tokenization
*/
static setClient(client) {
this.client = client;
this.tokenizationService = new tokenization_service_1.TokenizationService(client);
}
/**
* Create client from API key
*/
static setApiKey(apiKey) {
this.client = new openrouter_client_1.OpenRouterClient({ apiKey });
this.tokenizationService = new tokenization_service_1.TokenizationService(this.client);
}
/**
* Get accurate token count using OpenRouter's tokenization service
*/
static async getTokenCount(text, model = openrouter_types_1.DEFAULT_FREE_MODEL) {
if (!this.tokenizationService) {
// Fall back to estimation if no service is set
return this.estimateTokens(text, model);
}
return this.tokenizationService.getAccurateTokenCount(text, model);
}
/**
* Get detailed token count with native tokenizer information
*/
static async getDetailedTokenCount(text, model = openrouter_types_1.DEFAULT_FREE_MODEL) {
if (!this.tokenizationService) {
return { ...this.estimateTokens(text, model), generationId: undefined, nativeTokens: undefined };
}
return this.tokenizationService.getDetailedTokenCount(text, model);
}
/**
* Compare estimation accuracy against real API tokenization
*/
static async compareTokenCounts(text, model = openrouter_types_1.DEFAULT_FREE_MODEL) {
if (!this.tokenizationService) {
const estimated = this.estimateTokens(text, model);
return {
estimated,
actual: estimated,
difference: 0,
accuracy: 1
};
}
return this.tokenizationService.compareTokenCounts(text, model);
}
/**
* Estimate token count for a given text (fallback method)
* This is a rough estimation based on character count and word patterns
*/
static estimateTokens(text, model = openrouter_types_1.DEFAULT_FREE_MODEL) {
const characters = text.length;
const words = text.trim().split(/\s+/).filter(word => word.length > 0).length;
// Rough estimation: 1 token ≈ 4 characters for English text
// This varies by model and language, but provides a reasonable estimate
let tokens;
if (model.startsWith('gpt')) {
// GPT models: roughly 1 token per 4 characters
tokens = Math.ceil(characters / 4);
}
else if (model.startsWith('claude')) {
// Claude models: slightly different tokenization
tokens = Math.ceil(characters / 3.8);
}
else {
// Default estimation
tokens = Math.ceil(characters / 4);
}
const config = this.MODEL_CONFIGS[model];
const estimatedCost = config.costPerToken ? tokens * config.costPerToken : undefined;
return {
tokens,
characters,
words,
estimatedCost
};
}
/**
* Check if text fits within model's token limit
*/
static fitsInModel(text, model) {
const result = this.estimateTokens(text, model);
const config = this.MODEL_CONFIGS[model];
return result.tokens <= config.maxTokens;
}
/**
* Get model configuration
*/
static getModelConfig(model) {
return this.MODEL_CONFIGS[model];
}
/**
* Get all supported models
*/
static getSupportedModels() {
return Object.keys(this.MODEL_CONFIGS);
}
/**
* Find the best model for a given text length
*/
static recommendModel(text) {
const result = this.estimateTokens(text, openrouter_types_1.DEFAULT_FREE_MODEL);
if (result.tokens <= 8000) {
return { model: openrouter_types_1.DEFAULT_FREE_MODEL, reason: 'Text fits in free tier model (most cost-effective)' };
}
else if (result.tokens <= 32000) {
return { model: 'mistralai/mistral-small-3.2-24b-instruct', reason: 'Text requires medium context model' };
}
else if (result.tokens <= 100000) {
return { model: 'anthropic/claude-sonnet-4', reason: 'Text requires large context model' };
}
else {
return { model: 'anthropic/claude-opus-4', reason: 'Text requires very large context model' };
}
}
/**
* Calculate cost for processing text with a specific model
*/
static calculateCost(text, model) {
const result = this.estimateTokens(text, model);
return result.estimatedCost || 0;
}
}
exports.TokenCounter = TokenCounter;
TokenCounter.MODEL_CONFIGS = openrouter_models_1.MODEL_CONFIGS;
TokenCounter.client = null;
TokenCounter.tokenizationService = null;
//# sourceMappingURL=token-counter.js.map