@light-merlin-dark/tok
Version:
Fast token estimation and cost calculation for enterprise LLMs with CLI and MCP support
207 lines (202 loc) • 7.84 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
const core_1 = require("../shared/core");
const common_1 = require("../common");
const errors_1 = require("../shared/errors");
const index_1 = require("../../index");
const fs = __importStar(require("fs"));
const tracker_1 = require("../utils/tracker");
const estimateCommand = (0, core_1.createCommand)({
description: 'Estimate token count for given text',
help: `
Usage: tok estimate <text> [options]
Description:
Estimates the token count for the provided text and calculates cost based on model pricing.
By default, uses fast character-based estimation (chars/4). Use --exact for precise counting.
Arguments:
text Text to estimate tokens for (or file path with --file)
Options:
-m, --model <model> Model for cost calculation (default: gpt-4o)
-e, --exact Use exact token counting (requires tiktoken)
-f, --file Treat text argument as file path
--track Add to cost tracking session
--format <format> Output format: json, table, human (default: human)
-v, --verbose Enable verbose output
Examples:
tok estimate "Hello world"
tok estimate "Your prompt" --model gpt-4o --exact
tok estimate prompt.txt --file --track
tok estimate "Text" --format json
Available Models:
gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo,
claude-3-opus, claude-3-sonnet, claude-3-haiku,
llama-3-70b, llama-3-8b, mixtral-8x7b
`,
options: [
{
flag: 'm|model',
description: 'Model to calculate cost for',
type: 'string',
default: 'gpt-4o'
},
{
flag: 'e|exact',
description: 'Use exact token counting',
type: 'boolean'
},
{
flag: 'f|file',
description: 'Treat text as file path',
type: 'boolean'
},
{
flag: 'track',
description: 'Add to cost tracking',
type: 'boolean'
},
{
flag: 'format',
description: 'Output format',
type: 'string',
default: 'human'
}
],
async execute(args, options, ctx) {
try {
const opts = options;
const verbose = opts.verbose || ctx.verbose;
const text = args[0];
if (!text) {
throw (0, errors_1.createCommandError)(errors_1.ErrorCode.INVALID_ARGUMENT, 'Please provide text to estimate');
}
// Get content
let content = text;
if (opts.file) {
if (verbose) {
common_1.logger.debug(`Reading from file: ${text}`, true);
}
try {
content = fs.readFileSync(text, 'utf-8');
}
catch (error) {
throw (0, errors_1.createCommandError)(errors_1.ErrorCode.FILE_NOT_FOUND, `Could not read file: ${text}`);
}
}
// Estimate tokens
let tokens;
let method;
if (opts.exact) {
const estimator = new index_1.TiktokenEstimator();
try {
await estimator.initialize();
tokens = estimator.estimate(content);
estimator.dispose();
method = 'exact';
}
catch (error) {
if (verbose) {
common_1.logger.warn('Tiktoken not available, falling back to estimation');
}
const charEstimator = new index_1.CharDivEstimator();
tokens = charEstimator.estimate(content);
method = 'estimate (tiktoken unavailable)';
}
}
else {
const estimator = new index_1.CharDivEstimator();
tokens = estimator.estimate(content);
method = 'estimate';
}
// Calculate cost
const prices = new index_1.PriceTable();
const model = opts.model || 'gpt-4o';
const modelPrice = prices.get(model);
let cost = null;
let costFormatted = 'N/A';
if (modelPrice) {
cost = index_1.CostCalculator.cost(tokens, modelPrice.prompt);
costFormatted = index_1.CostCalculator.formatCost(cost);
// Track if requested
if (opts.track) {
const tracker = (0, tracker_1.getTracker)();
tracker.add(model, tokens, 0, modelPrice);
if (verbose) {
common_1.logger.debug('Added to cost tracking', true);
}
}
}
else {
common_1.logger.warn(`No pricing available for model: ${model}`);
}
// Format result
const result = {
text_length: content.length,
tokens,
model,
cost: costFormatted,
method
};
// Output based on format
switch (opts.format) {
case 'json':
(0, common_1.formatJson)(result);
break;
case 'table':
(0, common_1.formatTable)(result);
break;
default:
(0, common_1.formatHuman)('Token Estimation Results', [
{ label: 'Text length', value: `${result.text_length} characters` },
{ label: 'Tokens', value: `${result.tokens} (${result.method})` },
{ label: 'Model', value: result.model },
{ label: 'Cost', value: result.cost }
]);
}
return {
success: true,
data: result
};
}
catch (error) {
if (error instanceof Error && error.name === 'CommandError') {
throw error;
}
throw (0, errors_1.createCommandError)(errors_1.ErrorCode.PROCESSING_ERROR, `Failed to estimate tokens: ${error instanceof Error ? error.message : String(error)}`);
}
}
});
exports.default = estimateCommand;
//# sourceMappingURL=estimate.js.map