zon-format
Version:
ZON: The most token-efficient serialization format for LLMs - beats CSV, TOON, JSON, and all competitors
29 lines (28 loc) • 779 B
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.TokenCounter = void 0;
class TokenCounter {
/**
* Estimates the number of tokens in a string.
* Uses a simple heuristic: ~4 characters per token.
*
* @param text - Text to count tokens for
* @returns Estimated token count
*/
count(text) {
if (!text)
return 0;
return Math.ceil(text.length / 4);
}
/**
* Estimates tokens for a specific model (placeholder for future expansion).
*
* @param text - Text to count
* @param model - Model identifier
* @returns Estimated token count
*/
countForModel(text, model) {
return this.count(text);
}
}
exports.TokenCounter = TokenCounter;