ai-utils.js/model-function/tokenize-text/countTokens.cjs

Build AI applications, chatbots, and agents with JavaScript and TypeScript.

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.countTokens = void 0;
/**
 * Count the number of tokens in the given text.
 */
async function countTokens(tokenizer, text) {
    return (await tokenizer.tokenize(text)).length;
}
exports.countTokens = countTokens;