ai-utils.js/model-function/tokenize-text/countTokens.js

Build AI applications, chatbots, and agents with JavaScript and TypeScript.

/**
 * Count the number of tokens in the given text.
 */
export async function countTokens(tokenizer, text) {
    return (await tokenizer.tokenize(text)).length;
}