gemini-token-estimator/dist/index.d.ts

Estimate the number of tokens for Gemini models

/**
 * Tokenize a string similar to Gemma's tokenizer
 * @param input The text to tokenize
 * @returns The tokens
 * @example
 * const tokens = tokenize('Hello, world!')
 * console.log(tokens)
 * Output: ['Hello', ',', ' world', '!']
 */
export declare const tokenize: (input: string) => string[];
/**
 * Get the count of tokens in a text
 * @param text The text to tokenize
 * @returns The count of tokens in the text
 */
export declare const getTokenCount: (text: string) => number;
/**
 * Truncate text to a maximum number of tokens
 * @param text The text to truncate
 * @param maxTokens The maximum number of tokens to truncate to
 * @returns The truncated text
 */
export declare const truncateTextToMaxTokens: (text: string, maxTokens: number) => {
    truncatedText: string;
    truncatedTokenCount: number;
};