@bobmatnyc/ai-code-review/dist/tokenizers/baseTokenizer.d.ts

A TypeScript-based tool for automated code reviews using AI models from Google Gemini, Anthropic Claude, and OpenRouter

/**
 * @fileoverview Base interface for tokenizers.
 *
 * This module defines the common interface that all tokenizer implementations
 * must follow, ensuring consistent behavior across different AI providers.
 */
/**
 * Base interface for tokenizers
 *
 * This interface defines the common methods that all tokenizer implementations
 * must provide. Tokenizers are responsible for counting the number of tokens
 * in a text string according to the tokenization rules of a specific AI model.
 *
 * Different AI models use different tokenization algorithms, so we need separate
 * implementations for each model family (e.g., GPT, Claude, Gemini).
 */
export interface Tokenizer {
    /**
     * Count the number of tokens in a text
     * @param text Text to count tokens for
     * @returns Actual token count
     */
    countTokens(text: string): number;
    /**
     * Get the model name or family this tokenizer is for
     * @returns Model name or family
     */
    getModelName(): string;
    /**
     * Check if this tokenizer supports a given model
     * @param modelName Name of the model to check
     * @returns True if the model is supported, false otherwise
     */
    supportsModel(modelName: string): boolean;
}
/**
 * Factory function type for creating tokenizers
 */
export type TokenizerFactory = () => Tokenizer;
/**
 * Registry of tokenizers
 */
export declare class TokenizerRegistry {
    private static tokenizers;
    /**
     * Register a tokenizer
     * @param tokenizer Tokenizer to register
     */
    static register(tokenizer: Tokenizer): void;
    /**
     * Get a tokenizer for a specific model
     * @param modelName Name of the model
     * @returns Tokenizer for the model, or undefined if none found
     */
    static getTokenizer(modelName: string): Tokenizer | undefined;
    /**
     * Get all registered tokenizers
     * @returns Array of all registered tokenizers
     */
    static getAllTokenizers(): Tokenizer[];
}
/**
 * Fallback tokenizer that uses a simple character-based approximation
 */
export declare class FallbackTokenizer implements Tokenizer {
    /**
     * Count the number of tokens in a text using a simple approximation
     * @param text Text to count tokens for
     * @returns Estimated token count
     */
    countTokens(text: string): number;
    /**
     * Get the model name for this tokenizer
     * @returns 'fallback'
     */
    getModelName(): string;
    /**
     * This tokenizer is used as a fallback for any model
     * @param _modelName Name of the model (unused but required by interface)
     * @returns Always true
     */
    supportsModel(_modelName: string): boolean;
}
/**
 * Get the appropriate tokenizer for a model
 * @param modelName Name of the model
 * @returns Tokenizer for the model (falls back to FallbackTokenizer if none found)
 */
export declare function getTokenizer(modelName: string): Tokenizer;
/**
 * Count tokens in a text using the appropriate tokenizer for a model
 *
 * This function is the main entry point for token counting. It selects the
 * appropriate tokenizer based on the model name and uses it to count tokens
 * in the provided text.
 *
 * The function handles model name normalization and tokenizer selection,
 * so callers don't need to worry about which specific tokenizer to use.
 *
 * @param text Text to count tokens for
 * @param modelName Name of the model (e.g., 'gpt-4', 'claude-3-opus', 'gemini-1.5-pro')
 * @returns Token count as determined by the model's tokenization rules
 * @example
 * // Count tokens for a GPT-4 model
 * const tokens = countTokens('Hello, world!', 'gpt-4');
 *
 * // Count tokens for a Claude model
 * const tokens = countTokens('Hello, world!', 'claude-3-opus');
 */
export declare function countTokens(text: string, modelName: string): number;