@bobmatnyc/ai-code-review/dist/analysis/tokens/TokenAnalyzer.d.ts

A TypeScript-based tool for automated code reviews using AI models from Google Gemini, Anthropic Claude, and OpenRouter

/**
 * @fileoverview Token analysis service for pre-review token counting and estimation.
 *
 * This module provides fast, provider-agnostic token counting and analysis functionality
 * to estimate token usage and costs before performing actual reviews.
 */
import { FileInfo } from '../../types/review';
/**
 * Result of token analysis for a single file
 */
export interface FileTokenAnalysis {
    /** Path to the file */
    path: string;
    /** Relative path to the file */
    relativePath: string | undefined;
    /** Number of tokens in the file */
    tokenCount: number;
    /** Size of file in bytes */
    sizeInBytes: number;
    /** Tokens per byte ratio (used for optimization analysis) */
    tokensPerByte: number;
}
/**
 * Result of token analysis for a set of files
 */
export interface TokenAnalysisResult {
    /** Analysis of individual files */
    files: FileTokenAnalysis[];
    /** Total number of tokens across all files */
    totalTokens: number;
    /** Total size of all files in bytes */
    totalSizeInBytes: number;
    /** Average tokens per byte across all files */
    averageTokensPerByte: number;
    /** Total number of files analyzed */
    fileCount: number;
    /** Token overhead for prompts, instructions, etc. */
    promptOverheadTokens: number;
    /** Estimated total token count including overhead */
    estimatedTotalTokens: number;
    /** Maximum context window size for the model */
    contextWindowSize: number;
    /** Whether the content exceeds the context window */
    exceedsContextWindow: boolean;
    /** Number of passes needed for multi-pass review */
    estimatedPassesNeeded: number;
    /** Chunking strategy recommendation */
    chunkingRecommendation: ChunkingRecommendation;
}
/**
 * Recommendation for chunking strategy
 */
export interface ChunkingRecommendation {
    /** Whether chunking is recommended */
    chunkingRecommended: boolean;
    /** Approximate file chunks for multi-pass processing */
    recommendedChunks: FileChunk[];
    /** Reason for chunking recommendation */
    reason: string;
}
/**
 * A chunk of files for multi-pass processing
 */
export interface FileChunk {
    /** Files in this chunk */
    files: string[];
    /** Estimated token count for this chunk */
    estimatedTokenCount: number;
    /** Priority of this chunk (higher = more important) */
    priority: number;
}
/**
 * Options for token analysis
 */
export interface TokenAnalysisOptions {
    /** Type of review being performed */
    reviewType: string;
    /** Name of the model being used */
    modelName: string;
    /** Whether to optimize for speed (less accurate) or precision */
    optimizeForSpeed?: boolean;
    /** Additional prompt overhead to consider */
    additionalPromptOverhead?: number;
    /** Context maintenance factor for multi-pass reviews (0-1) */
    contextMaintenanceFactor?: number;
    /** Safety margin factor for context window (0-1) */
    safetyMarginFactor?: number;
    /** Force single pass mode regardless of token count */
    forceSinglePass?: boolean;
}
/**
 * Service for analyzing token usage in files
 */
export declare class TokenAnalyzer {
    private static DEFAULT_PROMPT_OVERHEAD;
    private static DEFAULT_CONTEXT_MAINTENANCE_FACTOR;
    private static DEFAULT_SAFETY_MARGIN_FACTOR;
    private static DEFAULT_CONTEXT_WINDOW;
    /**
     * Get the context window size for a model
     * @param modelName Name of the model
     * @returns Context window size in tokens
     */
    private static getContextWindowSize;
    /**
     * Analyze token usage for a set of files
     * @param files Files to analyze
     * @param options Analysis options
     * @returns Token analysis result
     */
    static analyzeFiles(files: FileInfo[], options: TokenAnalysisOptions): TokenAnalysisResult;
    /**
     * Generate a chunking recommendation for files that exceed context window
     * @param fileAnalyses Array of file token analyses
     * @param estimatedTotalTokens Total tokens including overhead
     * @param contextWindowSize Maximum context window size
     * @param contextMaintenanceFactor Context maintenance overhead factor
     * @param forceSinglePass Force single pass mode regardless of token count
     * @returns Chunking recommendation
     */
    private static generateChunkingRecommendation;
    /**
     * Analyze a single file for token usage
     * @param file File to analyze
     * @param options Analysis options
     * @returns Token analysis for the file
     */
    static analyzeFile(file: FileInfo, options: TokenAnalysisOptions): FileTokenAnalysis;
}