gather-ts/dist/src/core/tokenization/TokenCounter.d.ts

A powerful code analysis and packaging tool designed for creating AI-friendly code representations for javascript and typescript projects.

import { ITokenCounter, ITokenCounterDeps, ITokenCounterOptions, ITokenCounterStats, ITokenCounterMetrics, ITokenizeOptions } from "./interfaces/ITokenCounter";
import { BaseService } from "@/types/services";
import { IFileWithContent, ISummaryStats } from "@/types";
export declare class TokenCounter extends BaseService implements ITokenCounter {
    private readonly deps;
    private encoder;
    private readonly debug;
    private readonly batchSize;
    private readonly modelOverride?;
    private readonly enableCache;
    private startTime;
    private stats;
    constructor(deps: ITokenCounterDeps, options?: ITokenCounterOptions);
    initialize(): Promise<void>;
    cleanup(): void;
    private initializeStats;
    private logDebug;
    private handleError;
    private getCurrentModel;
    private getEncoder;
    countTokens(filePath: string, text: string, options?: ITokenizeOptions): Promise<number>;
    private updateStats;
    generateSummary(files: IFileWithContent[]): Promise<ISummaryStats>;
    processBatch(files: IFileWithContent[], batchSize?: number): Promise<Array<{
        file: string;
        tokens: number;
    }>>;
    getStats(): ITokenCounterStats;
    getMetrics(): ITokenCounterMetrics;
    resetStats(): void;
    generateSummaryText(stats: ISummaryStats): string;
    printSummary(stats: ISummaryStats, outputFile: string): void;
}