repomix/lib/core/metrics/TokenCounter.js

A tool to pack repository contents to single file for AI consumption

import { get_encoding } from 'tiktoken';
import { logger } from '../../shared/logger.js';
export class TokenCounter {
    constructor(encodingName) {
        const startTime = process.hrtime.bigint();
        // Setup encoding with the specified model
        this.encoding = get_encoding(encodingName);
        const endTime = process.hrtime.bigint();
        const initTime = Number(endTime - startTime) / 1e6; // Convert to milliseconds
        logger.debug(`TokenCounter initialization took ${initTime.toFixed(2)}ms`);
    }
    countTokens(content, filePath) {
        try {
            return this.encoding.encode(content).length;
        }
        catch (error) {
            let message = '';
            if (error instanceof Error) {
                message = error.message;
            }
            else {
                message = String(error);
            }
            if (filePath) {
                logger.warn(`Failed to count tokens. path: ${filePath}, error: ${message}`);
            }
            else {
                logger.warn(`Failed to count tokens. error: ${message}`);
            }
            return 0;
        }
    }
    free() {
        this.encoding.free();
    }
}
//# sourceMappingURL=TokenCounter.js.map