rxcc/lib/core/tokenCount/tokenCount.js

A tool to pack repository contents to single file for AI consumption

import { get_encoding } from 'tiktoken';
import { logger } from '../../shared/logger.js';
export class TokenCounter {
    constructor(encodingName) {
        // Setup encoding with the specified model
        this.encoding = get_encoding(encodingName);
    }
    countTokens(content, filePath) {
        try {
            return this.encoding.encode(content).length;
        }
        catch (error) {
            let message = '';
            if (error instanceof Error) {
                message = error.message;
            }
            else {
                message = String(error);
            }
            if (filePath) {
                logger.warn(`Failed to count tokens. path: ${filePath}, error: ${message}`);
            }
            else {
                logger.warn(`Failed to count tokens. error: ${message}`);
            }
            return 0;
        }
    }
    free() {
        this.encoding.free();
    }
}
//# sourceMappingURL=tokenCount.js.map