context-crystallizer/dist/utils/token-counter.d.ts

AI Crystallization Engineering for Large Repositories - Transform massive repositories into crystallized, AI-consumable knowledge through systematic analysis and optimization. Crystallization extracts meaningful context from any readable files.

export declare class TokenCounter {
    private static readonly CHARS_PER_TOKEN;
    private static readonly CODE_CHARS_PER_TOKEN;
    static countTokens(text: string, isCode?: boolean): number;
    static countMarkdownTokens(markdown: string): number;
    static truncateToTokenLimit(text: string, maxTokens: number, isCode?: boolean): string;
    static optimizeContent(content: string, maxTokens: number): string;
}
//# sourceMappingURL=token-counter.d.ts.map