@yogesh0333/yogiway-prompt/dist/tokenizer-official.d.ts

Free & Open Source Prompt Optimization Library - Save 30-50% on AI API costs. Multi-language, multi-platform support.

/**
 * Official Tokenizer Integration
 * Supports tiktoken (OpenAI), anthropic tokenizer, and other official tokenizers
 */
import { LLMProvider } from "./types";
export interface OfficialTokenizer {
    encode(text: string): number[];
    decode(tokens: number[]): string;
    count(text: string): number;
}
/**
 * Official tokenizer integration with fallback
 * Attempts to use official tokenizers when available, falls back to estimation
 */
export declare class TokenizerManager {
    private tokenizers;
    /**
     * Register an official tokenizer for a provider
     */
    registerTokenizer(provider: LLMProvider, tokenizer: OfficialTokenizer): void;
    /**
     * Count tokens using official tokenizer if available, otherwise fallback
     */
    countTokens(text: string, provider: LLMProvider, fallback: (text: string, provider: LLMProvider) => number): number;
    /**
     * Check if official tokenizer is available
     */
    hasTokenizer(provider: LLMProvider): boolean;
}
/**
 * Tiktoken integration (OpenAI)
 * Usage: Install tiktoken package: npm install tiktoken
 */
export declare function createTiktokenTokenizer(model?: string): OfficialTokenizer | null;
/**
 * Anthropic tokenizer integration
 * Usage: Install @anthropic-ai/tokenizer package
 */
export declare function createAnthropicTokenizer(): OfficialTokenizer | null;
/**
 * Global tokenizer manager instance
 */
export declare const tokenizerManager: TokenizerManager;
/**
 * Auto-detect and register available tokenizers
 */
export declare function autoRegisterTokenizers(): void;