@yogesh0333/yogiway-prompt/dist/tokenizer-official.js

Free & Open Source Prompt Optimization Library - Save 30-50% on AI API costs. Multi-language, multi-platform support.

"use strict";
/**
 * Official Tokenizer Integration
 * Supports tiktoken (OpenAI), anthropic tokenizer, and other official tokenizers
 */
Object.defineProperty(exports, "__esModule", { value: true });
exports.tokenizerManager = exports.TokenizerManager = void 0;
exports.createTiktokenTokenizer = createTiktokenTokenizer;
exports.createAnthropicTokenizer = createAnthropicTokenizer;
exports.autoRegisterTokenizers = autoRegisterTokenizers;
/**
 * Official tokenizer integration with fallback
 * Attempts to use official tokenizers when available, falls back to estimation
 */
class TokenizerManager {
    constructor() {
        this.tokenizers = new Map();
    }
    /**
     * Register an official tokenizer for a provider
     */
    registerTokenizer(provider, tokenizer) {
        this.tokenizers.set(provider, tokenizer);
    }
    /**
     * Count tokens using official tokenizer if available, otherwise fallback
     */
    countTokens(text, provider, fallback) {
        const tokenizer = this.tokenizers.get(provider);
        if (tokenizer) {
            try {
                return tokenizer.count(text);
            }
            catch (error) {
                // Fallback on error
                return fallback(text, provider);
            }
        }
        return fallback(text, provider);
    }
    /**
     * Check if official tokenizer is available
     */
    hasTokenizer(provider) {
        return (this.tokenizers.has(provider) && this.tokenizers.get(provider) !== null);
    }
}
exports.TokenizerManager = TokenizerManager;
/**
 * Tiktoken integration (OpenAI)
 * Usage: Install tiktoken package: npm install tiktoken
 */
function createTiktokenTokenizer(model = "gpt-5") {
    try {
        // Dynamic import to avoid breaking if tiktoken is not installed
        const { encoding_for_model } = require("tiktoken");
        const encoding = encoding_for_model(model);
        return {
            encode: (text) => encoding.encode(text),
            decode: (tokens) => encoding.decode(tokens),
            count: (text) => encoding.encode(text).length,
        };
    }
    catch (error) {
        // tiktoken not installed, return null
        return null;
    }
}
/**
 * Anthropic tokenizer integration
 * Usage: Install @anthropic-ai/tokenizer package
 */
function createAnthropicTokenizer() {
    try {
        const { countTokens } = require("@anthropic-ai/tokenizer");
        return {
            encode: (text) => {
                // Anthropic doesn't expose encode directly, use countTokens
                const count = countTokens(text);
                // Return array of token IDs (approximation)
                return Array(count)
                    .fill(0)
                    .map((_, i) => i);
            },
            decode: () => {
                throw new Error("Anthropic tokenizer decode not supported");
            },
            count: (text) => countTokens(text),
        };
    }
    catch (error) {
        return null;
    }
}
/**
 * Global tokenizer manager instance
 */
exports.tokenizerManager = new TokenizerManager();
/**
 * Auto-detect and register available tokenizers
 */
function autoRegisterTokenizers() {
    // Try to register tiktoken for OpenAI
    const tiktoken = createTiktokenTokenizer("gpt-5");
    if (tiktoken) {
        exports.tokenizerManager.registerTokenizer("openai", tiktoken);
    }
    // Try to register Anthropic tokenizer
    const anthropic = createAnthropicTokenizer();
    if (anthropic) {
        exports.tokenizerManager.registerTokenizer("anthropic", anthropic);
    }
}