@nanocollective/nanocoder/dist/tokenization/tokenizer-factory.js

A local-first CLI coding agent that brings the power of agentic coding tools like Claude Code and Gemini CLI to local models or controlled APIs like OpenRouter

/**
 * Tokenizer factory
 * Creates appropriate tokenizer based on provider and model
 */
import { AnthropicTokenizer } from './tokenizers/anthropic-tokenizer.js';
import { FallbackTokenizer } from './tokenizers/fallback-tokenizer.js';
import { LlamaTokenizer } from './tokenizers/llama-tokenizer.js';
import { OpenAITokenizer } from './tokenizers/openai-tokenizer.js';
/**
 * Detect provider from model ID or provider name
 */
function detectProvider(providerName, modelId) {
    const lowerProvider = providerName.toLowerCase();
    const lowerModel = modelId.toLowerCase();
    // Check provider name
    if (lowerProvider.includes('openai')) {
        return 'openai';
    }
    if (lowerProvider.includes('anthropic') || lowerProvider.includes('claude')) {
        return 'anthropic';
    }
    // Check model ID for common patterns
    if (lowerModel.includes('gpt') || lowerModel.includes('openai')) {
        return 'openai';
    }
    if (lowerModel.includes('claude')) {
        return 'anthropic';
    }
    if (lowerModel.includes('llama') ||
        lowerModel.includes('mistral') ||
        lowerModel.includes('qwen') ||
        lowerModel.includes('gemma') ||
        lowerModel.includes('phi') ||
        lowerModel.includes('codellama') ||
        lowerModel.includes('deepseek') ||
        lowerModel.includes('mixtral')) {
        return 'llama';
    }
    // Default to llama for local models (most common for local inference)
    if (lowerProvider.includes('ollama') ||
        lowerProvider.includes('llama.cpp') ||
        lowerProvider.includes('local')) {
        return 'llama';
    }
    return 'fallback';
}
/**
 * Create a tokenizer based on provider and model
 */
export function createTokenizer(providerName, modelId) {
    // Strip :cloud suffix if present (Ollama cloud models)
    const normalizedModelId = modelId.endsWith(':cloud') || modelId.endsWith('-cloud')
        ? modelId.slice(0, -6)
        : modelId;
    const provider = detectProvider(providerName, normalizedModelId);
    switch (provider) {
        case 'openai':
            return new OpenAITokenizer(normalizedModelId);
        case 'anthropic':
            return new AnthropicTokenizer(normalizedModelId);
        case 'llama':
            return new LlamaTokenizer(normalizedModelId);
        case 'fallback':
        default:
            return new FallbackTokenizer();
    }
}
/**
 * Create a tokenizer with explicit provider
 */
export function createTokenizerForProvider(provider, modelId) {
    switch (provider) {
        case 'openai':
            return new OpenAITokenizer(modelId);
        case 'anthropic':
            return new AnthropicTokenizer(modelId);
        case 'llama':
            return new LlamaTokenizer(modelId);
        case 'auto':
            // Auto-detect based on model ID if provided
            if (modelId) {
                return createTokenizer('', modelId);
            }
            return new FallbackTokenizer();
        case 'fallback':
        default:
            return new FallbackTokenizer();
    }
}
//# sourceMappingURL=tokenizer-factory.js.map