codevault/dist/providers/token-counter.js

AI-powered semantic code search via Model Context Protocol

let tiktokenEncoder = null;
export async function getTokenCounter(modelName) {
    if (modelName.includes('text-embedding') || modelName.includes('ada-002')) {
        if (!tiktokenEncoder) {
            try {
                const tiktoken = await import('tiktoken');
                tiktokenEncoder = tiktoken.encoding_for_model('text-embedding-3-large');
            }
            catch (error) {
                console.warn('tiktoken not available, falling back to character estimation');
                return null;
            }
        }
        return (text) => tiktokenEncoder.encode(text).length;
    }
    return (text) => Math.ceil(text.length / 4);
}
//# sourceMappingURL=token-counter.js.map