@juspay/neurolink/dist/utils/tokenEstimation.d.ts

Universal AI Development Platform with working MCP integration, multi-provider support, voice (TTS/STT/realtime), and professional CLI. 58+ external MCP servers discoverable, multimodal file processing, RAG pipelines. Build, test, and deploy AI applicatio

/**
 * Token Estimation Utilities
 *
 * Provides character-based token estimation with per-provider adjustment
 * multipliers. Uses the same approach as Continue (gpt-tokenizer baseline
 * + provider multipliers) but without requiring a tokenizer dependency.
 *
 * Multiplier sources: Continue project's getAdjustedTokenCount.ts
 * - Anthropic: 1.23x (Anthropic tokenizer produces ~23% more tokens)
 * - Google (AI Studio / Vertex): 1.18x
 * - Mistral/Codestral: 1.26x
 * - OpenAI/GPT: 1.0x (baseline)
 */
import type { ChatMessage } from "../types/index.js";
/** Characters per token for English text */
export declare const CHARS_PER_TOKEN = 4;
/** Characters per token for code */
export declare const CODE_CHARS_PER_TOKEN = 3;
/**
 * Safety margin: additive fraction of baseTokens added to the provider-adjusted estimate.
 * Using additive margin prevents compounding with provider multipliers.
 *
 * Old behavior: baseTokens * providerMultiplier * 1.15  (compounding)
 *   e.g. Anthropic: baseTokens * 1.23 * 1.15 = baseTokens * 1.4145
 * New behavior: baseTokens * providerMultiplier + baseTokens * 0.05  (additive)
 *   e.g. Anthropic: baseTokens * 1.23 + baseTokens * 0.05 = baseTokens * 1.28
 */
export declare const TOKEN_SAFETY_MARGIN_ADDITIVE = 0.05;
/** @deprecated Use TOKEN_SAFETY_MARGIN_ADDITIVE instead. Kept for backward compatibility. */
export declare const TOKEN_SAFETY_MARGIN = 1.15;
/** Message framing overhead in tokens (role + delimiters) */
export declare const TOKENS_PER_MESSAGE = 4;
/** Conversation-level overhead in tokens */
export declare const TOKENS_PER_CONVERSATION = 24;
/** Image token estimate (flat) */
export declare const IMAGE_TOKEN_ESTIMATE = 1024;
/**
 * Get the token multiplier for a given provider.
 */
export declare function getProviderMultiplier(provider?: string): number;
/**
 * Estimate token count for a string.
 *
 * @param text - Input text
 * @param provider - Optional provider for multiplier adjustment
 * @param isCode - Whether the text is code (uses CODE_CHARS_PER_TOKEN)
 * @returns Estimated token count
 */
export declare function estimateTokens(text: string, provider?: string, isCode?: boolean): number;
/**
 * Estimate token count for a single ChatMessage.
 * Includes message framing overhead.
 */
export declare function estimateMessageTokens(message: ChatMessage | {
    role: string;
    content: unknown;
}, provider?: string): number;
/**
 * Estimate total token count for an array of messages.
 * Includes conversation-level overhead.
 */
export declare function estimateMessagesTokens(messages: Array<ChatMessage | {
    role: string;
    content: unknown;
}>, provider?: string): number;
/**
 * Truncate text to fit within a token budget.
 * Tries to cut at sentence or word boundaries.
 *
 * @param text - Input text
 * @param maxTokens - Maximum tokens allowed
 * @param provider - Optional provider for multiplier
 * @returns Truncated text with "..." suffix if truncated
 */
export declare function truncateToTokenBudget(text: string, maxTokens: number, provider?: string): {
    text: string;
    truncated: boolean;
};