@juspay/neurolink
Version:
Universal AI Development Platform with working MCP integration, multi-provider support, voice (TTS/STT/realtime), and professional CLI. 58+ external MCP servers discoverable, multimodal file processing, RAG pipelines. Build, test, and deploy AI applicatio
79 lines (78 loc) • 3.1 kB
TypeScript
/**
* Token Estimation Utilities
*
* Provides character-based token estimation with per-provider adjustment
* multipliers. Uses the same approach as Continue (gpt-tokenizer baseline
* + provider multipliers) but without requiring a tokenizer dependency.
*
* Multiplier sources: Continue project's getAdjustedTokenCount.ts
* - Anthropic: 1.23x (Anthropic tokenizer produces ~23% more tokens)
* - Google (AI Studio / Vertex): 1.18x
* - Mistral/Codestral: 1.26x
* - OpenAI/GPT: 1.0x (baseline)
*/
import type { ChatMessage } from "../types/index.js";
/** Characters per token for English text */
export declare const CHARS_PER_TOKEN = 4;
/** Characters per token for code */
export declare const CODE_CHARS_PER_TOKEN = 3;
/**
* Safety margin: additive fraction of baseTokens added to the provider-adjusted estimate.
* Using additive margin prevents compounding with provider multipliers.
*
* Old behavior: baseTokens * providerMultiplier * 1.15 (compounding)
* e.g. Anthropic: baseTokens * 1.23 * 1.15 = baseTokens * 1.4145
* New behavior: baseTokens * providerMultiplier + baseTokens * 0.05 (additive)
* e.g. Anthropic: baseTokens * 1.23 + baseTokens * 0.05 = baseTokens * 1.28
*/
export declare const TOKEN_SAFETY_MARGIN_ADDITIVE = 0.05;
/** @deprecated Use TOKEN_SAFETY_MARGIN_ADDITIVE instead. Kept for backward compatibility. */
export declare const TOKEN_SAFETY_MARGIN = 1.15;
/** Message framing overhead in tokens (role + delimiters) */
export declare const TOKENS_PER_MESSAGE = 4;
/** Conversation-level overhead in tokens */
export declare const TOKENS_PER_CONVERSATION = 24;
/** Image token estimate (flat) */
export declare const IMAGE_TOKEN_ESTIMATE = 1024;
/**
* Get the token multiplier for a given provider.
*/
export declare function getProviderMultiplier(provider?: string): number;
/**
* Estimate token count for a string.
*
* @param text - Input text
* @param provider - Optional provider for multiplier adjustment
* @param isCode - Whether the text is code (uses CODE_CHARS_PER_TOKEN)
* @returns Estimated token count
*/
export declare function estimateTokens(text: string, provider?: string, isCode?: boolean): number;
/**
* Estimate token count for a single ChatMessage.
* Includes message framing overhead.
*/
export declare function estimateMessageTokens(message: ChatMessage | {
role: string;
content: unknown;
}, provider?: string): number;
/**
* Estimate total token count for an array of messages.
* Includes conversation-level overhead.
*/
export declare function estimateMessagesTokens(messages: Array<ChatMessage | {
role: string;
content: unknown;
}>, provider?: string): number;
/**
* Truncate text to fit within a token budget.
* Tries to cut at sentence or word boundaries.
*
* @param text - Input text
* @param maxTokens - Maximum tokens allowed
* @param provider - Optional provider for multiplier
* @returns Truncated text with "..." suffix if truncated
*/
export declare function truncateToTokenBudget(text: string, maxTokens: number, provider?: string): {
text: string;
truncated: boolean;
};