@toolprint/hypertool-mcp/dist/server/tools/utils/token-counter.d.ts

HyperTool MCP proxy server for routing requests between clients and multiple underlying MCP servers

/**
 * Token Counter Utility
 * Provides token counting ESTIMATES for MCP tools
 *
 * Why we use approximation:
 * - Different LLMs use different tokenizers (GPT uses tiktoken, Claude uses its own)
 * - We don't know which LLM will consume these tools at runtime
 * - For comparing relative tool sizes, estimates are sufficient
 *
 * How LLM tokenization actually works:
 * - Uses BPE (Byte Pair Encoding) or similar subword algorithms
 * - Common words = 1 token ("the", "is", "and")
 * - Uncommon words = multiple tokens ("extraordinary" → ["extra", "ordinary"])
 * - NOT simply words or characters
 *
 * Our approach: Simple word-length heuristic that approximates BPE behavior
 */
import { Tool } from "@modelcontextprotocol/sdk/types.js";
import type { ContextInfo, ToolInfoResponse } from "../schemas.js";
import type { DiscoveredTool } from "../../../discovery/types.js";
/**
 * Fast token estimator using word-length heuristics
 * Provides consistent estimates for comparing tool context usage
 */
export declare class TokenCounter {
    private cache;
    /**
     * Calculate tokens for a tool definition
     */
    calculateToolTokens(tool: {
        name: string;
        namespacedName?: string;
        description?: string;
        tool?: Tool;
    }): number;
    /**
     * Calculate tokens for an array of tools
     */
    calculateToolsetTokens(tools: Array<{
        name: string;
        namespacedName?: string;
        description?: string;
        tool?: Tool;
    }>): number;
    /**
     * BPE-based approximation using word length patterns
     *
     * In BPE (Byte Pair Encoding) tokenization:
     * - Short common words (1-3 chars): Usually 1 token ("the", "is", "a")
     * - Medium words (4-7 chars): Often 1-2 tokens ("hello" → 1, "running" → 1-2)
     * - Long words (8+ chars): Multiple tokens based on subword patterns
     *
     * We use a simple word-length based estimate that works well for English text
     * and JSON schemas (which MCP tools primarily contain).
     */
    private approximateTokens;
    /**
     * Clear the cache (useful when tools are updated)
     */
    clearCache(): void;
    /**
     * Calculate context info with percentage
     */
    calculateContextInfo(tokens: number, totalTokens: number | null): ContextInfo;
    /**
     * Add context info to tools array
     */
    addContextToTools<T extends {
        name: string;
        namespacedName?: string;
        description?: string;
        tool?: Tool;
    }>(tools: T[], totalTokens?: number): Array<T & {
        context: ContextInfo;
    }>;
    /**
     * Convert a DiscoveredTool to ToolInfoResponse with context
     */
    convertToToolInfoResponse(tool: DiscoveredTool, totalTokens: number): ToolInfoResponse;
}
export declare const tokenCounter: TokenCounter;
//# sourceMappingURL=token-counter.d.ts.map