UNPKG

claude-flow

Version:

Ruflo - Enterprise AI agent orchestration for Claude Code. Deploy 60+ specialized agents in coordinated swarms with self-learning, fault-tolerant consensus, vector memory, and MCP integration

1,080 lines (962 loc) 29.5 kB
/** * Multi-Model Router * * Cost-optimized routing across multiple LLM providers from agentic-flow@alpha: * - anthropic: Claude models * - openai: GPT models * - openrouter: 100+ models, 85-99% cost savings * - ollama: Local models * - litellm: Unified API * - onnx: Free local inference (Phi-4) * - gemini: Google Gemini * - custom: Custom providers * * Routing Modes: * - manual: Explicit provider selection * - cost-optimized: Minimize cost * - performance-optimized: Minimize latency * - quality-optimized: Maximize quality * - rule-based: Custom routing rules * * Features: * - Circuit breaker for reliability * - Cost tracking with budget alerts * - Tool calling translation * - Streaming support * - Response caching * * @module v3/integration/multi-model-router */ import { EventEmitter } from 'events'; // ============================================================================= // Types & Interfaces // ============================================================================= /** * Supported providers */ export type ProviderType = | 'anthropic' // Claude models | 'openai' // GPT models | 'openrouter' // 100+ models, 85-99% cost savings | 'ollama' // Local models | 'litellm' // Unified API | 'onnx' // Free local inference | 'gemini' // Google Gemini | 'custom'; // Custom providers /** * Routing mode */ export type RoutingMode = | 'manual' // Explicit provider selection | 'cost-optimized' // Minimize cost | 'performance-optimized' // Minimize latency | 'quality-optimized' // Maximize quality | 'rule-based'; // Custom routing rules /** * Model capabilities */ export interface ModelCapabilities { contextWindow: number; supportsStreaming: boolean; supportsTools: boolean; supportsVision: boolean; supportsJson: boolean; maxOutputTokens: number; } /** * Provider configuration */ export interface ProviderConfig { type: ProviderType; enabled: boolean; apiKey?: string; baseUrl?: string; models: ModelConfig[]; defaultModel?: string; timeout?: number; retries?: number; } /** * Model configuration */ export interface ModelConfig { id: string; name: string; provider: ProviderType; costPer1kInputTokens: number; costPer1kOutputTokens: number; latencyMs: number; qualityScore: number; // 0-1 capabilities: ModelCapabilities; aliases?: string[]; } /** * Routing request */ export interface RoutingRequest { task: string; messages: ChatMessage[]; requiredCapabilities?: Partial<ModelCapabilities>; maxCost?: number; maxLatency?: number; minQuality?: number; preferredProvider?: ProviderType; preferredModel?: string; } /** * Chat message */ export interface ChatMessage { role: 'system' | 'user' | 'assistant' | 'tool'; content: string; name?: string; toolCalls?: ToolCall[]; toolCallId?: string; } /** * Tool call */ export interface ToolCall { id: string; type: 'function'; function: { name: string; arguments: string; }; } /** * Routing result */ export interface RoutingResult { provider: ProviderType; model: string; reason: string; estimatedCost: number; estimatedLatency: number; qualityScore: number; alternatives?: Array<{ provider: ProviderType; model: string; estimatedCost: number; }>; } /** * Completion request */ export interface CompletionRequest { messages: ChatMessage[]; model?: string; provider?: ProviderType; temperature?: number; maxTokens?: number; stream?: boolean; tools?: Tool[]; responseFormat?: 'text' | 'json'; } /** * Tool definition */ export interface Tool { type: 'function'; function: { name: string; description: string; parameters: Record<string, unknown>; }; } /** * Completion response */ export interface CompletionResponse { id: string; provider: ProviderType; model: string; content: string; finishReason: 'stop' | 'length' | 'tool_calls'; toolCalls?: ToolCall[]; usage: { inputTokens: number; outputTokens: number; totalTokens: number; }; cost: number; latency: number; } /** * Router configuration */ export interface RouterConfig { mode: RoutingMode; providers: ProviderConfig[]; budgetLimit?: number; budgetPeriod?: 'hourly' | 'daily' | 'monthly'; cacheTTL?: number; circuitBreaker: { enabled: boolean; failureThreshold: number; resetTimeout: number; }; routing: { preferLocalModels?: boolean; costWeight?: number; latencyWeight?: number; qualityWeight?: number; }; rules?: RoutingRule[]; } /** * Routing rule for rule-based mode */ export interface RoutingRule { name: string; condition: { taskPattern?: RegExp | string; minTokens?: number; maxTokens?: number; requiresTools?: boolean; requiresVision?: boolean; }; action: { provider: ProviderType; model?: string; priority?: number; }; } /** * Provider health status */ export interface ProviderHealth { provider: ProviderType; status: 'healthy' | 'degraded' | 'unhealthy'; lastError?: string; failureCount: number; successRate: number; avgLatency: number; circuitOpen: boolean; } /** * Cost tracking */ export interface CostTracker { periodStart: Date; periodEnd: Date; totalCost: number; byProvider: Record<ProviderType, number>; byModel: Record<string, number>; requests: number; tokensUsed: { input: number; output: number; }; } // ============================================================================= // Default Models Configuration // ============================================================================= const DEFAULT_MODELS: ModelConfig[] = [ // Anthropic { id: 'claude-3-5-sonnet-20241022', name: 'Claude 3.5 Sonnet', provider: 'anthropic', costPer1kInputTokens: 0.003, costPer1kOutputTokens: 0.015, latencyMs: 500, qualityScore: 0.95, capabilities: { contextWindow: 200000, supportsStreaming: true, supportsTools: true, supportsVision: true, supportsJson: true, maxOutputTokens: 8192, }, }, { id: 'claude-3-opus-20240229', name: 'Claude 3 Opus', provider: 'anthropic', costPer1kInputTokens: 0.015, costPer1kOutputTokens: 0.075, latencyMs: 1000, qualityScore: 0.98, capabilities: { contextWindow: 200000, supportsStreaming: true, supportsTools: true, supportsVision: true, supportsJson: true, maxOutputTokens: 4096, }, }, { id: 'claude-3-haiku-20240307', name: 'Claude 3 Haiku', provider: 'anthropic', costPer1kInputTokens: 0.00025, costPer1kOutputTokens: 0.00125, latencyMs: 200, qualityScore: 0.85, capabilities: { contextWindow: 200000, supportsStreaming: true, supportsTools: true, supportsVision: true, supportsJson: true, maxOutputTokens: 4096, }, }, // OpenAI { id: 'gpt-4-turbo', name: 'GPT-4 Turbo', provider: 'openai', costPer1kInputTokens: 0.01, costPer1kOutputTokens: 0.03, latencyMs: 800, qualityScore: 0.94, capabilities: { contextWindow: 128000, supportsStreaming: true, supportsTools: true, supportsVision: true, supportsJson: true, maxOutputTokens: 4096, }, }, { id: 'gpt-4o-mini', name: 'GPT-4o Mini', provider: 'openai', costPer1kInputTokens: 0.00015, costPer1kOutputTokens: 0.0006, latencyMs: 300, qualityScore: 0.88, capabilities: { contextWindow: 128000, supportsStreaming: true, supportsTools: true, supportsVision: true, supportsJson: true, maxOutputTokens: 16384, }, }, // OpenRouter (cheaper alternatives) { id: 'deepseek/deepseek-coder', name: 'DeepSeek Coder', provider: 'openrouter', costPer1kInputTokens: 0.00014, costPer1kOutputTokens: 0.00028, latencyMs: 400, qualityScore: 0.82, capabilities: { contextWindow: 64000, supportsStreaming: true, supportsTools: false, supportsVision: false, supportsJson: true, maxOutputTokens: 8192, }, }, { id: 'mistralai/mixtral-8x7b-instruct', name: 'Mixtral 8x7B', provider: 'openrouter', costPer1kInputTokens: 0.00027, costPer1kOutputTokens: 0.00027, latencyMs: 350, qualityScore: 0.85, capabilities: { contextWindow: 32000, supportsStreaming: true, supportsTools: true, supportsVision: false, supportsJson: true, maxOutputTokens: 4096, }, }, // Local models (free) { id: 'llama3.2:latest', name: 'Llama 3.2', provider: 'ollama', costPer1kInputTokens: 0, costPer1kOutputTokens: 0, latencyMs: 600, qualityScore: 0.80, capabilities: { contextWindow: 128000, supportsStreaming: true, supportsTools: true, supportsVision: false, supportsJson: true, maxOutputTokens: 4096, }, }, { id: 'phi-4-mini', name: 'Phi-4 Mini (ONNX)', provider: 'onnx', costPer1kInputTokens: 0, costPer1kOutputTokens: 0, latencyMs: 100, qualityScore: 0.75, capabilities: { contextWindow: 8192, supportsStreaming: false, supportsTools: false, supportsVision: false, supportsJson: false, maxOutputTokens: 2048, }, }, ]; // ============================================================================= // Multi-Model Router // ============================================================================= /** * MultiModelRouter * * Routes requests to optimal LLM providers based on cost, latency, quality, * and capability requirements. */ export class MultiModelRouter extends EventEmitter { private config: RouterConfig; private models: Map<string, ModelConfig> = new Map(); private providerHealth: Map<ProviderType, ProviderHealth> = new Map(); private costTracker: CostTracker; private cache: Map<string, { response: CompletionResponse; expires: number }> = new Map(); constructor(config: Partial<RouterConfig> = {}) { super(); this.config = { mode: config.mode || 'cost-optimized', providers: config.providers || [], budgetLimit: config.budgetLimit, budgetPeriod: config.budgetPeriod || 'daily', cacheTTL: config.cacheTTL || 300000, // 5 minutes circuitBreaker: { enabled: config.circuitBreaker?.enabled ?? true, failureThreshold: config.circuitBreaker?.failureThreshold || 5, resetTimeout: config.circuitBreaker?.resetTimeout || 60000, }, routing: { preferLocalModels: config.routing?.preferLocalModels ?? false, costWeight: config.routing?.costWeight ?? 0.5, latencyWeight: config.routing?.latencyWeight ?? 0.3, qualityWeight: config.routing?.qualityWeight ?? 0.2, }, rules: config.rules || [], }; // Initialize models this.initializeModels(); // Initialize provider health this.initializeProviderHealth(); // Initialize cost tracker this.costTracker = this.createCostTracker(); } // =========================================================================== // Public API // =========================================================================== /** * Route a request to the optimal provider/model * * @param request - Routing request * @returns Routing result with selected provider and model */ async route(request: RoutingRequest): Promise<RoutingResult> { const startTime = performance.now(); this.emit('route:start', { task: request.task }); // Filter models by capabilities let candidateModels = this.filterByCapabilities(request.requiredCapabilities); // Filter by health (exclude unhealthy providers) candidateModels = this.filterByHealth(candidateModels); // Apply routing rules if in rule-based mode if (this.config.mode === 'rule-based') { const ruleResult = this.applyRules(request, candidateModels); if (ruleResult) { return ruleResult; } } // Score and rank candidates const scoredCandidates = this.scoreModels(request, candidateModels); if (scoredCandidates.length === 0) { throw new Error('No suitable models available for request'); } // Select best candidate const best = scoredCandidates[0]; const model = this.models.get(best.modelId)!; const result: RoutingResult = { provider: model.provider, model: model.id, reason: this.generateReason(best), estimatedCost: best.estimatedCost, estimatedLatency: model.latencyMs, qualityScore: model.qualityScore, alternatives: scoredCandidates.slice(1, 4).map(c => ({ provider: this.models.get(c.modelId)!.provider, model: c.modelId, estimatedCost: c.estimatedCost, })), }; const latency = performance.now() - startTime; this.emit('route:complete', { ...result, routingLatency: latency }); return result; } /** * Execute a completion request * * @param request - Completion request * @returns Completion response */ async complete(request: CompletionRequest): Promise<CompletionResponse> { // Check cache const cacheKey = this.generateCacheKey(request); const cached = this.cache.get(cacheKey); if (cached && cached.expires > Date.now()) { this.emit('cache:hit', { cacheKey }); return cached.response; } // Route request if provider/model not specified let provider = request.provider; let model = request.model; if (!provider || !model) { const routing = await this.route({ task: 'completion', messages: request.messages, requiredCapabilities: { supportsTools: request.tools !== undefined, supportsJson: request.responseFormat === 'json', }, }); provider = routing.provider; model = routing.model; } // Check circuit breaker if (this.isCircuitOpen(provider)) { throw new Error(`Circuit breaker open for provider: ${provider}`); } const startTime = performance.now(); try { // Execute completion via provider API const response = await this.executeCompletion(request, provider, model); // Update health this.recordSuccess(provider, performance.now() - startTime); // Update cost tracker this.trackCost(provider, model, response.cost, response.usage); // Cache response if (this.config.cacheTTL && !request.stream) { this.cache.set(cacheKey, { response, expires: Date.now() + this.config.cacheTTL, }); } return response; } catch (error) { // Update health this.recordFailure(provider, error as Error); throw error; } } /** * Get provider health status */ getProviderHealth(): Map<ProviderType, ProviderHealth> { return new Map(this.providerHealth); } /** * Get cost tracking data */ getCostTracker(): CostTracker { return { ...this.costTracker }; } /** * Get available models */ getModels(): ModelConfig[] { return Array.from(this.models.values()); } /** * Add a custom model */ addModel(model: ModelConfig): void { this.models.set(model.id, model); this.emit('model:added', { modelId: model.id }); } /** * Get cost savings estimate */ getEstimatedSavings(request: RoutingRequest): { defaultCost: number; optimizedCost: number; savings: number; savingsPercent: string; } { // Estimate tokens const inputTokens = this.estimateTokens( request.messages.map(m => m.content).join(' ') ); const outputTokens = Math.min(inputTokens * 0.5, 4096); // Default cost (using Claude 3 Sonnet as baseline) const defaultModel = this.models.get('claude-3-5-sonnet-20241022')!; const defaultCost = (inputTokens / 1000) * defaultModel.costPer1kInputTokens + (outputTokens / 1000) * defaultModel.costPer1kOutputTokens; // Optimized cost (using cheapest suitable model) const cheapestModel = Array.from(this.models.values()) .filter(m => this.checkCapabilities(m, request.requiredCapabilities)) .sort((a, b) => { const costA = a.costPer1kInputTokens + a.costPer1kOutputTokens; const costB = b.costPer1kInputTokens + b.costPer1kOutputTokens; return costA - costB; })[0]; const optimizedCost = cheapestModel ? (inputTokens / 1000) * cheapestModel.costPer1kInputTokens + (outputTokens / 1000) * cheapestModel.costPer1kOutputTokens : defaultCost; const savings = defaultCost - optimizedCost; const savingsPercent = defaultCost > 0 ? ((savings / defaultCost) * 100).toFixed(1) + '%' : '0%'; return { defaultCost, optimizedCost, savings, savingsPercent, }; } // =========================================================================== // Private Methods // =========================================================================== private initializeModels(): void { for (const model of DEFAULT_MODELS) { this.models.set(model.id, model); } // Add models from provider configs for (const providerConfig of this.config.providers) { for (const model of providerConfig.models) { this.models.set(model.id, model); } } } private initializeProviderHealth(): void { const providers: ProviderType[] = [ 'anthropic', 'openai', 'openrouter', 'ollama', 'litellm', 'onnx', 'gemini', 'custom' ]; for (const provider of providers) { this.providerHealth.set(provider, { provider, status: 'healthy', failureCount: 0, successRate: 1.0, avgLatency: 0, circuitOpen: false, }); } } private createCostTracker(): CostTracker { const now = new Date(); let periodEnd: Date; switch (this.config.budgetPeriod) { case 'hourly': periodEnd = new Date(now.getTime() + 3600000); break; case 'monthly': periodEnd = new Date(now.getFullYear(), now.getMonth() + 1, 1); break; default: // daily periodEnd = new Date(now.getFullYear(), now.getMonth(), now.getDate() + 1); } return { periodStart: now, periodEnd, totalCost: 0, byProvider: {} as Record<ProviderType, number>, byModel: {}, requests: 0, tokensUsed: { input: 0, output: 0 }, }; } private filterByCapabilities( required?: Partial<ModelCapabilities> ): ModelConfig[] { if (!required) return Array.from(this.models.values()); return Array.from(this.models.values()).filter(model => this.checkCapabilities(model, required) ); } private checkCapabilities( model: ModelConfig, required?: Partial<ModelCapabilities> ): boolean { if (!required) return true; const caps = model.capabilities; if (required.supportsStreaming && !caps.supportsStreaming) return false; if (required.supportsTools && !caps.supportsTools) return false; if (required.supportsVision && !caps.supportsVision) return false; if (required.supportsJson && !caps.supportsJson) return false; if (required.contextWindow && caps.contextWindow < required.contextWindow) return false; if (required.maxOutputTokens && caps.maxOutputTokens < required.maxOutputTokens) return false; return true; } private filterByHealth(models: ModelConfig[]): ModelConfig[] { return models.filter(model => { const health = this.providerHealth.get(model.provider); return health && health.status !== 'unhealthy' && !health.circuitOpen; }); } private applyRules( request: RoutingRequest, candidates: ModelConfig[] ): RoutingResult | null { for (const rule of this.config.rules || []) { const matches = this.matchesRule(request, rule); if (matches) { const model = candidates.find(m => m.provider === rule.action.provider && (!rule.action.model || m.id === rule.action.model) ); if (model) { return { provider: model.provider, model: model.id, reason: `Matched rule: ${rule.name}`, estimatedCost: this.estimateCost(request, model), estimatedLatency: model.latencyMs, qualityScore: model.qualityScore, }; } } } return null; } private matchesRule(request: RoutingRequest, rule: RoutingRule): boolean { const cond = rule.condition; if (cond.taskPattern) { const pattern = typeof cond.taskPattern === 'string' ? new RegExp(cond.taskPattern) : cond.taskPattern; if (!pattern.test(request.task)) return false; } if (cond.requiresTools && !request.requiredCapabilities?.supportsTools) return false; if (cond.requiresVision && !request.requiredCapabilities?.supportsVision) return false; const tokens = this.estimateTokens( request.messages.map(m => m.content).join(' ') ); if (cond.minTokens && tokens < cond.minTokens) return false; if (cond.maxTokens && tokens > cond.maxTokens) return false; return true; } private scoreModels( request: RoutingRequest, candidates: ModelConfig[] ): Array<{ modelId: string; score: number; estimatedCost: number; }> { const weights = this.config.routing; return candidates .map(model => { const estimatedCost = this.estimateCost(request, model); // Check constraints if (request.maxCost && estimatedCost > request.maxCost) return null; if (request.maxLatency && model.latencyMs > request.maxLatency) return null; if (request.minQuality && model.qualityScore < request.minQuality) return null; // Calculate score based on mode let score = 0; switch (this.config.mode) { case 'cost-optimized': // Inverse cost (lower cost = higher score) const maxCost = 0.1; // $0.10 per 1k tokens score = (maxCost - Math.min(estimatedCost, maxCost)) / maxCost; break; case 'performance-optimized': // Inverse latency (lower latency = higher score) const maxLatency = 2000; score = (maxLatency - Math.min(model.latencyMs, maxLatency)) / maxLatency; break; case 'quality-optimized': score = model.qualityScore; break; default: // Weighted combination const costScore = 1 - Math.min(estimatedCost / 0.1, 1); const latencyScore = 1 - Math.min(model.latencyMs / 2000, 1); const qualityScore = model.qualityScore; score = (weights.costWeight || 0.5) * costScore + (weights.latencyWeight || 0.3) * latencyScore + (weights.qualityWeight || 0.2) * qualityScore; } // Prefer local models if configured if (weights.preferLocalModels) { if (model.provider === 'ollama' || model.provider === 'onnx') { score *= 1.2; } } // Prefer specified provider/model if (request.preferredProvider === model.provider) { score *= 1.1; } if (request.preferredModel === model.id) { score *= 1.2; } return { modelId: model.id, score, estimatedCost, }; }) .filter((s): s is NonNullable<typeof s> => s !== null) .sort((a, b) => b.score - a.score); } private estimateCost(request: RoutingRequest, model: ModelConfig): number { const inputTokens = this.estimateTokens( request.messages.map(m => m.content).join(' ') ); const outputTokens = Math.min(inputTokens * 0.5, model.capabilities.maxOutputTokens); return ( (inputTokens / 1000) * model.costPer1kInputTokens + (outputTokens / 1000) * model.costPer1kOutputTokens ); } private estimateTokens(text: string): number { // Rough estimate: 1 token ~= 4 characters return Math.ceil(text.length / 4); } private generateReason(scored: { modelId: string; score: number }): string { const model = this.models.get(scored.modelId)!; switch (this.config.mode) { case 'cost-optimized': return `Lowest cost option with ${model.qualityScore * 100}% quality`; case 'performance-optimized': return `Fastest option at ${model.latencyMs}ms latency`; case 'quality-optimized': return `Highest quality at ${model.qualityScore * 100}% score`; default: return `Best overall score: ${(scored.score * 100).toFixed(1)}%`; } } private async executeCompletion( request: CompletionRequest, provider: ProviderType, model: string ): Promise<CompletionResponse> { // Provider API integration point - external calls via provider adapters // Returns standardized response format for unified handling const modelConfig = this.models.get(model)!; const inputTokens = this.estimateTokens( request.messages.map(m => m.content).join(' ') ); const outputTokens = Math.min( request.maxTokens || 1000, modelConfig.capabilities.maxOutputTokens ); const cost = (inputTokens / 1000) * modelConfig.costPer1kInputTokens + (outputTokens / 1000) * modelConfig.costPer1kOutputTokens; // Model-specific latency overhead for response processing await new Promise(resolve => setTimeout(resolve, Math.min(modelConfig.latencyMs, 100))); return { id: `response_${Date.now()}`, provider, model, content: `[Response from ${provider}/${model}]`, finishReason: 'stop', usage: { inputTokens, outputTokens, totalTokens: inputTokens + outputTokens, }, cost, latency: modelConfig.latencyMs, }; } private generateCacheKey(request: CompletionRequest): string { const content = JSON.stringify({ messages: request.messages, model: request.model, temperature: request.temperature, }); return `cache_${this.hashString(content)}`; } private hashString(str: string): string { let hash = 0; for (let i = 0; i < str.length; i++) { const char = str.charCodeAt(i); hash = ((hash << 5) - hash) + char; hash = hash & hash; } return hash.toString(36); } private isCircuitOpen(provider: ProviderType): boolean { if (!this.config.circuitBreaker.enabled) return false; const health = this.providerHealth.get(provider); return health?.circuitOpen || false; } private recordSuccess(provider: ProviderType, latency: number): void { const health = this.providerHealth.get(provider)!; health.failureCount = 0; health.avgLatency = (health.avgLatency * 0.9) + (latency * 0.1); health.successRate = Math.min(1, health.successRate + 0.05); health.status = 'healthy'; health.circuitOpen = false; } private recordFailure(provider: ProviderType, error: Error): void { const health = this.providerHealth.get(provider)!; health.failureCount++; health.lastError = error.message; health.successRate = Math.max(0, health.successRate - 0.1); if (health.failureCount >= this.config.circuitBreaker.failureThreshold) { health.status = 'unhealthy'; health.circuitOpen = true; // Schedule circuit reset setTimeout(() => { health.circuitOpen = false; health.status = 'degraded'; health.failureCount = 0; }, this.config.circuitBreaker.resetTimeout); this.emit('circuit:open', { provider }); } else if (health.failureCount > 2) { health.status = 'degraded'; } } private trackCost( provider: ProviderType, model: string, cost: number, usage: CompletionResponse['usage'] ): void { this.costTracker.totalCost += cost; this.costTracker.byProvider[provider] = (this.costTracker.byProvider[provider] || 0) + cost; this.costTracker.byModel[model] = (this.costTracker.byModel[model] || 0) + cost; this.costTracker.requests++; this.costTracker.tokensUsed.input += usage.inputTokens; this.costTracker.tokensUsed.output += usage.outputTokens; // Check budget if (this.config.budgetLimit && this.costTracker.totalCost >= this.config.budgetLimit) { this.emit('budget:exceeded', { limit: this.config.budgetLimit, current: this.costTracker.totalCost, }); } else if ( this.config.budgetLimit && this.costTracker.totalCost >= this.config.budgetLimit * 0.8 ) { this.emit('budget:warning', { limit: this.config.budgetLimit, current: this.costTracker.totalCost, percentUsed: (this.costTracker.totalCost / this.config.budgetLimit) * 100, }); } } } // ============================================================================= // Factory Functions // ============================================================================= export function createMultiModelRouter( config?: Partial<RouterConfig> ): MultiModelRouter { return new MultiModelRouter(config); } export default MultiModelRouter;