UNPKG

quantum-cli-core

Version:

Quantum CLI Core - Multi-LLM Collaboration System

693 lines 34.7 kB
/** * @license * Copyright 2025 Google LLC * SPDX-License-Identifier: Apache-2.0 */ import { QueryType } from '../types.js'; /** * Type of cost optimization */ export var OptimizationType; (function (OptimizationType) { OptimizationType["MODEL_SUBSTITUTION"] = "model_substitution"; OptimizationType["USAGE_PATTERN_CHANGE"] = "usage_pattern_change"; OptimizationType["THRESHOLD_ADJUSTMENT"] = "threshold_adjustment"; OptimizationType["BULK_PROCESSING"] = "bulk_processing"; OptimizationType["TIMING_OPTIMIZATION"] = "timing_optimization"; OptimizationType["QUALITY_TRADE"] = "quality_trade"; OptimizationType["VERIFICATION_REDUCTION"] = "verification_reduction"; OptimizationType["CACHING_STRATEGY"] = "caching_strategy"; })(OptimizationType || (OptimizationType = {})); /** * Default cost optimization configuration */ export const DEFAULT_COST_CONFIG = { enableOptimization: true, minSavingsThreshold: 5.0, // $5/month minimum minConfidenceThreshold: 0.7, maxQualityTradeoff: 0.1, // 10% quality reduction max riskTolerance: 'moderate', acceptableEffortLevels: ['minimal', 'low', 'medium'], enablePredictiveAnalysis: true, analysisLookbackDays: 90, }; /** * Cost optimization engine for generating savings suggestions */ export class CostOptimizationEngine { config; suggestions = new Map(); // userId -> suggestions usagePatterns = new Map(); // userId -> patterns implementedSuggestions = new Map(); // userId -> suggestion IDs constructor(config = {}) { this.config = { ...DEFAULT_COST_CONFIG, ...config }; } /** * Analyzes usage and generates cost optimization suggestions */ generateOptimizationSuggestions(userId, queryHistory, userStats, preferences) { if (!this.config.enableOptimization) { return []; } // Filter to recent history within lookback period const cutoffDate = new Date(); cutoffDate.setDate(cutoffDate.getDate() - this.config.analysisLookbackDays); const recentHistory = queryHistory.filter(q => q.timestamp > cutoffDate); if (recentHistory.length < 10) { return []; // Need sufficient data for meaningful analysis } // Analyze current usage patterns const patterns = this.analyzeUsagePatterns(recentHistory, userStats); this.usagePatterns.set(userId, patterns); const suggestions = []; // Generate different types of optimization suggestions suggestions.push(...this.generateModelSubstitutionSuggestions(patterns, preferences, recentHistory)); suggestions.push(...this.generateUsagePatternSuggestions(patterns, recentHistory)); suggestions.push(...this.generateThresholdAdjustmentSuggestions(patterns, recentHistory)); suggestions.push(...this.generateBulkProcessingSuggestions(patterns, recentHistory)); suggestions.push(...this.generateTimingOptimizationSuggestions(patterns, recentHistory)); suggestions.push(...this.generateVerificationReductionSuggestions(patterns, recentHistory)); // Filter and rank suggestions const filteredSuggestions = this.filterAndRankSuggestions(suggestions, userStats); // Store suggestions for tracking this.suggestions.set(userId, filteredSuggestions); return filteredSuggestions; } /** * Analyzes usage patterns from query history */ analyzeUsagePatterns(queryHistory, _userStats) { const patterns = []; const daysInPeriod = this.config.analysisLookbackDays; // Group queries by type and provider const groups = new Map(); for (const query of queryHistory) { const key = `${query.analysis.type}|${query.selectedProvider}`; const group = groups.get(key) || []; group.push(query); groups.set(key, group); } // Analyze each group for (const [key, queries] of groups.entries()) { const [queryType, provider] = key.split('|'); if (queries.length < 3) continue; // Need minimum queries for pattern const totalCost = queries.reduce((sum, q) => sum + q.cost, 0); const averageCost = totalCost / queries.length; const frequency = (queries.length / daysInPeriod) * 30; // Queries per month // Calculate satisfaction rating const ratedQueries = queries.filter(q => q.userRating); const satisfactionRating = ratedQueries.length > 0 ? ratedQueries.reduce((sum, q) => sum + (q.userRating - 1), 0) / (ratedQueries.length * 4) : 0.5; // Calculate cost efficiency (cost per satisfaction point) const costEfficiency = satisfactionRating > 0 ? averageCost / satisfactionRating : averageCost; // Analyze trend const sortedQueries = queries.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime()); const recentQueries = sortedQueries.slice(-Math.ceil(queries.length / 2)); const olderQueries = sortedQueries.slice(0, Math.floor(queries.length / 2)); const recentAvgCost = recentQueries.reduce((sum, q) => sum + q.cost, 0) / recentQueries.length; const olderAvgCost = olderQueries.reduce((sum, q) => sum + q.cost, 0) / olderQueries.length; let trend = 'stable'; if (recentAvgCost > olderAvgCost * 1.1) trend = 'increasing'; else if (recentAvgCost < olderAvgCost * 0.9) trend = 'decreasing'; // Analyze peak hours const hourCounts = new Array(24).fill(0); for (const query of queries) { hourCounts[query.timestamp.getHours()]++; } const peakHours = hourCounts .map((count, hour) => ({ hour, count })) .filter(item => item.count > 0) .sort((a, b) => b.count - a.count) .slice(0, 3) .map(item => item.hour); patterns.push({ queryType, provider, frequency, averageCost, totalCost, trend, peakHours, costEfficiency, satisfactionRating, }); } return patterns.sort((a, b) => b.totalCost - a.totalCost); // Sort by total cost descending } /** * Generates model substitution suggestions */ generateModelSubstitutionSuggestions(patterns, preferences, queryHistory) { const suggestions = []; // Model cost rankings (simplified - should be loaded from configuration) const costRankings = { 'google-gemini': 0.0001, // Cheapest 'openai-gpt4': 0.03, // Medium cost 'anthropic-claude': 0.015, // Medium-high cost }; for (const pattern of patterns) { if (pattern.totalCost < this.config.minSavingsThreshold) continue; // Find alternative models for this query type const alternatives = this.findAlternativeModels(pattern.queryType, pattern.provider, queryHistory, costRankings); for (const alternative of alternatives) { if (alternative.costSavings < this.config.minSavingsThreshold) continue; if (Math.abs(alternative.qualityDifference) > this.config.maxQualityTradeoff) continue; const monthlySavings = alternative.costSavings * pattern.frequency; const percentageSavings = alternative.costSavings / pattern.averageCost; suggestions.push({ id: this.generateId(), type: OptimizationType.MODEL_SUBSTITUTION, title: `Switch to ${alternative.suggestedProvider} for ${pattern.queryType} queries`, description: `Replace ${alternative.currentProvider} with ${alternative.suggestedProvider} for ${pattern.queryType} tasks`, impact: { monthlySavings, percentageSavings, timeframe: 'immediate', confidence: alternative.confidence, risk: this.assessRisk(alternative.qualityDifference, alternative.performanceDifference), }, implementation: { effort: 'minimal', duration: '5 minutes', steps: [ 'Update model preferences in settings', 'Test with a few queries to verify quality', 'Monitor satisfaction over first week', ], requirements: ['Access to alternative model API'], automation: true, reversible: true, }, confidence: alternative.confidence, priority: this.calculatePriority(monthlySavings, percentageSavings), category: 'immediate', evidence: [ `Based on ${alternative.sampleSize} similar queries`, `Average cost reduction: ${(percentageSavings * 100).toFixed(1)}%`, `Quality difference: ${(alternative.qualityDifference * 100).toFixed(1)}%`, ], tradeoffs: this.calculateTradeoffs(alternative), }); } } return suggestions; } /** * Finds alternative models for a query type */ findAlternativeModels(queryType, currentProvider, queryHistory, costRankings) { const alternatives = []; const availableProviders = Object.keys(costRankings).filter(p => p !== currentProvider); const currentCost = costRankings[currentProvider] || 0.01; for (const alternativeProvider of availableProviders) { const alternativeCost = costRankings[alternativeProvider]; const costSavings = currentCost - alternativeCost; if (costSavings <= 0) continue; // Only suggest cheaper alternatives // Analyze quality difference from historical data const currentQueries = queryHistory.filter(q => q.selectedProvider === currentProvider && q.analysis.type === queryType); const alternativeQueries = queryHistory.filter(q => q.selectedProvider === alternativeProvider && q.analysis.type === queryType); let qualityDifference = 0; let performanceDifference = 0; let confidence = 0.5; if (currentQueries.length > 0 && alternativeQueries.length > 0) { // Calculate satisfaction difference const currentSatisfaction = this.calculateAverageSatisfaction(currentQueries); const alternativeSatisfaction = this.calculateAverageSatisfaction(alternativeQueries); qualityDifference = alternativeSatisfaction - currentSatisfaction; // Calculate performance difference const currentAvgTime = currentQueries.reduce((sum, q) => sum + q.responseTime, 0) / currentQueries.length; const alternativeAvgTime = alternativeQueries.reduce((sum, q) => sum + q.responseTime, 0) / alternativeQueries.length; performanceDifference = (currentAvgTime - alternativeAvgTime) / currentAvgTime; // Positive = alternative is faster // Confidence based on sample size confidence = Math.min(1, (currentQueries.length + alternativeQueries.length) / 20); } alternatives.push({ currentProvider, suggestedProvider: alternativeProvider, queryType, costSavings, qualityDifference, performanceDifference, confidence, riskAssessment: this.assessRisk(qualityDifference, performanceDifference), sampleSize: currentQueries.length + alternativeQueries.length, }); } return alternatives; } /** * Generates usage pattern optimization suggestions */ generateUsagePatternSuggestions(patterns, _queryHistory) { const suggestions = []; // Look for expensive low-satisfaction patterns const problematicPatterns = patterns.filter(p => p.costEfficiency > 0.1 && p.satisfactionRating < 0.6 && p.totalCost > this.config.minSavingsThreshold); for (const pattern of problematicPatterns) { const potentialSavings = pattern.totalCost * 0.3; // Estimate 30% improvement possible suggestions.push({ id: this.generateId(), type: OptimizationType.USAGE_PATTERN_CHANGE, title: `Optimize ${pattern.queryType} query patterns`, description: `Your ${pattern.queryType} queries show low satisfaction (${(pattern.satisfactionRating * 100).toFixed(0)}%) for the cost. Consider breaking complex queries into simpler ones or using verification more selectively.`, impact: { monthlySavings: potentialSavings, percentageSavings: 0.3, timeframe: '2-4 weeks', confidence: 0.6, risk: 'low', }, implementation: { effort: 'medium', duration: '2 weeks', steps: [ 'Review recent unsatisfactory queries', 'Identify common patterns or complexity issues', 'Experiment with breaking down complex queries', 'Use verification only when needed', ], requirements: ['Time to review and adjust query patterns'], automation: false, reversible: true, }, confidence: 0.6, priority: 'medium', category: 'short_term', evidence: [ `Cost efficiency: $${pattern.costEfficiency.toFixed(4)} per satisfaction point`, `${pattern.frequency.toFixed(0)} queries per month`, `Average satisfaction: ${(pattern.satisfactionRating * 100).toFixed(0)}%`, ], tradeoffs: [ { aspect: 'convenience', impact: -0.2, description: 'May require more thought in query formulation', severity: 'minor', mitigation: 'Develop templates for common query patterns', }, ], }); } return suggestions; } /** * Generates threshold adjustment suggestions */ generateThresholdAdjustmentSuggestions(patterns, queryHistory) { const suggestions = []; // Analyze verification usage const verificationQueries = queryHistory.filter(q => q.alternativeProviders.length > 0); const verificationRate = verificationQueries.length / queryHistory.length; const verificationCostRatio = verificationQueries.reduce((sum, q) => sum + q.cost, 0) / queryHistory.reduce((sum, q) => sum + q.cost, 0); if (verificationRate > 0.5 && verificationCostRatio > 0.4) { // High verification usage const verificationSatisfaction = this.calculateAverageSatisfaction(verificationQueries); const regularSatisfaction = this.calculateAverageSatisfaction(queryHistory.filter(q => q.alternativeProviders.length === 0)); if (verificationSatisfaction - regularSatisfaction < 0.1) { // Verification isn't providing much value const potentialSavings = verificationCostRatio * 0.5; // Reduce verification by 50% suggestions.push({ id: this.generateId(), type: OptimizationType.THRESHOLD_ADJUSTMENT, title: 'Reduce unnecessary verification', description: `You verify ${(verificationRate * 100).toFixed(0)}% of queries but satisfaction improvement is minimal. Raising uncertainty thresholds could reduce costs while maintaining quality.`, impact: { monthlySavings: potentialSavings * 30, // Assuming $30/month baseline percentageSavings: potentialSavings, timeframe: 'immediate', confidence: 0.8, risk: 'low', }, implementation: { effort: 'minimal', duration: '2 minutes', steps: [ 'Increase uncertainty threshold from current to higher value', 'Monitor satisfaction for one week', 'Adjust further if needed', ], requirements: ['Access to threshold settings'], automation: true, reversible: true, }, confidence: 0.8, priority: 'high', category: 'immediate', evidence: [ `Verification rate: ${(verificationRate * 100).toFixed(0)}%`, `Verification satisfaction improvement: ${((verificationSatisfaction - regularSatisfaction) * 100).toFixed(1)}%`, `Verification cost ratio: ${(verificationCostRatio * 100).toFixed(0)}%`, ], tradeoffs: [ { aspect: 'reliability', impact: -0.1, description: 'Slightly reduced verification coverage', severity: 'minor', mitigation: 'Monitor satisfaction and adjust threshold if needed', }, ], }); } } return suggestions; } /** * Generates bulk processing suggestions */ generateBulkProcessingSuggestions(patterns, queryHistory) { const suggestions = []; // Look for repetitive query patterns that could be batched const bulkOpportunities = this.identifyBulkProcessingOpportunities(queryHistory); for (const opportunity of bulkOpportunities) { if (opportunity.savings < this.config.minSavingsThreshold) continue; suggestions.push({ id: this.generateId(), type: OptimizationType.BULK_PROCESSING, title: `Batch similar ${opportunity.queryPattern} queries`, description: `You frequently make similar queries (${opportunity.frequency} times/month). Batching these could reduce costs through more efficient processing.`, impact: { monthlySavings: opportunity.savings, percentageSavings: opportunity.savings / opportunity.currentIndividualCost, timeframe: '1-2 weeks', confidence: 0.7, risk: 'medium', }, implementation: { effort: 'medium', duration: '1 week', steps: [ 'Identify queries that can be batched', 'Collect multiple queries before processing', 'Submit as batch request', 'Process results appropriately', ], requirements: ['Support for batch processing in workflow'], automation: true, reversible: true, }, confidence: 0.7, priority: 'medium', category: 'short_term', evidence: [ `Pattern frequency: ${opportunity.frequency}/month`, `Current individual cost: $${opportunity.currentIndividualCost.toFixed(4)}`, `Estimated bulk cost: $${opportunity.estimatedBulkCost.toFixed(4)}`, ], tradeoffs: [ { aspect: 'convenience', impact: -0.3, description: 'Requires batching queries instead of immediate processing', severity: 'moderate', mitigation: 'Implement smart batching with time limits', }, { aspect: 'performance', impact: -0.2, description: 'Slightly delayed results for individual queries', severity: 'minor', }, ], }); } return suggestions; } /** * Generates timing optimization suggestions */ generateTimingOptimizationSuggestions(patterns, queryHistory) { const suggestions = []; // Analyze if there are cost differences by time of day (simplified analysis) const hourlyUsage = new Array(24).fill(0); const hourlyCosts = new Array(24).fill(0); for (const query of queryHistory) { const hour = query.timestamp.getHours(); hourlyUsage[hour]++; hourlyCosts[hour] += query.cost; } const avgHourlyCosts = hourlyCosts.map((cost, hour) => hourlyUsage[hour] > 0 ? cost / hourlyUsage[hour] : 0); const maxCost = Math.max(...avgHourlyCosts.filter(c => c > 0)); const minCost = Math.min(...avgHourlyCosts.filter(c => c > 0)); if (maxCost > minCost * 1.2) { // Significant cost difference by time const expensiveHours = avgHourlyCosts .map((cost, hour) => ({ hour, cost })) .filter(item => item.cost > minCost * 1.15) .map(item => item.hour); const totalExpensiveQueries = expensiveHours.reduce((sum, hour) => sum + hourlyUsage[hour], 0); const potentialSavings = (maxCost - minCost) * totalExpensiveQueries * 0.5; // 50% could be shifted if (potentialSavings > this.config.minSavingsThreshold && totalExpensiveQueries > 10) { suggestions.push({ id: this.generateId(), type: OptimizationType.TIMING_OPTIMIZATION, title: 'Shift queries to lower-cost time periods', description: `Your queries are more expensive during ${expensiveHours.join(', ')}:00. Consider shifting non-urgent queries to off-peak hours.`, impact: { monthlySavings: potentialSavings, percentageSavings: (maxCost - minCost) / maxCost, timeframe: '2-3 weeks', confidence: 0.6, risk: 'low', }, implementation: { effort: 'low', duration: '1 week', steps: [ 'Identify non-urgent queries during peak hours', 'Schedule these for off-peak processing', 'Set up automated timing optimization', ], requirements: ['Query scheduling capability'], automation: true, reversible: true, }, confidence: 0.6, priority: 'low', category: 'short_term', evidence: [ `Peak hour cost: $${maxCost.toFixed(4)} average`, `Off-peak cost: $${minCost.toFixed(4)} average`, `${totalExpensiveQueries} queries during expensive hours`, ], tradeoffs: [ { aspect: 'convenience', impact: -0.2, description: 'Some queries processed with delay', severity: 'minor', mitigation: 'Only apply to non-urgent queries', }, ], }); } } return suggestions; } /** * Generates verification reduction suggestions */ generateVerificationReductionSuggestions(patterns, queryHistory) { const suggestions = []; // Analyze verification effectiveness by query type for (const queryType of Object.values(QueryType)) { const typeQueries = queryHistory.filter(q => q.analysis.type === queryType); if (typeQueries.length < 10) continue; const verifiedQueries = typeQueries.filter(q => q.alternativeProviders.length > 0); const nonVerifiedQueries = typeQueries.filter(q => q.alternativeProviders.length === 0); if (verifiedQueries.length > 5 && nonVerifiedQueries.length > 5) { const verifiedSatisfaction = this.calculateAverageSatisfaction(verifiedQueries); const nonVerifiedSatisfaction = this.calculateAverageSatisfaction(nonVerifiedQueries); const satisfactionImprovement = verifiedSatisfaction - nonVerifiedSatisfaction; const verifiedAvgCost = verifiedQueries.reduce((sum, q) => sum + q.cost, 0) / verifiedQueries.length; const nonVerifiedAvgCost = nonVerifiedQueries.reduce((sum, q) => sum + q.cost, 0) / nonVerifiedQueries.length; const costIncrease = verifiedAvgCost - nonVerifiedAvgCost; // If verification provides minimal benefit but significant cost if (satisfactionImprovement < 0.1 && costIncrease > 0.01) { const monthlyVerificationQueries = (verifiedQueries.length / this.config.analysisLookbackDays) * 30; const monthlySavings = costIncrease * monthlyVerificationQueries * 0.7; // Reduce 70% of verification if (monthlySavings > this.config.minSavingsThreshold) { suggestions.push({ id: this.generateId(), type: OptimizationType.VERIFICATION_REDUCTION, title: `Reduce verification for ${queryType} queries`, description: `Verification for ${queryType} queries shows minimal satisfaction improvement (${(satisfactionImprovement * 100).toFixed(1)}%) but increases costs significantly.`, impact: { monthlySavings, percentageSavings: 0.7, timeframe: 'immediate', confidence: 0.8, risk: 'low', }, implementation: { effort: 'minimal', duration: '5 minutes', steps: [ `Disable automatic verification for ${queryType} queries`, 'Monitor satisfaction for one week', 'Re-enable if satisfaction drops significantly', ], requirements: ['Access to verification settings'], automation: true, reversible: true, }, confidence: 0.8, priority: 'medium', category: 'immediate', evidence: [ `Verification satisfaction improvement: ${(satisfactionImprovement * 100).toFixed(1)}%`, `Verification cost increase: $${costIncrease.toFixed(4)}`, `Monthly verification queries: ${monthlyVerificationQueries.toFixed(0)}`, ], tradeoffs: [ { aspect: 'reliability', impact: -0.1, description: 'Reduced error detection for this query type', severity: 'minor', mitigation: 'Monitor and re-enable if issues arise', }, ], }); } } } } return suggestions; } /** * Helper methods */ calculateAverageSatisfaction(queries) { const ratedQueries = queries.filter(q => q.userRating); if (ratedQueries.length === 0) return 0.5; return ratedQueries.reduce((sum, q) => sum + (q.userRating - 1), 0) / (ratedQueries.length * 4); } identifyBulkProcessingOpportunities(queryHistory) { // Simplified implementation - would use more sophisticated pattern matching const opportunities = []; // Group similar queries (simplified similarity check) const patterns = new Map(); for (const query of queryHistory) { // Create a simple pattern key based on query characteristics const key = `${query.analysis.type}|${query.analysis.domain}|${Math.round(query.analysis.complexity * 2) / 2}`; const group = patterns.get(key) || []; group.push(query); patterns.set(key, group); } for (const [pattern, queries] of patterns.entries()) { if (queries.length < 5) continue; // Need minimum frequency const frequency = (queries.length / this.config.analysisLookbackDays) * 30; const avgCost = queries.reduce((sum, q) => sum + q.cost, 0) / queries.length; const estimatedBulkCost = avgCost * 0.7; // Assume 30% bulk discount const savings = (avgCost - estimatedBulkCost) * frequency; opportunities.push({ queryPattern: pattern, frequency, currentIndividualCost: avgCost, estimatedBulkCost, savings, feasibility: 0.7, // Simplified assessment implementation: 'Batch similar queries with delay tolerance', }); } return opportunities; } assessRisk(qualityDifference, performanceDifference) { const qualityRisk = Math.abs(qualityDifference); const performanceRisk = Math.abs(performanceDifference); if (qualityRisk > 0.2 || performanceRisk > 0.3) return 'high'; if (qualityRisk > 0.1 || performanceRisk > 0.15) return 'medium'; return 'low'; } calculatePriority(monthlySavings, percentageSavings) { if (monthlySavings > 50 || percentageSavings > 0.5) return 'critical'; if (monthlySavings > 20 || percentageSavings > 0.3) return 'high'; if (monthlySavings > 10 || percentageSavings > 0.15) return 'medium'; return 'low'; } calculateTradeoffs(alternative) { const tradeoffs = []; if (alternative.qualityDifference < -0.05) { tradeoffs.push({ aspect: 'quality', impact: alternative.qualityDifference, description: `${Math.abs(alternative.qualityDifference * 100).toFixed(1)}% reduction in average quality`, severity: Math.abs(alternative.qualityDifference) > 0.15 ? 'significant' : 'minor', mitigation: 'Monitor quality closely and revert if unsatisfactory', }); } if (alternative.performanceDifference < -0.1) { tradeoffs.push({ aspect: 'performance', impact: alternative.performanceDifference, description: `${Math.abs(alternative.performanceDifference * 100).toFixed(1)}% increase in response time`, severity: Math.abs(alternative.performanceDifference) > 0.3 ? 'moderate' : 'minor', }); } return tradeoffs; } filterAndRankSuggestions(suggestions, _userStats) { return suggestions .filter(s => s.impact.monthlySavings >= this.config.minSavingsThreshold && s.confidence >= this.config.minConfidenceThreshold && this.config.acceptableEffortLevels.includes(s.implementation.effort)) .sort((a, b) => { // Sort by priority first, then by savings amount const priorityOrder = { critical: 4, high: 3, medium: 2, low: 1 }; const priorityDiff = priorityOrder[b.priority] - priorityOrder[a.priority]; if (priorityDiff !== 0) return priorityDiff; return b.impact.monthlySavings - a.impact.monthlySavings; }) .slice(0, 10); // Limit to top 10 suggestions } generateId() { return Date.now().toString(36) + Math.random().toString(36).substr(2); } /** * Records implementation of a suggestion for tracking */ recordImplementation(userId, suggestionId) { const implemented = this.implementedSuggestions.get(userId) || []; implemented.push(suggestionId); this.implementedSuggestions.set(userId, implemented); } /** * Gets user's usage patterns for analysis */ getUserUsagePatterns(userId) { return this.usagePatterns.get(userId) || []; } /** * Updates configuration */ updateConfig(newConfig) { this.config = { ...this.config, ...newConfig }; } /** * Gets current configuration */ getConfig() { return { ...this.config }; } } //# sourceMappingURL=cost-optimization.js.map