@quantumai/quantum-cli-core
Version:
Quantum CLI Core - Multi-LLM Collaboration System
526 lines • 24.3 kB
JavaScript
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { QueryType } from '../types.js';
/**
* Types of metrics that can have personalized thresholds
*/
export var ThresholdMetricType;
(function (ThresholdMetricType) {
ThresholdMetricType["UNCERTAINTY"] = "uncertainty";
ThresholdMetricType["COST_LIMIT"] = "cost_limit";
ThresholdMetricType["RESPONSE_TIME"] = "response_time";
ThresholdMetricType["VERIFICATION_TRIGGER"] = "verification_trigger";
ThresholdMetricType["COMPARISON_TRIGGER"] = "comparison_trigger";
ThresholdMetricType["SATISFACTION_MINIMUM"] = "satisfaction_minimum";
})(ThresholdMetricType || (ThresholdMetricType = {}));
/**
* Default threshold configuration
*/
export const DEFAULT_THRESHOLD_CONFIG = {
enableOptimization: true,
minInteractionsForOptimization: 10,
learningRate: 0.15,
confidenceThreshold: 0.75,
optimizationFrequency: 15,
riskTolerance: 0.3,
bounds: {
[ThresholdMetricType.UNCERTAINTY]: { min: 0.3, max: 0.9, default: 0.7 },
[ThresholdMetricType.COST_LIMIT]: { min: 0.01, max: 1.0, default: 0.1 },
[ThresholdMetricType.RESPONSE_TIME]: { min: 1000, max: 30000, default: 10000 },
[ThresholdMetricType.VERIFICATION_TRIGGER]: { min: 0.4, max: 0.8, default: 0.6 },
[ThresholdMetricType.COMPARISON_TRIGGER]: { min: 0.5, max: 0.9, default: 0.7 },
[ThresholdMetricType.SATISFACTION_MINIMUM]: { min: 0.5, max: 0.9, default: 0.7 },
},
};
/**
* Personalized threshold optimization engine
*/
export class PersonalizedThresholdEngine {
config;
userProfiles = new Map();
optimizationHistory = new Map();
constructor(config = {}) {
this.config = { ...DEFAULT_THRESHOLD_CONFIG, ...config };
}
/**
* Optimizes thresholds based on user behavior and preferences
*/
optimizeThresholds(userId, queryHistory, _userStats, _preferences) {
// Get or create user profile
let profile = this.userProfiles.get(userId);
if (!profile) {
profile = this.createDefaultProfile(userId);
this.userProfiles.set(userId, profile);
}
if (queryHistory.length < this.config.minInteractionsForOptimization) {
return this.generateDefaultOptimization(userId, profile);
}
// Analyze user behavior patterns
const behaviorAnalysis = this.analyzeBehaviorPatterns(queryHistory, _userStats);
// Calculate optimal thresholds for each metric type
const optimizedThresholds = this.calculateOptimalThresholds(profile, behaviorAnalysis, queryHistory, _preferences);
// Generate threshold changes
const changes = this.generateThresholdChanges(profile, optimizedThresholds);
// Calculate expected improvements
const expectedImprovement = this.calculateExpectedImprovement(profile, changes, behaviorAnalysis);
// Generate reasoning
const reasoning = this.generateOptimizationReasoning(changes, behaviorAnalysis, expectedImprovement);
// Calculate confidence
const confidence = this.calculateOptimizationConfidence(changes, behaviorAnalysis, queryHistory.length);
// Apply changes if confidence is high enough
if (confidence >= this.config.confidenceThreshold) {
this.applyThresholdChanges(profile, changes);
profile.totalOptimizations++;
profile.lastOptimization = new Date();
}
const result = {
thresholds: Array.from(profile.thresholds.values()),
changes,
reasoning,
expectedImprovement,
confidence,
};
// Store optimization history
const history = this.optimizationHistory.get(userId) || [];
history.push(result);
if (history.length > 20)
history.shift(); // Keep only recent history
this.optimizationHistory.set(userId, history);
return result;
}
/**
* Analyzes user behavior patterns from query history
*/
analyzeBehaviorPatterns(queryHistory, _userStats) {
const ratedQueries = queryHistory.filter(q => q.userRating);
const avgSatisfaction = ratedQueries.length > 0
? ratedQueries.reduce((sum, q) => sum + (q.userRating - 1), 0) / (ratedQueries.length * 4)
: 0.5;
// Analyze cost sensitivity based on rating correlation with cost
const costSensitivity = this.calculateCostSensitivity(queryHistory);
// Analyze performance sensitivity based on rating correlation with response time
const performanceSensitivity = this.calculatePerformanceSensitivity(queryHistory);
// Estimate risk tolerance based on usage of verification features
const verificationUsage = queryHistory.filter(q => q.alternativeProviders.length > 0).length / queryHistory.length;
const riskTolerance = 1 - verificationUsage; // Less verification = higher risk tolerance
// Calculate quality preference based on satisfaction vs cost/performance trade-offs
const qualityPreference = this.calculateQualityPreference(queryHistory);
// Usage of comparison features
const comparisonUsage = queryHistory.filter(q => q.alternativeProviders.length > 1).length / queryHistory.length;
// Satisfaction variability
const satisfactionVariability = this.calculateSatisfactionVariability(ratedQueries);
return {
averageSatisfaction: avgSatisfaction,
costSensitivity,
performanceSensitivity,
riskTolerance,
qualityPreference,
verificationUsage,
comparisonUsage,
satisfactionVariability,
};
}
/**
* Calculates optimal thresholds based on behavior analysis
*/
calculateOptimalThresholds(profile, behavior, queryHistory, _preferences) {
const optimizedThresholds = [];
// Optimize uncertainty threshold
for (const queryType of Object.values(QueryType)) {
const typeQueries = queryHistory.filter(q => q.analysis.type === queryType);
if (typeQueries.length >= 3) {
const uncertaintyThreshold = this.optimizeUncertaintyThreshold(profile.userId, queryType, typeQueries, behavior);
optimizedThresholds.push(uncertaintyThreshold);
}
}
// Optimize cost limit
const costThreshold = this.optimizeCostThreshold(profile.userId, queryHistory, behavior);
optimizedThresholds.push(costThreshold);
// Optimize response time threshold
const responseTimeThreshold = this.optimizeResponseTimeThreshold(profile.userId, queryHistory, behavior);
optimizedThresholds.push(responseTimeThreshold);
// Optimize verification trigger threshold
const verificationThreshold = this.optimizeVerificationThreshold(profile.userId, queryHistory, behavior);
optimizedThresholds.push(verificationThreshold);
return optimizedThresholds;
}
/**
* Optimizes uncertainty threshold for a specific query type
*/
optimizeUncertaintyThreshold(userId, queryType, typeQueries, behavior) {
const bounds = this.config.bounds[ThresholdMetricType.UNCERTAINTY];
let optimalValue = bounds.default;
// If user has low risk tolerance, lower uncertainty threshold (more verification)
if (behavior.riskTolerance < 0.3) {
optimalValue = bounds.min + (bounds.default - bounds.min) * 0.5;
}
else if (behavior.riskTolerance > 0.7) {
optimalValue = bounds.default + (bounds.max - bounds.default) * 0.5;
}
// Adjust based on satisfaction patterns
if (behavior.averageSatisfaction < 0.6) {
optimalValue = Math.max(bounds.min, optimalValue - 0.1); // More verification for low satisfaction
}
// Adjust based on quality preference
optimalValue += (behavior.qualityPreference - 0.5) * 0.2;
optimalValue = Math.max(bounds.min, Math.min(bounds.max, optimalValue));
return {
userId,
metricType: ThresholdMetricType.UNCERTAINTY,
queryType,
currentValue: optimalValue,
defaultValue: bounds.default,
confidenceBounds: {
lower: optimalValue - 0.1,
upper: optimalValue + 0.1
},
lastUpdated: new Date(),
adaptationHistory: [],
stability: 0.8,
effectiveness: this.estimateThresholdEffectiveness(ThresholdMetricType.UNCERTAINTY, optimalValue, typeQueries),
};
}
/**
* Optimizes cost limit threshold
*/
optimizeCostThreshold(userId, queryHistory, behavior) {
const bounds = this.config.bounds[ThresholdMetricType.COST_LIMIT];
let optimalValue = bounds.default;
// Adjust based on cost sensitivity
if (behavior.costSensitivity > 0.7) {
optimalValue = bounds.min + (bounds.default - bounds.min) * 0.6; // Lower limit for cost-sensitive users
}
else if (behavior.costSensitivity < 0.3) {
optimalValue = bounds.default + (bounds.max - bounds.default) * 0.4; // Higher limit for cost-insensitive users
}
// Adjust based on usage patterns
const avgCost = queryHistory.reduce((sum, q) => sum + q.cost, 0) / queryHistory.length;
if (avgCost > bounds.default * 0.8) {
optimalValue = Math.min(bounds.max, avgCost * 1.2); // Set limit above current usage
}
optimalValue = Math.max(bounds.min, Math.min(bounds.max, optimalValue));
return {
userId,
metricType: ThresholdMetricType.COST_LIMIT,
queryType: QueryType.GENERAL, // Apply to all query types
currentValue: optimalValue,
defaultValue: bounds.default,
confidenceBounds: {
lower: optimalValue * 0.8,
upper: optimalValue * 1.2
},
lastUpdated: new Date(),
adaptationHistory: [],
stability: 0.9,
effectiveness: this.estimateThresholdEffectiveness(ThresholdMetricType.COST_LIMIT, optimalValue, queryHistory),
};
}
/**
* Optimizes response time threshold
*/
optimizeResponseTimeThreshold(userId, queryHistory, behavior) {
const bounds = this.config.bounds[ThresholdMetricType.RESPONSE_TIME];
let optimalValue = bounds.default;
// Adjust based on performance sensitivity
if (behavior.performanceSensitivity > 0.7) {
optimalValue = bounds.min + (bounds.default - bounds.min) * 0.7; // Lower threshold for performance-sensitive users
}
else if (behavior.performanceSensitivity < 0.3) {
optimalValue = bounds.default + (bounds.max - bounds.default) * 0.3; // Higher threshold for performance-tolerant users
}
// Adjust based on actual response times
const p95ResponseTime = this.calculatePercentile(queryHistory.map(q => q.responseTime), 0.95);
// Set threshold to accommodate 95% of queries
optimalValue = Math.min(bounds.max, p95ResponseTime * 1.1);
optimalValue = Math.max(bounds.min, Math.min(bounds.max, optimalValue));
return {
userId,
metricType: ThresholdMetricType.RESPONSE_TIME,
queryType: QueryType.GENERAL,
currentValue: optimalValue,
defaultValue: bounds.default,
confidenceBounds: {
lower: optimalValue * 0.9,
upper: optimalValue * 1.1
},
lastUpdated: new Date(),
adaptationHistory: [],
stability: 0.8,
effectiveness: this.estimateThresholdEffectiveness(ThresholdMetricType.RESPONSE_TIME, optimalValue, queryHistory),
};
}
/**
* Optimizes verification trigger threshold
*/
optimizeVerificationThreshold(userId, queryHistory, behavior) {
const bounds = this.config.bounds[ThresholdMetricType.VERIFICATION_TRIGGER];
let optimalValue = bounds.default;
// Adjust based on quality preference and risk tolerance
if (behavior.qualityPreference > 0.7 && behavior.riskTolerance < 0.4) {
optimalValue = bounds.min + (bounds.default - bounds.min) * 0.5; // Lower threshold = more verification
}
else if (behavior.qualityPreference < 0.3 && behavior.riskTolerance > 0.6) {
optimalValue = bounds.default + (bounds.max - bounds.default) * 0.5; // Higher threshold = less verification
}
// Adjust based on verification usage patterns
const verificationQueries = queryHistory.filter(q => q.alternativeProviders.length > 0);
if (verificationQueries.length > 0) {
const verificationSatisfaction = verificationQueries
.filter(q => q.userRating)
.reduce((sum, q) => sum + (q.userRating - 1), 0) / (verificationQueries.length * 4);
if (verificationSatisfaction > behavior.averageSatisfaction + 0.1) {
optimalValue = Math.max(bounds.min, optimalValue - 0.1); // More verification helps
}
}
optimalValue = Math.max(bounds.min, Math.min(bounds.max, optimalValue));
return {
userId,
metricType: ThresholdMetricType.VERIFICATION_TRIGGER,
queryType: QueryType.GENERAL,
currentValue: optimalValue,
defaultValue: bounds.default,
confidenceBounds: {
lower: optimalValue - 0.05,
upper: optimalValue + 0.05
},
lastUpdated: new Date(),
adaptationHistory: [],
stability: 0.75,
effectiveness: this.estimateThresholdEffectiveness(ThresholdMetricType.VERIFICATION_TRIGGER, optimalValue, queryHistory),
};
}
/**
* Helper calculation methods
*/
calculateCostSensitivity(queryHistory) {
const ratedQueries = queryHistory.filter(q => q.userRating);
if (ratedQueries.length < 3)
return 0.5;
const ratings = ratedQueries.map(q => (q.userRating - 1) / 4);
const costs = ratedQueries.map(q => q.cost);
const correlation = this.calculateCorrelation(ratings, costs.map(c => -c)); // Negative because lower cost should correlate with higher rating
return Math.max(0, Math.min(1, correlation + 0.5)); // Convert -1,1 to 0,1
}
calculatePerformanceSensitivity(queryHistory) {
const ratedQueries = queryHistory.filter(q => q.userRating);
if (ratedQueries.length < 3)
return 0.5;
const ratings = ratedQueries.map(q => (q.userRating - 1) / 4);
const responseTimes = ratedQueries.map(q => q.responseTime);
const correlation = this.calculateCorrelation(ratings, responseTimes.map(t => -t)); // Negative because lower time should correlate with higher rating
return Math.max(0, Math.min(1, correlation + 0.5));
}
calculateQualityPreference(queryHistory) {
const ratedQueries = queryHistory.filter(q => q.userRating);
if (ratedQueries.length < 3)
return 0.5;
// Users who consistently rate queries highly despite higher costs/times prefer quality
const highRatingQueries = ratedQueries.filter(q => q.userRating >= 4);
const highRatingRatio = highRatingQueries.length / ratedQueries.length;
const avgCostOfHighRated = highRatingQueries.length > 0
? highRatingQueries.reduce((sum, q) => sum + q.cost, 0) / highRatingQueries.length
: 0;
const avgCostOverall = ratedQueries.reduce((sum, q) => sum + q.cost, 0) / ratedQueries.length;
const costPremiumTolerance = avgCostOverall > 0 ? avgCostOfHighRated / avgCostOverall : 1;
return Math.min(1, highRatingRatio * costPremiumTolerance);
}
calculateSatisfactionVariability(ratedQueries) {
if (ratedQueries.length < 2)
return 0;
const ratings = ratedQueries.map(q => q.userRating);
const mean = ratings.reduce((a, b) => a + b, 0) / ratings.length;
const variance = ratings.reduce((sum, rating) => sum + Math.pow(rating - mean, 2), 0) / ratings.length;
return Math.min(1, variance / 4); // Normalize to 0-1
}
calculateCorrelation(x, y) {
if (x.length !== y.length || x.length < 2)
return 0;
const n = x.length;
const meanX = x.reduce((a, b) => a + b, 0) / n;
const meanY = y.reduce((a, b) => a + b, 0) / n;
let numerator = 0;
let sumXSquared = 0;
let sumYSquared = 0;
for (let i = 0; i < n; i++) {
const dx = x[i] - meanX;
const dy = y[i] - meanY;
numerator += dx * dy;
sumXSquared += dx * dx;
sumYSquared += dy * dy;
}
const denominator = Math.sqrt(sumXSquared * sumYSquared);
return denominator === 0 ? 0 : numerator / denominator;
}
calculatePercentile(values, percentile) {
const sorted = values.slice().sort((a, b) => a - b);
const index = Math.ceil(sorted.length * percentile) - 1;
return sorted[Math.max(0, index)];
}
estimateThresholdEffectiveness(_metricType, _value, _queries) {
// Simplified effectiveness estimation
// In practice, this would analyze how well the threshold performs
return 0.8; // Placeholder
}
/**
* Creates default profile for new user
*/
createDefaultProfile(userId) {
const thresholds = new Map();
// Create default thresholds for each metric type
for (const [metricType, bounds] of Object.entries(this.config.bounds)) {
const key = `${metricType}|${QueryType.GENERAL}`;
thresholds.set(key, {
userId,
metricType: metricType,
queryType: QueryType.GENERAL,
currentValue: bounds.default,
defaultValue: bounds.default,
confidenceBounds: {
lower: bounds.default * 0.9,
upper: bounds.default * 1.1
},
lastUpdated: new Date(),
adaptationHistory: [],
stability: 1.0,
effectiveness: 0.5,
});
}
return {
userId,
thresholds,
riskTolerance: 0.5,
costSensitivity: 0.5,
qualityPreference: 0.5,
adaptationEnabled: true,
totalOptimizations: 0,
lastOptimization: new Date(),
};
}
generateDefaultOptimization(userId, profile) {
return {
thresholds: Array.from(profile.thresholds.values()),
changes: [],
reasoning: ['Insufficient interaction history for optimization'],
expectedImprovement: { satisfaction: 0, cost: 0, performance: 0 },
confidence: 0.1,
};
}
generateThresholdChanges(profile, optimizedThresholds) {
const changes = [];
for (const optimized of optimizedThresholds) {
const key = `${optimized.metricType}|${optimized.queryType}`;
const current = profile.thresholds.get(key);
if (current && Math.abs(current.currentValue - optimized.currentValue) > 0.05) {
changes.push({
timestamp: new Date(),
oldValue: current.currentValue,
newValue: optimized.currentValue,
reason: {
type: 'user_behavior',
description: `Optimizing ${optimized.metricType} threshold based on behavior analysis`,
evidence: ['User behavior pattern analysis', 'Satisfaction correlation analysis'],
dataPoints: 10, // Placeholder
},
performance: { before: current.effectiveness, after: optimized.effectiveness },
confidence: 0.8,
});
}
}
return changes;
}
calculateExpectedImprovement(profile, changes, _behavior) {
let satisfaction = 0;
let cost = 0;
let performance = 0;
for (const change of changes) {
satisfaction += (change.performance.after - change.performance.before) * 0.1;
if (change.reason.description.includes('cost')) {
cost += change.confidence * 0.05;
}
if (change.reason.description.includes('performance') || change.reason.description.includes('response_time')) {
performance += change.confidence * 0.05;
}
}
return {
satisfaction: Math.max(0, Math.min(0.2, satisfaction)),
cost: Math.max(0, Math.min(0.1, cost)),
performance: Math.max(0, Math.min(0.1, performance)),
};
}
generateOptimizationReasoning(changes, _behavior, expectedImprovement) {
const reasoning = [];
if (changes.length === 0) {
reasoning.push('No significant threshold adjustments needed');
return reasoning;
}
reasoning.push(`Optimizing ${changes.length} threshold(s) based on behavior analysis`);
for (const change of changes) {
const direction = change.newValue > change.oldValue ? 'increased' : 'decreased';
reasoning.push(`${change.reason.description}: ${direction} by ${Math.abs((change.newValue - change.oldValue) * 100).toFixed(1)}%`);
}
if (expectedImprovement.satisfaction > 0.02) {
reasoning.push(`Expected satisfaction improvement: ${(expectedImprovement.satisfaction * 100).toFixed(1)}%`);
}
return reasoning;
}
calculateOptimizationConfidence(changes, behavior, historyLength) {
if (changes.length === 0)
return 1.0;
let confidence = 0.5;
// More history = higher confidence
confidence += Math.min(historyLength / 50, 0.3);
// Average change confidence
const avgChangeConfidence = changes.reduce((sum, c) => sum + c.confidence, 0) / changes.length;
confidence += avgChangeConfidence * 0.3;
// Behavioral consistency bonus
if (behavior.satisfactionVariability < 0.3) {
confidence += 0.2; // Consistent user behavior = higher confidence
}
return Math.max(0, Math.min(1, confidence));
}
applyThresholdChanges(profile, changes) {
for (const change of changes) {
// Find the threshold to update (simplified - would need better matching in practice)
for (const [_key, threshold] of profile.thresholds.entries()) {
if (Math.abs(threshold.currentValue - change.oldValue) < 0.001) {
threshold.currentValue = change.newValue;
threshold.lastUpdated = change.timestamp;
threshold.adaptationHistory.push(change);
// Keep only recent history
if (threshold.adaptationHistory.length > 10) {
threshold.adaptationHistory = threshold.adaptationHistory.slice(-10);
}
break;
}
}
}
}
/**
* Gets current threshold value for a user
*/
getThreshold(userId, metricType, queryType = QueryType.GENERAL) {
const profile = this.userProfiles.get(userId);
if (!profile) {
return this.config.bounds[metricType].default;
}
const key = `${metricType}|${queryType}`;
const threshold = profile.thresholds.get(key);
return threshold ? threshold.currentValue : this.config.bounds[metricType].default;
}
/**
* Updates configuration
*/
updateConfig(newConfig) {
this.config = { ...this.config, ...newConfig };
}
/**
* Gets current configuration
*/
getConfig() {
return { ...this.config };
}
}
//# sourceMappingURL=personalized-thresholds.js.map