@gork-labs/secondbrain-mcp
Version:
Second Brain MCP Server - Agent team orchestration with dynamic tool discovery
1,040 lines • 120 kB
JavaScript
/**
* Machine Learning Engine for SecondBrain MCP
*
* Provides adaptive intelligence with ensemble methods and auto-optimization:
* - Ensemble predictions using multiple specialized models
* - A/B testing framework for continuous experimentation
* - Auto-optimization for parameter tuning and adaptation
* - Meta-learning capabilities for improved learning effectiveness
* - Predictive quality scoring and analytics (Phase 7 foundation)
*
* Author: @bohdan-shulha
* Created: 2025-07-24T23:19:59+02:00
* Phase: 8 - Ensemble Methods & Auto-Optimization (Adaptive Intelligence)
*/
import { logger } from '../utils/logger.js';
export class MLEngine {
analyticsManager;
config;
models;
trainingData;
lastModelUpdate;
learningPatterns;
optimizationHistory;
constructor(analyticsManager, config) {
this.analyticsManager = analyticsManager;
this.config = {
enabled: true,
modelUpdateInterval: 24 * 60 * 60 * 1000, // 24 hours
minTrainingDataPoints: 50,
predictionConfidenceThreshold: 0.7,
adaptiveThresholdsEnabled: true,
autoOptimizationEnabled: true,
...config
};
this.models = new Map();
this.trainingData = [];
this.lastModelUpdate = 0;
this.learningPatterns = new Map();
this.optimizationHistory = new Map();
this.initializeModels();
logger.info('ML Engine initialized', {
enabled: this.config.enabled,
modelUpdateInterval: this.config.modelUpdateInterval,
minTrainingDataPoints: this.config.minTrainingDataPoints
});
}
/**
* Initialize ML models for different prediction tasks
*/
initializeModels() {
// Quality prediction model
this.models.set('quality_prediction', {
name: 'quality_prediction',
type: 'regression',
version: '1.0.0',
features: [
'chatmode_encoded',
'requirements_length',
'criteria_complexity',
'historical_avg_score',
'recent_trend',
'time_of_day',
'refinement_count'
],
weights: new Map([
['chatmode_encoded', 0.25],
['requirements_length', 0.15],
['criteria_complexity', 0.20],
['historical_avg_score', 0.30],
['recent_trend', 0.10]
]),
accuracy: 0.0,
lastTrained: 0,
trainingCount: 0
});
// Refinement success prediction model
this.models.set('refinement_success', {
name: 'refinement_success',
type: 'classification',
version: '1.0.0',
features: [
'initial_score',
'score_gap',
'chatmode_encoded',
'refinement_attempt',
'issue_types',
'previous_refinement_success'
],
weights: new Map([
['initial_score', 0.30],
['score_gap', 0.25],
['refinement_attempt', 0.20],
['issue_types', 0.15],
['previous_refinement_success', 0.10]
]),
accuracy: 0.0,
lastTrained: 0,
trainingCount: 0
});
// Threshold optimization model
this.models.set('threshold_optimization', {
name: 'threshold_optimization',
type: 'regression',
version: '1.0.0',
features: [
'chatmode_encoded',
'current_threshold',
'success_rate',
'refinement_rate',
'user_satisfaction',
'processing_time'
],
weights: new Map([
['success_rate', 0.35],
['refinement_rate', 0.25],
['user_satisfaction', 0.20],
['processing_time', 0.10],
['current_threshold', 0.10]
]),
accuracy: 0.0,
lastTrained: 0,
trainingCount: 0
});
}
/**
* Predict quality score for a given context before actual validation
*/
async predictQualityScore(context) {
if (!this.config.enabled) {
return {
predictedScore: 0.75, // Default fallback
confidence: 0.0,
predictionBasis: ['ML disabled - using default'],
riskFactors: [],
successFactors: []
};
}
try {
const features = await this.extractFeatures(context);
// Always run risk factor analysis regardless of model state
const riskFactors = await this.identifyRiskFactors(context, features);
const successFactors = await this.identifySuccessFactors(context, features);
const model = this.models.get('quality_prediction');
if (!model || model.trainingCount < this.config.minTrainingDataPoints) {
const fallback = this.fallbackPrediction(context, 'Insufficient training data');
// Override with computed risk/success factors
fallback.riskFactors = riskFactors;
fallback.successFactors = successFactors;
return fallback;
}
const prediction = this.runInference(model, features);
const confidence = this.calculatePredictionConfidence(model, features);
logger.info('Quality score predicted', {
subagent: context.subagent,
predictedScore: prediction,
confidence,
riskFactorCount: riskFactors.length,
successFactorCount: successFactors.length
});
return {
predictedScore: Math.max(0, Math.min(1, prediction)),
confidence,
predictionBasis: [`Model: ${model.name} v${model.version}`, `Training samples: ${model.trainingCount}`],
riskFactors,
successFactors
};
}
catch (error) {
logger.error('Quality prediction failed', {
error: error instanceof Error ? error.message : 'Unknown error',
context: context.subagent
});
return this.fallbackPrediction(context, 'Prediction error');
}
}
/**
* Predict refinement success probability
*/
async predictRefinementSuccess(currentScore, context, refinementAttempt) {
if (!this.config.enabled) {
return {
prediction: 0.5,
confidence: 0.0,
reasoning: 'ML disabled',
features: new Map()
};
}
try {
const features = new Map([
['initial_score', currentScore],
['score_gap', this.getQualityThreshold(context.subagent) - currentScore],
['chatmode_encoded', this.encodeChatmode(context.subagent)],
['refinement_attempt', refinementAttempt],
['issue_types', await this.analyzeIssueTypes(context)],
['previous_refinement_success', await this.getPreviousRefinementSuccess(context.subagent)]
]);
const model = this.models.get('refinement_success');
if (!model || model.trainingCount < this.config.minTrainingDataPoints) {
return {
prediction: currentScore > 0.6 ? 0.7 : 0.3,
confidence: 0.3,
reasoning: 'Insufficient training data - using heuristic',
features
};
}
const prediction = this.runInference(model, features);
const confidence = this.calculatePredictionConfidence(model, features);
return {
prediction: Math.max(0, Math.min(1, prediction)),
confidence,
reasoning: `Based on ${model.trainingCount} training samples`,
features
};
}
catch (error) {
logger.error('Refinement success prediction failed', {
error: error instanceof Error ? error.message : 'Unknown error',
currentScore,
refinementAttempt
});
return {
prediction: 0.5,
confidence: 0.0,
reasoning: 'Prediction error',
features: new Map()
};
}
}
/**
* Generate optimization suggestions based on ML analysis
*/
async generateOptimizationSuggestions() {
if (!this.config.autoOptimizationEnabled) {
return [];
}
const suggestions = [];
try {
// Analyze threshold optimization opportunities
const thresholdSuggestions = await this.analyzeThresholdOptimization();
suggestions.push(...thresholdSuggestions);
// Analyze rule weight optimization
const ruleWeightSuggestions = await this.analyzeRuleWeightOptimization();
suggestions.push(...ruleWeightSuggestions);
// Analyze refinement strategy optimization
const refinementSuggestions = await this.analyzeRefinementStrategyOptimization();
suggestions.push(...refinementSuggestions);
// Filter suggestions by confidence threshold
const filteredSuggestions = suggestions.filter(s => s.confidence >= this.config.predictionConfidenceThreshold);
logger.info('Generated optimization suggestions', {
totalSuggestions: suggestions.length,
filteredSuggestions: filteredSuggestions.length,
confidenceThreshold: this.config.predictionConfidenceThreshold
});
return filteredSuggestions;
}
catch (error) {
logger.error('Failed to generate optimization suggestions', {
error: error instanceof Error ? error.message : 'Unknown error'
});
return [];
}
}
/**
* Learn from validation results to improve predictions
*/
async learnFromValidation(context, assessment, prediction) {
if (!this.config.enabled) {
return;
}
try {
// Create training data point
const features = await this.extractFeatures(context);
const trainingPoint = {
timestamp: Date.now(),
features,
actualScore: assessment.overallScore,
actualPassed: assessment.passed,
processingTime: assessment.processingTime,
subagent: context.subagent,
prediction: prediction?.predictedScore
};
this.trainingData.push(trainingPoint);
// Update learning patterns
await this.updateLearningPatterns(context, assessment);
// Check if model retraining is needed
if (this.shouldRetrain()) {
await this.retrainModels();
}
logger.debug('Learning from validation completed', {
subagent: context.subagent,
actualScore: assessment.overallScore,
predictedScore: prediction?.predictedScore,
trainingDataSize: this.trainingData.length
});
}
catch (error) {
logger.error('Failed to learn from validation', {
error: error instanceof Error ? error.message : 'Unknown error',
subagent: context.subagent
});
}
}
/**
* Get machine learning insights about system performance
*/
async getMLInsights() {
const insights = [];
try {
// Model performance insights with enhanced analysis
for (const [name, model] of this.models) {
if (model.trainingCount > 0) {
const accuracyLevel = model.accuracy > 0.8 ? 'excellent' : model.accuracy > 0.6 ? 'good' : 'poor';
const dataSufficiency = model.trainingCount >= this.config.minTrainingDataPoints ? 'sufficient' : 'insufficient';
insights.push({
type: 'model_performance',
severity: model.accuracy > 0.8 ? 'info' : model.accuracy > 0.6 ? 'warning' : 'critical',
title: `${name} Model Performance (${accuracyLevel})`,
description: `Accuracy: ${(model.accuracy * 100).toFixed(1)}% with ${model.trainingCount} training samples (${dataSufficiency} data)`,
actionRequired: model.accuracy < 0.6 || model.trainingCount < this.config.minTrainingDataPoints,
recommendation: this.generateModelRecommendation(model)
});
}
}
// Learning pattern insights with pattern analysis
const highValuePatterns = Array.from(this.learningPatterns.values())
.filter(p => p.frequency > 10 && p.confidence > 0.7)
.sort((a, b) => b.confidence - a.confidence);
for (const pattern of highValuePatterns.slice(0, 5)) { // Top 5 patterns
const patternType = pattern.successRate > 0.8 ? 'success_pattern' : 'improvement_opportunity';
const severity = pattern.successRate > 0.8 ? 'info' : pattern.successRate < 0.4 ? 'critical' : 'warning';
insights.push({
type: 'learning_pattern',
severity,
title: `${patternType === 'success_pattern' ? 'Success' : 'Improvement'} Pattern: ${this.formatPatternName(pattern.pattern)}`,
description: `Frequency: ${pattern.frequency}, Success Rate: ${(pattern.successRate * 100).toFixed(1)}%, Confidence: ${(pattern.confidence * 100).toFixed(1)}%`,
actionRequired: pattern.successRate < 0.5,
recommendation: pattern.recommendedActions.length > 0 ? pattern.recommendedActions[0] : 'Monitor pattern development'
});
}
// Training data quality insights
const dataAge = this.trainingData.length > 0 ? Date.now() - Math.min(...this.trainingData.map(d => d.timestamp)) : 0;
const stalenessThreshold = 7 * 24 * 60 * 60 * 1000; // 7 days
if (dataAge > stalenessThreshold && this.trainingData.length > 0) {
insights.push({
type: 'training_data',
severity: dataAge > 14 * 24 * 60 * 60 * 1000 ? 'critical' : 'warning',
title: 'Training Data Staleness',
description: `Oldest training data is ${Math.floor(dataAge / (24 * 60 * 60 * 1000))} days old`,
actionRequired: true,
recommendation: 'Consider refreshing training data to maintain model relevance and accuracy'
});
}
// Data distribution insights
if (this.trainingData.length > 20) {
const chatmodeDistribution = this.analyzeDataDistribution();
const imbalancedChatmodes = Object.entries(chatmodeDistribution)
.filter(([_, count]) => count < 5)
.map(([chatmode, _]) => chatmode);
if (imbalancedChatmodes.length > 0) {
insights.push({
type: 'data_distribution',
severity: 'warning',
title: 'Imbalanced Training Data',
description: `Limited data for chatmodes: ${imbalancedChatmodes.join(', ')}`,
actionRequired: true,
recommendation: 'Collect more training data for underrepresented chatmodes to improve prediction accuracy'
});
}
}
// Real-time adaptation insights
const adaptationInsights = this.generateAdaptationInsights();
insights.push(...adaptationInsights);
// Phase 7: Predictive analytics insights
const predictiveAnalytics = await this.generatePredictiveAnalytics();
// Add quality forecast insights
if (predictiveAnalytics.qualityForecast.confidence > 0.5) {
insights.push({
type: 'quality_forecast',
severity: predictiveAnalytics.qualityForecast.trendDirection === 'declining' ? 'warning' : 'info',
title: `Quality Forecast: ${predictiveAnalytics.qualityForecast.subagent}`,
description: `Predicted score: ${(predictiveAnalytics.qualityForecast.predictedScore * 100).toFixed(1)}% (${predictiveAnalytics.qualityForecast.trendDirection})`,
actionRequired: predictiveAnalytics.qualityForecast.trendDirection === 'declining',
recommendation: predictiveAnalytics.qualityForecast.trendDirection === 'declining'
? 'Monitor quality trends and consider intervention strategies'
: 'Quality trends are stable or improving',
forecastData: {
prediction: predictiveAnalytics.qualityForecast.predictedScore,
confidenceInterval: predictiveAnalytics.qualityForecast.confidenceInterval,
timeHorizon: predictiveAnalytics.qualityForecast.forecastHorizon,
methodology: 'Linear trend analysis with confidence intervals'
}
});
}
// Add top anomaly insights
for (const anomaly of predictiveAnalytics.anomalies.slice(0, 3)) {
insights.push({
type: 'anomaly_detection',
severity: anomaly.severity === 'high' ? 'critical' : anomaly.severity === 'medium' ? 'warning' : 'info',
title: `Anomaly Detected: ${anomaly.subagent}`,
description: anomaly.description,
actionRequired: anomaly.severity === 'high',
recommendation: anomaly.severity === 'high'
? 'Investigate immediately - significant deviation detected'
: 'Monitor for pattern continuation',
anomalyDetails: {
detectionMethod: 'Z-score statistical analysis',
severity: anomaly.deviationScore,
expectedValue: anomaly.expectedValue,
actualValue: anomaly.actualValue,
deviation: anomaly.deviationScore
}
});
}
// Add cross-chatmode pattern insights
for (const pattern of predictiveAnalytics.crossSubagentPatterns.slice(0, 2)) {
insights.push({
type: 'cross_chatmode_analysis',
severity: pattern.significance === 'high' ? 'warning' : 'info',
title: `Cross-Chatmode Pattern: ${pattern.pattern}`,
description: pattern.description,
actionRequired: pattern.actionable && pattern.significance === 'high',
recommendation: pattern.actionable
? 'Consider optimizing affected chatmodes together'
: 'Pattern noted for future optimization'
});
}
// Add proactive alerts
for (const alert of predictiveAnalytics.proactiveAlerts.slice(0, 2)) {
insights.push({
type: 'proactive_alert',
severity: alert.severity,
title: `Proactive Alert: ${alert.alertType.replace(/_/g, ' ').replace(/\b\w/g, l => l.toUpperCase())}`,
description: alert.description,
actionRequired: alert.severity === 'critical',
recommendation: alert.recommendedActions.join('; ')
});
}
}
catch (error) {
logger.error('Failed to generate ML insights', {
error: error instanceof Error ? error.message : 'Unknown error'
});
insights.push({
type: 'system_error',
severity: 'critical',
title: 'ML Insights Generation Failed',
description: 'Unable to generate ML insights due to system error',
actionRequired: true,
recommendation: 'Check ML Engine configuration and logs for detailed error information'
});
}
return insights;
}
/**
* Phase 7: Advanced Predictive Analytics
* Generate comprehensive predictive analytics with forecasting and anomaly detection
*/
async generatePredictiveAnalytics() {
try {
const qualityForecast = await this.generateQualityForecast();
const anomalies = await this.detectAnomalies();
const crossSubagentPatterns = await this.analyzeCrossChatmodePatterns();
const proactiveAlerts = await this.generateProactiveAlerts(anomalies);
return {
qualityForecast,
anomalies,
crossSubagentPatterns,
proactiveAlerts
};
}
catch (error) {
logger.error('Failed to generate predictive analytics', {
error: error instanceof Error ? error.message : 'Unknown error'
});
return {
qualityForecast: {
subagent: 'system',
currentScore: 0,
predictedScore: 0,
confidenceInterval: [0, 0],
forecastHorizon: '24h',
trendDirection: 'stable',
confidence: 0
},
anomalies: [],
crossSubagentPatterns: [],
proactiveAlerts: [{
alertType: 'anomaly_cluster',
severity: 'critical',
description: 'Predictive analytics system error',
affectedComponents: ['ML Engine'],
recommendedActions: ['Check system logs', 'Restart analytics service'],
confidence: 1.0,
estimatedImpact: 'High - Predictive capabilities unavailable'
}]
};
}
}
/**
* Generate quality forecast using time series analysis
*/
async generateQualityForecast() {
const chatmodes = ['Security Engineer', 'Software Engineer', 'DevOps Engineer', 'Database Architect', 'Software Architect'];
// Find chatmode with most data for primary forecast
let primaryChatmode = 'Security Engineer';
let maxDataPoints = 0;
for (const chatmode of chatmodes) {
const chatmodeData = this.trainingData.filter(d => d.subagent === chatmode);
if (chatmodeData.length > maxDataPoints) {
maxDataPoints = chatmodeData.length;
primaryChatmode = chatmode;
}
}
const chatmodeData = this.trainingData.filter(d => d.subagent === primaryChatmode);
if (chatmodeData.length < 5) {
return {
subagent: primaryChatmode,
currentScore: 0.75,
predictedScore: 0.75,
confidenceInterval: [0.65, 0.85],
forecastHorizon: '24h',
trendDirection: 'stable',
confidence: 0.3
};
}
// Sort by timestamp for time series analysis
const sortedData = chatmodeData.sort((a, b) => a.timestamp - b.timestamp);
const scores = sortedData.map(d => d.actualScore);
// Calculate current moving average and trend
const windowSize = Math.min(5, scores.length);
const recentScores = scores.slice(-windowSize);
const currentScore = recentScores.reduce((sum, score) => sum + score, 0) / recentScores.length;
// Linear trend analysis
const { slope, intercept, rSquared } = this.calculateLinearTrend(scores);
const predictedScore = Math.max(0, Math.min(1, currentScore + slope * 24)); // 24 hour projection
// Calculate confidence interval based on prediction variance
const residuals = scores.map((score, i) => score - (intercept + slope * i));
const variance = residuals.reduce((sum, r) => sum + r * r, 0) / residuals.length;
const standardError = Math.sqrt(variance);
const confidenceMargin = 1.96 * standardError; // 95% confidence interval
const confidenceInterval = [
Math.max(0, predictedScore - confidenceMargin),
Math.min(1, predictedScore + confidenceMargin)
];
// Determine trend direction
let trendDirection = 'stable';
if (Math.abs(slope) > 0.01) { // 1% change threshold
trendDirection = slope > 0 ? 'improving' : 'declining';
}
// Confidence based on R-squared and data quantity
const confidence = Math.min(0.95, rSquared * 0.7 + (Math.min(chatmodeData.length, 50) / 50) * 0.3);
return {
subagent: primaryChatmode,
currentScore,
predictedScore,
confidenceInterval,
forecastHorizon: '24h',
trendDirection,
confidence
};
}
/**
* Detect anomalies in system behavior using statistical methods
*/
async detectAnomalies() {
const anomalies = [];
if (this.trainingData.length < 10) {
return anomalies; // Need sufficient data for anomaly detection
}
const chatmodes = [...new Set(this.trainingData.map(d => d.subagent))];
for (const chatmode of chatmodes) {
const chatmodeData = this.trainingData.filter(d => d.subagent === chatmode);
if (chatmodeData.length < 5)
continue;
// Detect score anomalies using z-score method
const scores = chatmodeData.map(d => d.actualScore);
const scoreAnomalies = this.detectZScoreAnomalies(scores, 'quality_score', chatmode);
anomalies.push(...scoreAnomalies);
// Detect processing time anomalies
const processingTimes = chatmodeData.map(d => d.processingTime);
const timeAnomalies = this.detectZScoreAnomalies(processingTimes, 'processing_time', chatmode);
anomalies.push(...timeAnomalies);
// Detect pattern-based anomalies
const patternAnomalies = this.detectPatternAnomalies(chatmodeData, chatmode);
anomalies.push(...patternAnomalies);
}
// Sort by severity and recency
return anomalies
.sort((a, b) => {
const severityOrder = { 'high': 3, 'medium': 2, 'low': 1 };
if (severityOrder[a.severity] !== severityOrder[b.severity]) {
return severityOrder[b.severity] - severityOrder[a.severity];
}
return new Date(b.detectedAt).getTime() - new Date(a.detectedAt).getTime();
})
.slice(0, 10); // Top 10 most significant anomalies
}
/**
* Analyze patterns across different chatmodes for correlations
*/
async analyzeCrossChatmodePatterns() {
const patterns = [];
if (this.trainingData.length < 20) {
return patterns; // Need sufficient data for cross-analysis
}
const chatmodes = [...new Set(this.trainingData.map(d => d.subagent))];
if (chatmodes.length < 2) {
return patterns; // Need multiple chatmodes for correlation
}
// Analyze quality score correlations between chatmodes
for (let i = 0; i < chatmodes.length; i++) {
for (let j = i + 1; j < chatmodes.length; j++) {
const chatmode1 = chatmodes[i];
const chatmode2 = chatmodes[j];
const correlation = this.calculateChatmodeCorrelation(chatmode1, chatmode2);
if (Math.abs(correlation) > 0.6) { // Strong correlation threshold
const significance = Math.abs(correlation) > 0.8 ? 'high' : 'medium';
patterns.push({
pattern: `Quality Correlation: ${chatmode1} ↔ ${chatmode2}`,
affectedSubagents: [chatmode1, chatmode2],
correlation,
significance,
description: correlation > 0
? `Strong positive correlation (${correlation.toFixed(3)}) suggests these chatmodes share quality patterns`
: `Strong negative correlation (${correlation.toFixed(3)}) suggests inverse quality relationship`,
actionable: true
});
}
}
}
// Analyze processing time patterns
const processingTimePattern = this.analyzeProcessingTimePatterns(chatmodes);
if (processingTimePattern) {
patterns.push(processingTimePattern);
}
// Analyze success rate patterns
const successRatePattern = this.analyzeSuccessRatePatterns(chatmodes);
if (successRatePattern) {
patterns.push(successRatePattern);
}
return patterns.slice(0, 5); // Top 5 most significant patterns
}
/**
* Generate proactive alerts based on detected patterns and anomalies
*/
async generateProactiveAlerts(anomalies) {
const alerts = [];
// High severity anomaly clusters
const highSeverityAnomalies = anomalies.filter(a => a.severity === 'high');
if (highSeverityAnomalies.length >= 3) {
alerts.push({
alertType: 'anomaly_cluster',
severity: 'critical',
description: `Multiple high-severity anomalies detected (${highSeverityAnomalies.length} incidents)`,
affectedComponents: [...new Set(highSeverityAnomalies.map(a => a.subagent))],
recommendedActions: [
'Investigate system configuration changes',
'Review recent deployment changes',
'Check resource availability',
'Analyze error logs for patterns'
],
confidence: 0.9,
estimatedImpact: 'High - System stability at risk'
});
}
// Performance degradation trend
const performanceTrend = this.analyzePerformanceTrend();
if (performanceTrend.declining) {
alerts.push({
alertType: 'performance_degradation',
severity: performanceTrend.severity,
description: performanceTrend.description,
affectedComponents: performanceTrend.affectedSubagents,
recommendedActions: [
'Monitor system resources',
'Review recent configuration changes',
'Consider scaling up resources',
'Analyze bottlenecks in processing pipeline'
],
confidence: performanceTrend.confidence,
estimatedImpact: 'Medium - User experience degradation'
});
}
// Quality decline prediction
const qualityTrend = this.analyzeQualityTrend();
if (qualityTrend.declining) {
alerts.push({
alertType: 'quality_decline',
severity: qualityTrend.severity,
description: qualityTrend.description,
affectedComponents: qualityTrend.affectedSubagents,
recommendedActions: [
'Review quality criteria settings',
'Retrain ML models with recent data',
'Analyze failing validation patterns',
'Update refinement strategies'
],
confidence: qualityTrend.confidence,
estimatedImpact: 'High - Output quality degradation'
});
}
// Trend reversal detection
const trendReversals = this.detectTrendReversals();
for (const reversal of trendReversals) {
alerts.push({
alertType: 'trend_reversal',
severity: 'warning',
description: `Unexpected trend reversal detected in ${reversal.subagent}: ${reversal.description}`,
affectedComponents: [reversal.subagent],
recommendedActions: [
'Investigate underlying causes',
'Review recent data patterns',
'Validate trend detection algorithms',
'Monitor for continued reversal'
],
confidence: reversal.confidence,
estimatedImpact: 'Medium - Predictive accuracy affected'
});
}
return alerts.slice(0, 8); // Top 8 most critical alerts
}
/**
* Extract features from validation context for ML models
*/
async extractFeatures(context) {
const features = new Map();
// Chatmode encoding
features.set('chatmode_encoded', this.encodeChatmode(context.subagent));
// Requirements complexity
features.set('requirements_length', context.requirements.length);
features.set('criteria_complexity', context.qualityCriteria.split(',').length);
// Content complexity analysis
features.set('content_complexity', this.calculateContentComplexity(context.requirements));
// Historical performance
const trends = this.analyticsManager.getQualityTrends(context.subagent, 7);
features.set('historical_avg_score', trends.scoreAverage);
features.set('recent_trend', this.calculateTrend(context.subagent));
// Time-based features
const now = new Date();
features.set('time_of_day', now.getHours());
features.set('day_of_week', now.getDay());
return features;
}
/**
* Calculate content complexity based on technical terms and concepts
*/
calculateContentComplexity(requirements) {
const technicalTerms = [
'microservices', 'architecture', 'scalable', 'fault-tolerant', 'real-time',
'event sourcing', 'CQRS', 'security', 'monitoring', 'observability',
'cloud', 'GDPR', 'compliance', 'performance', 'API', 'database',
'authentication', 'authorization', 'encryption', 'async', 'sync',
'distributed', 'containerization', 'orchestration', 'DevOps',
'CI/CD', 'testing', 'deployment', 'infrastructure', 'networking'
];
const lowerReq = requirements.toLowerCase();
let complexity = 0;
// Technical term density
const termCount = technicalTerms.filter(term => lowerReq.includes(term)).length;
complexity += Math.min(termCount / 10, 0.5); // Max 0.5 from terms
// Sentence complexity (based on length and structure)
const sentences = requirements.split(/[.!?]+/).filter(s => s.trim().length > 0);
const avgSentenceLength = sentences.reduce((sum, s) => sum + s.length, 0) / sentences.length;
complexity += Math.min(avgSentenceLength / 200, 0.3); // Max 0.3 from sentence length
// Multiple concepts detection
const conceptKeywords = ['and', 'while', 'with', 'including', 'supporting', 'maintaining'];
const conceptCount = conceptKeywords.filter(keyword => lowerReq.includes(keyword)).length;
complexity += Math.min(conceptCount / 10, 0.2); // Max 0.2 from concepts
return Math.min(complexity, 1.0);
}
/**
* Run inference using a trained model
*/
runInference(model, features) {
let result = 0;
let totalWeight = 0;
for (const [feature, weight] of model.weights) {
const featureValue = features.get(feature) || 0;
result += featureValue * weight;
totalWeight += Math.abs(weight);
}
// Normalize result
if (totalWeight > 0) {
result = result / totalWeight;
}
// Apply sigmoid for bounded output
return 1 / (1 + Math.exp(-result));
}
/**
* Calculate prediction confidence based on model and features
*/
calculatePredictionConfidence(model, features) {
if (model.trainingCount === 0)
return 0;
let confidence = model.accuracy;
// Adjust confidence based on feature completeness
const featureCompleteness = model.features.filter((f) => features.has(f)).length / model.features.length;
confidence *= featureCompleteness;
// Adjust confidence based on training data recency
const dataAge = Date.now() - model.lastTrained;
const ageDecay = Math.exp(-dataAge / (30 * 24 * 60 * 60 * 1000)); // 30 day half-life
confidence *= ageDecay;
return Math.max(0, Math.min(1, confidence));
}
/**
* Encode chatmode as numeric value for ML models
*/
encodeChatmode(subagent) {
const chatmodes = [
'Software Engineer', 'Security Engineer', 'DevOps Engineer',
'Database Architect', 'Software Architect', 'Test Engineer',
'Technical Writer', 'Design Reviewer', 'Memory Curator', 'Prompt Writer'
];
const index = chatmodes.indexOf(subagent);
return index >= 0 ? index / chatmodes.length : 0.5;
}
/**
* Get quality threshold for a specific chatmode
*/
getQualityThreshold(subagent) {
// Security Engineer has higher threshold (80%)
if (subagent === 'Security Engineer')
return 0.80;
// Default threshold for other chatmodes (75%)
return 0.75;
}
/**
* Generate fallback prediction when ML is unavailable
*/
fallbackPrediction(context, reason) {
const historicalTrends = this.analyticsManager.getQualityTrends(context.subagent, 7);
const baseScore = historicalTrends.scoreAverage || 0.75;
return {
predictedScore: baseScore,
confidence: 0.3,
predictionBasis: [reason, 'Using historical average'],
riskFactors: ['No ML prediction available'],
successFactors: []
};
}
/**
* Identify risk factors that might lead to poor quality
*/
async identifyRiskFactors(context, features) {
const risks = [];
// Long requirements detection with lower threshold
const reqLength = features.get('requirements_length') || 0;
if (reqLength > 5000) {
risks.push('Extremely long requirements may lead to incomplete analysis');
}
else if (reqLength > 200) {
risks.push('long requirements may require careful analysis');
}
// Complex requirements detection based on content
const contentComplexity = features.get('content_complexity') || 0;
if (contentComplexity > 0.7) {
risks.push('complex technical requirements detected');
}
// Complex criteria
const criteriaComplexity = features.get('criteria_complexity') || 0;
if (criteriaComplexity > 5) {
risks.push('Multiple quality criteria may be difficult to satisfy simultaneously');
}
// Poor historical performance
const historicalAvg = features.get('historical_avg_score') || 0;
if (historicalAvg < 0.7) {
risks.push(`${context.subagent} has below-average recent performance`);
}
// Declining trend
const trend = features.get('recent_trend') || 0;
if (trend < -0.1) {
risks.push('Quality scores have been declining recently');
}
return risks;
}
/**
* Identify success factors that might lead to high quality
*/
async identifySuccessFactors(context, features) {
const factors = [];
// Good historical performance
const historicalAvg = features.get('historical_avg_score') || 0;
if (historicalAvg > 0.85) {
factors.push(`${context.subagent} has excellent recent performance`);
}
// Improving trend
const trend = features.get('recent_trend') || 0;
if (trend > 0.1) {
factors.push('Quality scores have been improving recently');
}
// Reasonable complexity
const reqLength = features.get('requirements_length') || 0;
if (reqLength < 500 && reqLength > 100) {
factors.push('Requirements are well-scoped for analysis');
}
return factors;
}
/**
* Additional helper methods for ML operations
*/
calculateTrend(subagent) {
const trends = this.analyticsManager.getQualityTrends(subagent, 14);
const recentTrends = this.analyticsManager.getQualityTrends(subagent, 7);
return recentTrends.scoreAverage - trends.scoreAverage;
}
async analyzeIssueTypes(context) {
// Placeholder: analyze common issue patterns
return 0.5;
}
async getPreviousRefinementSuccess(subagent) {
const usagePatterns = this.analyticsManager.getUsagePatterns(subagent, 7);
const pattern = usagePatterns.find(p => p.subagent === subagent);
return pattern?.successRate || 0.5;
}
async analyzeThresholdOptimization() {
const suggestions = [];
try {
// Analyze threshold effectiveness per chatmode
const chatmodes = ['Security Engineer', 'Software Engineer', 'DevOps Engineer', 'Database Architect', 'Software Architect'];
for (const chatmode of chatmodes) {
const currentThreshold = this.getQualityThreshold(chatmode);
const trends = this.analyticsManager.getQualityTrends(chatmode, 30);
const usagePatterns = this.analyticsManager.getUsagePatterns(chatmode, 30);
// Get chatmode-specific training data
const chatmodeData = this.trainingData.filter(d => d.subagent === chatmode);
if (chatmodeData.length < 10)
continue; // Need sufficient data
// Calculate current performance metrics
const passRate = chatmodeData.filter(d => d.actualPassed).length / chatmodeData.length;
const avgScore = chatmodeData.reduce((sum, d) => sum + d.actualScore, 0) / chatmodeData.length;
const scoreVariance = this.calculateVariance(chatmodeData.map(d => d.actualScore));
// Analyze score distribution to find optimal threshold
const scores = chatmodeData.map(d => d.actualScore).sort((a, b) => a - b);
const optimalThreshold = this.calculateOptimalThreshold(scores, passRate);
// Check if significant improvement is possible
const thresholdDiff = Math.abs(optimalThreshold - currentThreshold);
if (thresholdDiff > 0.05) { // 5% minimum difference
const expectedPassRate = scores.filter(s => s >= optimalThreshold).length / scores.length;
const expectedImprovement = Math.abs(expectedPassRate - passRate);
if (expectedImprovement > 0.1) { // 10% improvement threshold
suggestions.push({
type: 'threshold',
component: `${chatmode} Quality Threshold`,
currentValue: currentThreshold,
suggestedValue: optimalThreshold,
expectedImprovement,
confidence: Math.min(0.9, chatmodeData.length / 50), // Higher confidence with more data
rationale: optimalThreshold > currentThreshold
? `Current threshold too low: ${(passRate * 100).toFixed(1)}% pass rate, ${scoreVariance.toFixed(3)} score variance. Raising threshold should improve quality consistency.`
: `Current threshold too high: ${(passRate * 100).toFixed(1)}% pass rate suggests over-filtering. Lowering threshold should maintain quality while reducing unnecessary refinements.`
});
}
}
}
logger.debug('Threshold optimization analysis completed', {
chatmodesAnalyzed: chatmodes.length,
suggestionsGenerated: suggestions.length,
totalTrainingData: this.trainingData.length
});
}
catch (error) {
logger.error('Threshold optimization analysis failed', {
error: error instanceof Error ? error.message : 'Unknown error'
});
}
return suggestions;
}
async analyzeRuleWeightOptimization() {
const suggestions = [];
try {
if (this.trainingData.length < 20) {
return suggestions; // Need sufficient data for correlation analysis
}
// Analyze correlation between rule performance and overall quality
const ruleNames = ['format_compliance', 'deliverables_completeness', 'response_quality', 'memory_operations_validity', 'task_completion_assessment'];
for (const ruleName of ruleNames) {
// Extract rule scores and overall scores from training data
const dataPoints = [];
// Simulate rule score extraction (in real implementation, this would come from stored rule results)
for (const data of this.trainingData) {
// Estimate individual rule contribution based on chatmode and features
const estimatedRuleScore = this.estimateRuleScore(ruleName, data);
dataPoints.push({
ruleScore: estimatedRuleScore,
overallScore: data.actualScore
});
}
// Calculate correlation coefficient
const correlation = this.calculateCorrelation(dataPoints.map(p => p.ruleScore), dataPoints.map(p => p.overallScore));
// Current weight (assuming equal weights initially)
const currentWeight = 0.2; // 20% for each of 5 rules
// Suggest weight adjustment based on correlation strength
if (Math.abs(correlation) > 0.7) { // Strong correlation
const suggestedWeight = Math.max(0.05, Math.min(0.4, currentWeight * (1 + correlation * 0.5)));
const weightDiff = Math.abs(suggestedWeight - currentWeight);
if (weightDiff > 0.05) { // 5% minimum change
suggestions.push({
type: 'rule_weight',
component: `Quality Rule: ${ruleName}`,
currentValue: currentWeight,
suggestedValue: suggestedWeight,
expectedImprovement: Math.abs(correlation) * 0.1, // Correlation strength indicates improvement potential
confidence: Math.min(0.9, this.trainingData.length / 100), // Higher confidence with more data
rationale: correlation > 0
? `Strong positive correlation (${correlation.toFixed(3)}) suggests this rule is highly predictive of quality. Increasing weight should improve overall assessment accuracy.`
: `Strong negative correlation (${correlation.toFixed(3)}) suggests this rule may be counter-productive. Decreasing weight should improve assessment accuracy.`
});
}
}
else if (Math.abs(correlation) < 0.3) { // Weak correlation
const suggestedWeight = Math.max(0.05, currentWeight * 0.7); // Reduce weight for weak predictors
suggestions.push({
type: 'rule_weight',
component: `Quality Rule: ${ruleName}`,
currentValue: currentWeight,
suggestedValue: suggestedWeight,
expectedImprovement: 0.05,
confidence: Math.min(0.8, this.trainingData.length / 80),
rationale: `Weak correlation (${correlation.toFixed(3)}) suggests this rule has limited predictive value. Reducing weight should improve overall assessment focus on more important factors.`
});
}
}
logger.debug('Rule weight optimization analysis completed', {
rulesAnalyzed: ruleNames.length,
suggestionsGenerated: suggestions.length,
trainingDataSize: this.trainingData.length
});
}
catch (error) {
logger.error('Rule weight optimization analysis failed', {
error: error instanceof Error ? error.message : 'Unknown error'
});
}
return suggestions;
}
async analyzeRefinementStrategyOptimization() {
const suggestions = [];
try {
if (this.trainingData.length < 15) {
return suggestions; // Need suffici