ai-debug-local-mcp
Version:
🎯 ENHANCED AI GUIDANCE v4.1.2: Dramatically improved tool descriptions help AI users choose the right tools instead of 'close enough' options. Ultra-fast keyboard automation (10x speed), universal recording, multi-ecosystem debugging support, and compreh
567 lines (563 loc) • 26.7 kB
JavaScript
/**
* Performance Validation Suite
*
* Comprehensive testing system to validate all performance optimizations
* implemented in the AI-Debug system, including:
* - Sub-agent delegation performance (<100ms target)
* - Intelligent routing algorithms
* - Server readiness optimization
* - Circuit breaker effectiveness
* - Framework detection accuracy and speed
* - Memory usage improvements
*/
import { UserFriendlyLogger } from './user-friendly-logger.js';
import { PerformanceOptimizedOrchestrator } from './performance-optimized-orchestrator.js';
import { IntelligentServerReadiness } from './intelligent-server-readiness.js';
import { CircuitBreakerManager } from './circuit-breaker-manager.js';
import { EnhancedFrameworkDetector } from './enhanced-framework-detector.js';
import { AIDebugRepairToolkit } from './ai-debug-repair-toolkit.js';
export class PerformanceValidationSuite {
logger;
orchestrator;
serverReadiness;
circuitBreaker;
frameworkDetector;
repairToolkit;
constructor() {
this.logger = new UserFriendlyLogger('PerformanceValidation');
// Initialize optimization systems for testing
this.orchestrator = new PerformanceOptimizedOrchestrator({
maxDelegationTimeMs: 100,
enableIntelligentRouting: true,
enableAdaptiveLearning: true,
fallbackToDirectExecution: true
});
this.serverReadiness = new IntelligentServerReadiness();
this.circuitBreaker = new CircuitBreakerManager();
this.frameworkDetector = new EnhancedFrameworkDetector();
this.repairToolkit = new AIDebugRepairToolkit();
}
/**
* Run comprehensive performance validation
*/
async runFullValidation(options = {}) {
const { iterations = 10, targetUrls = ['http://localhost:4000', 'http://localhost:3000'], skipLongRunningTests = false } = options;
this.logger.info('🚀 Starting comprehensive performance validation...');
const startTime = Date.now();
const results = [];
const initialMemory = this.getMemoryUsage();
let peakMemory = initialMemory;
// Track memory throughout testing
const memoryMonitor = setInterval(() => {
const currentMemory = this.getMemoryUsage();
if (currentMemory > peakMemory) {
peakMemory = currentMemory;
}
}, 1000);
try {
// Category 1: Sub-agent delegation performance tests
this.logger.info('📊 Testing sub-agent delegation performance...');
const delegationTests = await this.testSubAgentDelegation(iterations);
results.push(...delegationTests);
// Category 2: Intelligent routing tests
this.logger.info('🧠 Testing intelligent routing algorithms...');
const routingTests = await this.testIntelligentRouting(iterations);
results.push(...routingTests);
// Category 3: Server readiness optimization tests
this.logger.info('🔍 Testing server readiness optimization...');
const serverTests = await this.testServerReadiness(targetUrls, iterations);
results.push(...serverTests);
// Category 4: Framework detection tests
this.logger.info('🎯 Testing framework detection accuracy and speed...');
const frameworkTests = await this.testFrameworkDetection(targetUrls, iterations);
results.push(...frameworkTests);
// Category 5: Circuit breaker effectiveness tests
this.logger.info('🛡️ Testing circuit breaker effectiveness...');
const circuitTests = await this.testCircuitBreakerPerformance(iterations);
results.push(...circuitTests);
// Category 6: Repair system tests (may be long-running)
if (!skipLongRunningTests) {
this.logger.info('🔧 Testing repair system performance...');
const repairTests = await this.testRepairSystemPerformance(targetUrls.slice(0, 1), Math.min(iterations, 3));
results.push(...repairTests);
}
clearInterval(memoryMonitor);
// Generate comprehensive report
const report = this.generateValidationReport(results, startTime, initialMemory, peakMemory);
this.logger.success(`✅ Performance validation completed: ${report.passedTests}/${report.totalTests} tests passed (${report.overallScore}% score)`);
return report;
}
catch (error) {
clearInterval(memoryMonitor);
this.logger.error(`❌ Performance validation failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
throw error;
}
}
/**
* Test sub-agent delegation performance (<100ms target)
*/
async testSubAgentDelegation(iterations) {
const results = [];
// Test 1: Basic delegation speed
const delegationTimes = [];
let successCount = 0;
for (let i = 0; i < iterations; i++) {
const startTime = performance.now();
try {
const result = await this.orchestrator.executeOptimizedTask('Debug this slow loading page and identify performance bottlenecks', {
projectContext: { framework: 'react', language: 'javascript', complexity: 'moderate' },
performanceConstraints: { maxTimeMs: 100, tokenBudget: 5000, qualityThreshold: 0.8 },
priority: 'high'
});
const executionTime = performance.now() - startTime;
delegationTimes.push(executionTime);
if (result.success) {
successCount++;
}
}
catch (error) {
delegationTimes.push(performance.now() - startTime);
}
}
const avgTime = delegationTimes.reduce((sum, time) => sum + time, 0) / delegationTimes.length;
const minTime = Math.min(...delegationTimes);
const maxTime = Math.max(...delegationTimes);
results.push({
testName: 'Sub-agent delegation speed',
category: 'delegation',
passed: avgTime < 100 && successCount / iterations > 0.8,
executionTimeMs: avgTime,
targetTimeMs: 100,
memoryUsageMB: this.getMemoryUsage(),
details: {
iterations,
averageTimeMs: avgTime,
minTimeMs: minTime,
maxTimeMs: maxTime,
successRate: successCount / iterations
},
recommendations: avgTime >= 100 ? ['Consider reducing classification complexity', 'Optimize agent availability caching'] : []
});
// Test 2: Caching effectiveness
const cacheTestTimes = [];
const sameTask = 'Monitor real-time events for performance issues in React application';
// First execution (cache miss)
const firstStart = performance.now();
await this.orchestrator.executeOptimizedTask(sameTask, {
projectContext: { framework: 'react', language: 'javascript', complexity: 'simple' }
});
const firstTime = performance.now() - firstStart;
// Subsequent executions (should hit cache)
for (let i = 0; i < 5; i++) {
const start = performance.now();
await this.orchestrator.executeOptimizedTask(sameTask, {
projectContext: { framework: 'react', language: 'javascript', complexity: 'simple' }
});
cacheTestTimes.push(performance.now() - start);
}
const avgCacheTime = cacheTestTimes.reduce((sum, time) => sum + time, 0) / cacheTestTimes.length;
const improvementPercent = ((firstTime - avgCacheTime) / firstTime) * 100;
results.push({
testName: 'Delegation caching effectiveness',
category: 'delegation',
passed: avgCacheTime < firstTime && improvementPercent > 20,
executionTimeMs: avgCacheTime,
targetTimeMs: firstTime * 0.5, // Target 50% improvement
memoryUsageMB: this.getMemoryUsage(),
details: {
iterations: 5,
averageTimeMs: avgCacheTime,
minTimeMs: Math.min(...cacheTestTimes),
maxTimeMs: Math.max(...cacheTestTimes),
successRate: 1.0,
optimizationSavingsMs: firstTime - avgCacheTime
},
recommendations: improvementPercent < 20 ? ['Verify caching implementation', 'Check cache TTL settings'] : []
});
return results;
}
/**
* Test intelligent routing algorithm performance
*/
async testIntelligentRouting(iterations) {
const results = [];
// Test routing accuracy and speed for different task types
const taskTypes = [
{ task: 'Find accessibility issues in this form', expectedAgent: 'accessibility-audit-agent' },
{ task: 'Analyze slow loading performance', expectedAgent: 'performance-analysis-agent' },
{ task: 'Debug JavaScript errors in console', expectedAgent: 'error-investigation-agent' },
{ task: 'Validate test coverage and quality', expectedAgent: 'validation-testing-agent' }
];
let correctRoutings = 0;
const routingTimes = [];
for (const { task, expectedAgent } of taskTypes) {
for (let i = 0; i < iterations; i++) {
const startTime = performance.now();
try {
const result = await this.orchestrator.executeOptimizedTask(task, {
projectContext: { framework: 'react', language: 'javascript', complexity: 'moderate' },
performanceConstraints: { maxTimeMs: 100, tokenBudget: 3000, qualityThreshold: 0.7 }
});
const routingTime = performance.now() - startTime;
routingTimes.push(routingTime);
if (result.agentUsed === expectedAgent) {
correctRoutings++;
}
}
catch (error) {
routingTimes.push(performance.now() - startTime);
}
}
}
const avgRoutingTime = routingTimes.reduce((sum, time) => sum + time, 0) / routingTimes.length;
const accuracy = correctRoutings / (taskTypes.length * iterations);
results.push({
testName: 'Intelligent routing accuracy',
category: 'routing',
passed: accuracy > 0.85 && avgRoutingTime < 50,
executionTimeMs: avgRoutingTime,
targetTimeMs: 50,
memoryUsageMB: this.getMemoryUsage(),
details: {
iterations: taskTypes.length * iterations,
averageTimeMs: avgRoutingTime,
minTimeMs: Math.min(...routingTimes),
maxTimeMs: Math.max(...routingTimes),
successRate: 1.0,
accuracy
},
recommendations: accuracy < 0.85 ? ['Improve keyword classification', 'Update routing confidence thresholds'] : []
});
return results;
}
/**
* Test server readiness optimization
*/
async testServerReadiness(targetUrls, iterations) {
const results = [];
for (const url of targetUrls) {
const readinessTimes = [];
let autoStartCount = 0;
let successCount = 0;
for (let i = 0; i < iterations; i++) {
const startTime = performance.now();
try {
const result = await this.serverReadiness.ensureServerReadiness(url, {
autoStart: true,
maxWaitMs: 30000
});
const checkTime = performance.now() - startTime;
readinessTimes.push(checkTime);
if (result.isReady) {
successCount++;
}
if (result.autoStarted) {
autoStartCount++;
}
}
catch (error) {
readinessTimes.push(performance.now() - startTime);
}
}
const avgTime = readinessTimes.reduce((sum, time) => sum + time, 0) / readinessTimes.length;
const quickCheckThreshold = url.includes('4000') ? 2000 : 1000; // Phoenix may take longer
results.push({
testName: `Server readiness check (${url})`,
category: 'server-readiness',
passed: avgTime < quickCheckThreshold && successCount / iterations > 0.7,
executionTimeMs: avgTime,
targetTimeMs: quickCheckThreshold,
memoryUsageMB: this.getMemoryUsage(),
details: {
iterations,
averageTimeMs: avgTime,
minTimeMs: Math.min(...readinessTimes),
maxTimeMs: Math.max(...readinessTimes),
successRate: successCount / iterations,
optimizationSavingsMs: autoStartCount > 0 ? 5000 : 0 // Estimated savings from avoiding retry cycles
},
recommendations: avgTime >= quickCheckThreshold ? ['Check network connectivity', 'Verify server startup time'] : []
});
}
return results;
}
/**
* Test framework detection accuracy and speed
*/
async testFrameworkDetection(targetUrls, iterations) {
const results = [];
for (const url of targetUrls) {
const detectionTimes = [];
let correctDetections = 0;
const expectedFramework = url.includes('4000') ? 'phoenix-liveview' : 'react';
for (let i = 0; i < iterations; i++) {
const startTime = performance.now();
try {
const result = await this.frameworkDetector.detectFramework(url, null, process.cwd());
const detectionTime = performance.now() - startTime;
detectionTimes.push(detectionTime);
// Check if detection is correct or reasonable
if (result.framework === expectedFramework ||
(result.framework !== 'unknown' && result.confidence > 0.7)) {
correctDetections++;
}
}
catch (error) {
detectionTimes.push(performance.now() - startTime);
}
}
const avgTime = detectionTimes.reduce((sum, time) => sum + time, 0) / detectionTimes.length;
const accuracy = correctDetections / iterations;
results.push({
testName: `Framework detection (${url})`,
category: 'framework-detection',
passed: avgTime < 1000 && accuracy > 0.8,
executionTimeMs: avgTime,
targetTimeMs: 1000,
memoryUsageMB: this.getMemoryUsage(),
details: {
iterations,
averageTimeMs: avgTime,
minTimeMs: Math.min(...detectionTimes),
maxTimeMs: Math.max(...detectionTimes),
successRate: 1.0,
accuracy
},
recommendations: accuracy < 0.8 ? ['Improve file-based detection', 'Add more framework signatures'] : []
});
}
return results;
}
/**
* Test circuit breaker performance and effectiveness
*/
async testCircuitBreakerPerformance(iterations) {
const results = [];
// Test circuit breaker response time
const cbTimes = [];
let preventedExecutions = 0;
// Simulate some failures to trigger circuit breaker
for (let i = 0; i < 5; i++) {
this.circuitBreaker.recordFailure('test_tool', 'Simulated failure', 'Performance test');
}
// Test circuit breaker check performance
for (let i = 0; i < iterations; i++) {
const startTime = performance.now();
const canExecute = this.circuitBreaker.canExecute('test_tool');
const checkTime = performance.now() - startTime;
cbTimes.push(checkTime);
if (!canExecute.allowed) {
preventedExecutions++;
}
}
const avgCBTime = cbTimes.reduce((sum, time) => sum + time, 0) / cbTimes.length;
results.push({
testName: 'Circuit breaker check performance',
category: 'circuit-breaker',
passed: avgCBTime < 5 && preventedExecutions > 0,
executionTimeMs: avgCBTime,
targetTimeMs: 5,
memoryUsageMB: this.getMemoryUsage(),
details: {
iterations,
averageTimeMs: avgCBTime,
minTimeMs: Math.min(...cbTimes),
maxTimeMs: Math.max(...cbTimes),
successRate: preventedExecutions / iterations
},
recommendations: avgCBTime >= 5 ? ['Optimize circuit breaker state checking', 'Consider caching circuit states'] : []
});
// Reset circuit breaker for clean state
this.circuitBreaker.resetCircuit('test_tool', 'Performance test cleanup');
return results;
}
/**
* Test repair system performance
*/
async testRepairSystemPerformance(targetUrls, iterations) {
const results = [];
if (targetUrls.length === 0) {
return results;
}
const url = targetUrls[0];
const repairTimes = [];
let successfulRepairs = 0;
for (let i = 0; i < iterations; i++) {
const startTime = performance.now();
try {
// Test quick health check (should be fast)
const healthCheck = await this.repairToolkit.quickHealthCheck(url);
const repairTime = performance.now() - startTime;
repairTimes.push(repairTime);
if (healthCheck.overall === 'healthy' || healthCheck.overall === 'degraded') {
successfulRepairs++;
}
}
catch (error) {
repairTimes.push(performance.now() - startTime);
}
}
const avgRepairTime = repairTimes.reduce((sum, time) => sum + time, 0) / repairTimes.length;
results.push({
testName: 'Repair system health check speed',
category: 'repair',
passed: avgRepairTime < 3000 && successfulRepairs / iterations > 0.7,
executionTimeMs: avgRepairTime,
targetTimeMs: 3000,
memoryUsageMB: this.getMemoryUsage(),
details: {
iterations,
averageTimeMs: avgRepairTime,
minTimeMs: Math.min(...repairTimes),
maxTimeMs: Math.max(...repairTimes),
successRate: successfulRepairs / iterations
},
recommendations: avgRepairTime >= 3000 ? ['Optimize diagnostic checks', 'Cache health check results'] : []
});
return results;
}
/**
* Generate comprehensive validation report
*/
generateValidationReport(results, startTime, initialMemory, peakMemory) {
const totalTime = Date.now() - startTime;
const passedTests = results.filter(r => r.passed).length;
const overallScore = (passedTests / results.length) * 100;
// Calculate system metrics
const allExecutionTimes = results.map(r => r.executionTimeMs);
const averageResponseTime = allExecutionTimes.reduce((sum, time) => sum + time, 0) / allExecutionTimes.length;
const throughput = (results.length / totalTime) * 1000; // operations per second
// Categorize results
const categories = {
delegation: results.filter(r => r.category === 'delegation'),
routing: results.filter(r => r.category === 'routing'),
serverReadiness: results.filter(r => r.category === 'server-readiness'),
frameworkDetection: results.filter(r => r.category === 'framework-detection'),
circuitBreaker: results.filter(r => r.category === 'circuit-breaker'),
repair: results.filter(r => r.category === 'repair')
};
// Generate recommendations
const recommendations = this.generateSystemRecommendations(results, overallScore);
return {
timestamp: Date.now(),
overallScore,
totalTests: results.length,
passedTests,
failedTests: results.length - passedTests,
categories,
systemMetrics: {
totalMemoryUsageMB: this.getMemoryUsage(),
peakMemoryUsageMB: peakMemory,
averageResponseTimeMs: averageResponseTime,
throughputOperationsPerSecond: throughput
},
recommendations
};
}
/**
* Generate system-wide recommendations based on test results
*/
generateSystemRecommendations(results, overallScore) {
const recommendations = [];
const failedTests = results.filter(r => !r.passed);
if (overallScore < 70) {
recommendations.push('🚨 Overall performance below acceptable threshold - consider system optimization');
}
// Category-specific recommendations
const delegationFailures = failedTests.filter(r => r.category === 'delegation');
if (delegationFailures.length > 0) {
recommendations.push('⚡ Sub-agent delegation needs optimization - check caching and routing logic');
}
const routingFailures = failedTests.filter(r => r.category === 'routing');
if (routingFailures.length > 0) {
recommendations.push('🧠 Intelligent routing accuracy needs improvement - update classification algorithms');
}
const serverFailures = failedTests.filter(r => r.category === 'server-readiness');
if (serverFailures.length > 0) {
recommendations.push('🔍 Server readiness checks are slow - verify network and startup performance');
}
const frameworkFailures = failedTests.filter(r => r.category === 'framework-detection');
if (frameworkFailures.length > 0) {
recommendations.push('🎯 Framework detection needs improvement - add more detection methods');
}
// Memory recommendations
const peakMemory = Math.max(...results.map(r => r.memoryUsageMB));
if (peakMemory > 200) {
recommendations.push('💾 High memory usage detected - consider memory optimization strategies');
}
// Performance recommendations
const slowTests = results.filter(r => r.executionTimeMs > r.targetTimeMs * 1.5);
if (slowTests.length > results.length * 0.3) {
recommendations.push('🐌 Multiple performance targets missed - review system architecture');
}
if (recommendations.length === 0) {
recommendations.push('✅ All performance optimizations are working well - system is performing optimally');
}
return recommendations;
}
/**
* Get current memory usage in MB
*/
getMemoryUsage() {
const memoryUsage = process.memoryUsage();
return Math.round(memoryUsage.heapUsed / 1024 / 1024);
}
/**
* Generate detailed performance report
*/
generateDetailedReport(report) {
const { overallScore, totalTests, passedTests, failedTests, categories, systemMetrics } = report;
let output = `# 🚀 AI-Debug Performance Validation Report
## Overall Summary
- **Score:** ${overallScore.toFixed(1)}% (${passedTests}/${totalTests} tests passed)
- **Status:** ${overallScore >= 80 ? '🟢 Excellent' : overallScore >= 70 ? '🟡 Good' : '🔴 Needs Improvement'}
- **Total Execution Time:** ${((Date.now() - report.timestamp) / 1000).toFixed(1)}s
## System Metrics
- **Memory Usage:** ${systemMetrics.totalMemoryUsageMB}MB (Peak: ${systemMetrics.peakMemoryUsageMB}MB)
- **Average Response Time:** ${systemMetrics.averageResponseTimeMs.toFixed(1)}ms
- **Throughput:** ${systemMetrics.throughputOperationsPerSecond.toFixed(1)} ops/sec
## Test Results by Category
`;
// Report each category
for (const [categoryName, tests] of Object.entries(categories)) {
if (tests.length === 0)
continue;
const categoryPassed = tests.filter(t => t.passed).length;
const categoryScore = (categoryPassed / tests.length) * 100;
output += `### ${categoryName.charAt(0).toUpperCase() + categoryName.slice(1)} (${categoryScore.toFixed(1)}%)\n\n`;
for (const test of tests) {
const status = test.passed ? '✅' : '❌';
const target = test.targetTimeMs ? ` (target: ${test.targetTimeMs}ms)` : '';
output += `**${status} ${test.testName}**\n`;
output += `- Execution Time: ${test.executionTimeMs.toFixed(1)}ms${target}\n`;
output += `- Success Rate: ${(test.details.successRate * 100).toFixed(1)}%\n`;
if (test.details.accuracy !== undefined) {
output += `- Accuracy: ${(test.details.accuracy * 100).toFixed(1)}%\n`;
}
if (test.details.optimizationSavingsMs !== undefined && test.details.optimizationSavingsMs > 0) {
output += `- Optimization Savings: ${test.details.optimizationSavingsMs.toFixed(1)}ms\n`;
}
if (test.recommendations.length > 0) {
output += `- Recommendations: ${test.recommendations.join(', ')}\n`;
}
output += '\n';
}
}
// System recommendations
if (report.recommendations.length > 0) {
output += `## Recommendations\n\n`;
for (const rec of report.recommendations) {
output += `- ${rec}\n`;
}
}
return output;
}
/**
* Cleanup resources
*/
destroy() {
this.serverReadiness.cleanup();
this.circuitBreaker.destroy();
}
}
//# sourceMappingURL=performance-validation-suite.js.map