UNPKG

ai-debug-local-mcp

Version:

๐ŸŽฏ ENHANCED AI GUIDANCE v4.1.2: Dramatically improved tool descriptions help AI users choose the right tools instead of 'close enough' options. Ultra-fast keyboard automation (10x speed), universal recording, multi-ecosystem debugging support, and compreh

800 lines โ€ข 32.7 kB
/** * V2 Advanced Stability Monitor - Revolutionary HTTP-First Architecture * * Comprehensive stability monitoring for V2 stateless architecture with: * - Circuit breaker integration for tool reliability * - HTTP request health monitoring * - Real-time resource tracking and alerting * - Automatic recovery and degradation handling * - Performance trend analysis and prediction * - Multi-session coordination stability */ import { EventEmitter } from 'events'; import * as os from 'os'; import * as process from 'process'; import { V2_STABILITY_DEFAULTS } from './v2-stability-types.js'; export class V2StabilityMonitor extends EventEmitter { config; sessionRegistry; memoryManager; circuitBreakers = new Map(); metrics; events = []; monitoringInterval; healthCheckInterval; performanceTrends = []; lastHealthReport; alertHistory = []; constructor(sessionRegistry, memoryManager, config = {}) { super(); this.config = { ...V2_STABILITY_DEFAULTS, ...config }; this.sessionRegistry = sessionRegistry; this.memoryManager = memoryManager; this.initializeMetrics(); this.initializeCircuitBreakers(); this.startMonitoring(); console.error('๐Ÿ›ก๏ธ V2 Stability Monitor initialized with advanced HTTP monitoring'); } /** * Initialize baseline metrics */ initializeMetrics() { this.metrics = { httpTransport: { requestsPerSecond: 0, averageResponseTime: 0, errorRate: 0, activeConnections: 0, totalRequests: 0, lastRequestTime: Date.now() }, systemResources: { cpuUsagePercent: 0, memoryUsageMB: 0, availableMemoryMB: os.totalmem() / 1024 / 1024, diskUsagePercent: 0, networkLatency: 0 }, sessionHealth: { activeSessions: 0, averageSessionDuration: 0, sessionCreationRate: 0, sessionFailureRate: 0, resourceUtilization: 0 }, toolReliability: { totalTools: 0, healthyTools: 0, degradedTools: 0, failedTools: 0, averageSuccessRate: 100 }, performanceIndicators: { throughput: 0, latency: 0, availability: 100, reliability: 100, scalability: 100 }, alertsSummary: { critical: 0, warning: 0, info: 0, lastAlert: 0 }, timestamp: Date.now() }; } /** * Initialize circuit breakers for critical system components */ initializeCircuitBreakers() { const criticalComponents = [ 'http_transport', 'session_registry', 'memory_manager', 'browser_instances', 'screenshot_service', 'audit_engine' ]; for (const component of criticalComponents) { this.circuitBreakers.set(component, { status: 'closed', failureCount: 0, successCount: 0, lastFailureTime: 0, lastSuccessTime: Date.now(), openedAt: 0, nextAttemptTime: 0, config: { failureThreshold: this.config.circuitBreaker.failureThreshold, recoveryTimeoutMs: this.config.circuitBreaker.recoveryTimeoutMs, halfOpenMaxAttempts: this.config.circuitBreaker.halfOpenMaxAttempts, successThreshold: this.config.circuitBreaker.successThreshold } }); } } /** * Start comprehensive monitoring */ startMonitoring() { // Main monitoring loop this.monitoringInterval = setInterval(async () => { try { await this.performMonitoringCycle(); } catch (error) { this.handleMonitoringError(error); } }, this.config.monitoringIntervalMs); // Health check loop this.healthCheckInterval = setInterval(async () => { try { await this.performHealthCheck(); } catch (error) { this.handleMonitoringError(error); } }, this.config.healthCheckIntervalMs); console.error(`๐Ÿ”„ V2 Stability monitoring started (${this.config.monitoringIntervalMs}ms cycles)`); } /** * Core monitoring cycle - HTTP-optimized */ async performMonitoringCycle() { const startTime = Date.now(); // Update system metrics await this.updateSystemMetrics(); // Update HTTP transport metrics await this.updateHttpTransportMetrics(); // Update session health metrics await this.updateSessionHealthMetrics(); // Update tool reliability metrics await this.updateToolReliabilityMetrics(); // Calculate performance indicators this.calculatePerformanceIndicators(); // Check for alerts await this.checkForAlerts(); // Record performance trend this.recordPerformanceTrend(Date.now() - startTime); // Emit monitoring event this.emitStabilityEvent('monitoring_cycle_complete', 'info', { cycleTime: Date.now() - startTime, metricsSnapshot: this.getMetricsSnapshot() }); } /** * Comprehensive health check with recovery recommendations */ async performHealthCheck() { const healthReport = { timestamp: Date.now(), overallHealth: 'healthy', systemHealth: await this.assessSystemHealth(), componentHealth: await this.assessComponentHealth(), recommendations: [], alerts: [], trends: this.analyzeTrends(), recoverySuggestions: [] }; // Determine overall health const componentScores = Object.values(healthReport.componentHealth).map(c => c.score); const averageScore = componentScores.reduce((sum, score) => sum + score, 0) / componentScores.length; if (averageScore >= 90) { healthReport.overallHealth = 'healthy'; } else if (averageScore >= 70) { healthReport.overallHealth = 'degraded'; } else if (averageScore >= 50) { healthReport.overallHealth = 'unhealthy'; } else { healthReport.overallHealth = 'critical'; } // Generate recommendations healthReport.recommendations = await this.generateRecommendations(healthReport); // Generate recovery suggestions for degraded components healthReport.recoverySuggestions = await this.generateRecoverySuggestions(healthReport); this.lastHealthReport = healthReport; this.emit('health_report', healthReport); // Trigger automatic recovery if configured if (this.config.autoRecovery.enabled && healthReport.overallHealth === 'critical') { await this.triggerAutoRecovery(healthReport); } } /** * Record component success for circuit breaker */ recordComponentSuccess(component, responseTime = 0) { const breaker = this.circuitBreakers.get(component); if (!breaker) return; breaker.successCount++; breaker.lastSuccessTime = Date.now(); // Handle half-open to closed transition if (breaker.status === 'half-open' && breaker.successCount >= breaker.config.successThreshold) { breaker.status = 'closed'; breaker.failureCount = 0; breaker.successCount = 0; breaker.openedAt = 0; breaker.nextAttemptTime = 0; this.emitStabilityEvent('circuit_closed', 'info', { component, reason: 'Success threshold reached in half-open state' }); } } /** * Record component failure for circuit breaker */ recordComponentFailure(component, error) { const breaker = this.circuitBreakers.get(component); if (!breaker) return; const now = Date.now(); breaker.failureCount++; breaker.lastFailureTime = now; // Check if we should open the circuit if (breaker.status === 'closed' && breaker.failureCount >= breaker.config.failureThreshold) { breaker.status = 'open'; breaker.openedAt = now; breaker.nextAttemptTime = now + breaker.config.recoveryTimeoutMs; breaker.successCount = 0; this.emitStabilityEvent('circuit_opened', 'warning', { component, failureCount: breaker.failureCount, error }); } else if (breaker.status === 'half-open') { breaker.status = 'open'; breaker.openedAt = now; breaker.nextAttemptTime = now + breaker.config.recoveryTimeoutMs; this.emitStabilityEvent('circuit_opened', 'warning', { component, reason: 'Failure in half-open state', error }); } } /** * Check if component can be executed (circuit is not open) */ canExecuteComponent(component) { const breaker = this.circuitBreakers.get(component); if (!breaker) return { allowed: true }; const now = Date.now(); switch (breaker.status) { case 'closed': return { allowed: true }; case 'open': if (now >= breaker.nextAttemptTime) { breaker.status = 'half-open'; breaker.successCount = 0; this.emitStabilityEvent('circuit_half_open', 'info', { component, reason: 'Recovery timeout elapsed' }); return { allowed: true }; } return { allowed: false, reason: `Circuit open until ${new Date(breaker.nextAttemptTime).toLocaleTimeString()}` }; case 'half-open': if (breaker.successCount < breaker.config.halfOpenMaxAttempts) { return { allowed: true }; } return { allowed: false, reason: 'Half-open state max attempts reached' }; default: return { allowed: false, reason: 'Unknown circuit state' }; } } /** * Get current stability metrics */ getStabilityMetrics() { return { ...this.metrics }; } /** * Get latest health report */ getHealthReport() { return this.lastHealthReport; } /** * Get system health overview */ getSystemHealthOverview() { const memStats = process.memoryUsage(); const loadAvg = os.loadavg(); return { cpuUsage: process.cpuUsage(), memoryUsage: memStats, systemLoad: { oneMinute: loadAvg[0], fiveMinute: loadAvg[1], fifteenMinute: loadAvg[2] }, uptime: process.uptime(), platform: os.platform(), nodeVersion: process.version, pid: process.pid }; } /** * Force emergency recovery */ async emergencyRecovery(reason = 'Manual trigger') { console.error(`๐Ÿšจ V2 Emergency recovery initiated: ${reason}`); this.emitStabilityEvent('emergency_recovery_started', 'critical', { reason }); try { // Reset all circuit breakers for (const [component, breaker] of this.circuitBreakers) { if (breaker.status !== 'closed') { breaker.status = 'closed'; breaker.failureCount = 0; breaker.successCount = 0; breaker.openedAt = 0; breaker.nextAttemptTime = 0; } } // Trigger memory cleanup await this.memoryManager.triggerEmergencyCleanup('emergency_recovery'); // Cleanup expired sessions await this.sessionRegistry.performCleanup(); // Force garbage collection if available if (global.gc) { global.gc(); } this.emitStabilityEvent('emergency_recovery_complete', 'info', { reason, timestamp: Date.now() }); console.error('โœ… V2 Emergency recovery completed successfully'); } catch (error) { this.emitStabilityEvent('emergency_recovery_failed', 'critical', { reason, error: error.message }); console.error('โŒ V2 Emergency recovery failed:', error); } } // Private helper methods async updateSystemMetrics() { const memUsage = process.memoryUsage(); const cpuUsage = process.cpuUsage(); this.metrics.systemResources = { cpuUsagePercent: this.calculateCpuUsage(cpuUsage), memoryUsageMB: memUsage.heapUsed / 1024 / 1024, availableMemoryMB: (os.totalmem() - memUsage.heapUsed) / 1024 / 1024, diskUsagePercent: await this.calculateDiskUsage(), networkLatency: await this.measureNetworkLatency() }; } async updateHttpTransportMetrics() { // In real implementation, would gather from HTTP server metrics // For now, simulate based on session registry data const resourceSummary = this.sessionRegistry.getResourceSummary(); this.metrics.httpTransport = { requestsPerSecond: resourceSummary.totalHttpRequests / 60, // Rough estimate averageResponseTime: 150, // Would be measured in real implementation errorRate: 0.02, // Would be tracked activeConnections: resourceSummary.activeSessions, totalRequests: resourceSummary.totalHttpRequests, lastRequestTime: Date.now() }; } async updateSessionHealthMetrics() { const resourceSummary = this.sessionRegistry.getResourceSummary(); this.metrics.sessionHealth = { activeSessions: resourceSummary.activeSessions, averageSessionDuration: 300000, // Would be calculated from session data sessionCreationRate: resourceSummary.activeSessions / 3600, // Sessions per hour sessionFailureRate: 0.05, // Would be tracked resourceUtilization: resourceSummary.quotaUtilization }; } async updateToolReliabilityMetrics() { // Count circuit breaker states const healthy = Array.from(this.circuitBreakers.values()).filter(b => b.status === 'closed').length; const degraded = Array.from(this.circuitBreakers.values()).filter(b => b.status === 'half-open').length; const failed = Array.from(this.circuitBreakers.values()).filter(b => b.status === 'open').length; const total = this.circuitBreakers.size; this.metrics.toolReliability = { totalTools: total, healthyTools: healthy, degradedTools: degraded, failedTools: failed, averageSuccessRate: total > 0 ? (healthy / total) * 100 : 100 }; } calculatePerformanceIndicators() { const { systemResources, httpTransport, sessionHealth, toolReliability } = this.metrics; // Calculate throughput (requests per second) const throughput = httpTransport.requestsPerSecond; // Calculate latency (response time) const latency = httpTransport.averageResponseTime; // Calculate availability - FIXED: Based on actual service health, not CPU usage // Availability measures whether the service is responding, not resource utilization // HTTP availability: Only very high error rates (>5%) significantly impact availability const httpAvailability = httpTransport.errorRate > 0.05 ? Math.max(0, 100 - (httpTransport.errorRate * 100)) : 100; // Service responsiveness: Service is available if responding reasonably fast const serviceAvailability = httpTransport.averageResponseTime < 30000 ? 100 : 0; // Service responding within 30s = available // Circuit breaker availability: Only count actual failures, not startup state const circuitBreakerAvailability = this.calculateRobustCircuitBreakerAvailability(); // Availability is based on actual service health - all factors must be healthy const availability = Math.min(httpAvailability, serviceAvailability, circuitBreakerAvailability); // Calculate reliability (based on tool health) const reliability = toolReliability.averageSuccessRate; // Calculate scalability (based on resource utilization pressure) // High resource usage affects scalability, not availability const memoryUtilization = (systemResources.memoryUsageMB / systemResources.availableMemoryMB) * 100; const resourcePressure = Math.max(systemResources.cpuUsagePercent, memoryUtilization); const scalability = Math.max(0, 100 - Math.max(0, resourcePressure - 80)); // Only pressure above 80% affects scalability this.metrics.performanceIndicators = { throughput, latency, availability, reliability, scalability }; } /** * Calculate availability based on circuit breaker states * This gives a true measure of service availability */ calculateCircuitBreakerAvailability() { const breakerStates = Array.from(this.circuitBreakers.values()); if (breakerStates.length === 0) return 100; const healthyCount = breakerStates.filter(b => b.status === 'closed').length; const degradedCount = breakerStates.filter(b => b.status === 'half-open').length; const failedCount = breakerStates.filter(b => b.status === 'open').length; // Weighted availability: healthy=100%, degraded=50%, failed=0% const weightedHealth = (healthyCount * 100 + degradedCount * 50 + failedCount * 0); const totalComponents = breakerStates.length; return totalComponents > 0 ? weightedHealth / totalComponents : 100; } /** * Robust circuit breaker availability calculation that considers startup state * Only counts actual failures, not components that simply haven't been tested yet */ calculateRobustCircuitBreakerAvailability() { const breakerStates = Array.from(this.circuitBreakers.values()); if (breakerStates.length === 0) return 100; // During startup/normal operation, only open circuits indicate actual unavailability const openCount = breakerStates.filter(b => b.status === 'open').length; const totalComponents = breakerStates.length; // If there are no open circuits, the service is available // Open circuits reduce availability proportionally const availabilityPercentage = ((totalComponents - openCount) / totalComponents) * 100; return Math.max(0, availabilityPercentage); } async checkForAlerts() { const alerts = []; const { systemResources, httpTransport, performanceIndicators } = this.metrics; // System resource alerts - only alert on sustained high usage if (systemResources.cpuUsagePercent > this.config.alerts.cpuThreshold) { alerts.push({ level: 'warning', message: `High CPU usage: ${systemResources.cpuUsagePercent.toFixed(1)}%` }); } if (systemResources.memoryUsageMB > this.config.alerts.memoryThresholdMB) { alerts.push({ level: 'warning', message: `High memory usage: ${systemResources.memoryUsageMB.toFixed(1)}MB` }); } // HTTP transport alerts - only for significant error rates if (httpTransport.errorRate > this.config.alerts.errorRateThreshold) { alerts.push({ level: 'critical', message: `High error rate: ${(httpTransport.errorRate * 100).toFixed(1)}%` }); } if (httpTransport.averageResponseTime > this.config.alerts.responseTimeThreshold) { alerts.push({ level: 'warning', message: `Slow response time: ${httpTransport.averageResponseTime}ms` }); } // Performance alerts - FIXED: Use realistic availability thresholds and add throttling // Only alert if availability drops below 85% (real service unavailability) if (performanceIndicators.availability < 85) { const alertKey = `availability_${Math.floor(performanceIndicators.availability / 5) * 5}`; // Group by 5% ranges if (this.shouldEmitAlert(alertKey)) { alerts.push({ level: 'critical', message: `Service availability degraded: ${performanceIndicators.availability.toFixed(1)}%` }); } } // Update alert summary this.metrics.alertsSummary = { critical: alerts.filter(a => a.level === 'critical').length, warning: alerts.filter(a => a.level === 'warning').length, info: alerts.filter(a => a.level === 'info').length, lastAlert: alerts.length > 0 ? Date.now() : this.metrics.alertsSummary.lastAlert }; // Emit alert events with throttling for (const alert of alerts) { this.emitAlert(alert.level, alert.message); } } /** * Alert throttling to prevent spam */ alertThrottleMap = new Map(); ALERT_THROTTLE_MS = 300000; // 5 minutes between same alerts shouldEmitAlert(alertKey) { const now = Date.now(); const lastAlert = this.alertThrottleMap.get(alertKey) || 0; if (now - lastAlert > this.ALERT_THROTTLE_MS) { this.alertThrottleMap.set(alertKey, now); return true; } return false; } recordPerformanceTrend(cycleTime) { const trend = { timestamp: Date.now(), metrics: { ...this.metrics }, cycleTime }; this.performanceTrends.push(trend); // Keep only recent trends (last hour) const cutoff = Date.now() - (60 * 60 * 1000); this.performanceTrends = this.performanceTrends.filter(t => t.timestamp > cutoff); } async assessSystemHealth() { return this.getSystemHealthOverview(); } async assessComponentHealth() { const components = {}; for (const [component, breaker] of this.circuitBreakers) { let score = 100; let status = 'healthy'; if (breaker.status === 'open') { score = 0; status = 'failed'; } else if (breaker.status === 'half-open') { score = 50; status = 'degraded'; } else if (breaker.failureCount > 0) { score = Math.max(50, 100 - (breaker.failureCount * 20)); status = score < 80 ? 'degraded' : 'healthy'; } components[component] = { status, score, details: { circuitState: breaker.status, failureCount: breaker.failureCount, successCount: breaker.successCount, lastFailureTime: breaker.lastFailureTime, lastSuccessTime: breaker.lastSuccessTime } }; } return components; } analyzeTrends() { if (this.performanceTrends.length < 5) { return ['Insufficient data for trend analysis']; } const trends = []; const recent = this.performanceTrends.slice(-10); // Analyze CPU usage trend const cpuTrend = this.calculateTrend(recent.map(t => t.metrics.systemResources.cpuUsagePercent)); if (cpuTrend > 5) { trends.push('๐Ÿ“ˆ CPU usage trending upward'); } else if (cpuTrend < -5) { trends.push('๐Ÿ“‰ CPU usage trending downward'); } // Analyze response time trend const responseTrend = this.calculateTrend(recent.map(t => t.metrics.httpTransport.averageResponseTime)); if (responseTrend > 10) { trends.push('๐Ÿ“ˆ Response times increasing'); } else if (responseTrend < -10) { trends.push('๐Ÿ“‰ Response times improving'); } // Analyze error rate trend const errorTrend = this.calculateTrend(recent.map(t => t.metrics.httpTransport.errorRate)); if (errorTrend > 0.01) { trends.push('โš ๏ธ Error rate increasing'); } return trends.length > 0 ? trends : ['๐Ÿ“Š All metrics stable']; } calculateTrend(values) { if (values.length < 2) return 0; const first = values[0]; const last = values[values.length - 1]; return last - first; } async generateRecommendations(healthReport) { const recommendations = []; const { systemResources, httpTransport, performanceIndicators } = this.metrics; if (systemResources.cpuUsagePercent > 80) { recommendations.push('๐Ÿ”ง Consider scaling up CPU resources or optimizing high-CPU operations'); } if (systemResources.memoryUsageMB > 400) { recommendations.push('๐Ÿงน Memory usage is high - trigger cleanup or consider memory optimization'); } if (httpTransport.errorRate > 0.05) { recommendations.push('๐Ÿšจ High error rate detected - investigate recent failures and implement fixes'); } if (performanceIndicators.availability < 95) { recommendations.push('โšก Low availability - check system health and implement recovery procedures'); } if (performanceIndicators.scalability < 70) { recommendations.push('๐Ÿ“ˆ Low scalability - consider resource optimization or horizontal scaling'); } return recommendations.length > 0 ? recommendations : ['โœ… System performing optimally']; } async generateRecoverySuggestions(healthReport) { const suggestions = []; // Check for failed circuit breakers for (const [component, health] of Object.entries(healthReport.componentHealth)) { if (health.status === 'failed') { suggestions.push({ type: 'circuit_reset', component, description: `Reset circuit breaker for ${component}`, priority: 'high', estimatedImpact: 'immediate', autoExecutable: true }); } } // Check for memory pressure if (this.metrics.systemResources.memoryUsageMB > 400) { suggestions.push({ type: 'memory_cleanup', component: 'memory_manager', description: 'Trigger aggressive memory cleanup', priority: 'medium', estimatedImpact: 'short-term', autoExecutable: true }); } // Check for session cleanup const sessionSummary = this.sessionRegistry.getResourceSummary(); if (sessionSummary.idleSessions > 5) { suggestions.push({ type: 'session_cleanup', component: 'session_registry', description: 'Clean up idle sessions to free resources', priority: 'medium', estimatedImpact: 'immediate', autoExecutable: true }); } return suggestions; } async triggerAutoRecovery(healthReport) { console.error('๐Ÿค– V2 Auto-recovery triggered for critical health status'); for (const suggestion of healthReport.recoverySuggestions) { if (!suggestion.autoExecutable) continue; try { switch (suggestion.type) { case 'circuit_reset': this.resetCircuitBreaker(suggestion.component); break; case 'memory_cleanup': await this.memoryManager.triggerEmergencyCleanup('auto_recovery'); break; case 'session_cleanup': await this.sessionRegistry.performCleanup(); break; } this.emitStabilityEvent('auto_recovery_action', 'info', { action: suggestion.type, component: suggestion.component }); } catch (error) { this.emitStabilityEvent('auto_recovery_failed', 'warning', { action: suggestion.type, component: suggestion.component, error: error.message }); } } } resetCircuitBreaker(component) { const breaker = this.circuitBreakers.get(component); if (!breaker) return; breaker.status = 'closed'; breaker.failureCount = 0; breaker.successCount = 0; breaker.openedAt = 0; breaker.nextAttemptTime = 0; console.error(`๐Ÿ”„ V2 Circuit breaker reset for ${component}`); } calculateCpuUsage(cpuUsage) { // Simple estimation - in production would use more sophisticated calculation return Math.min(100, (cpuUsage.user + cpuUsage.system) / 10000); } async calculateDiskUsage() { // Simplified disk usage calculation return 45; // Would implement real disk usage check } async measureNetworkLatency() { // Simplified network latency measurement return 25; // Would implement real network latency check } getMetricsSnapshot() { return { timestamp: Date.now(), cpu: this.metrics.systemResources.cpuUsagePercent, memory: this.metrics.systemResources.memoryUsageMB, httpRequests: this.metrics.httpTransport.requestsPerSecond, availability: this.metrics.performanceIndicators.availability }; } emitStabilityEvent(type, level, data) { const event = { timestamp: Date.now(), type, level, data, source: 'stability_monitor' }; this.events.push(event); // Keep only recent events (last 1000) if (this.events.length > 1000) { this.events = this.events.slice(-500); } this.emit('stability_event', event); } emitAlert(level, message) { const alert = { timestamp: Date.now(), level, message }; this.alertHistory.push(alert); // Keep only recent alerts (last 100) if (this.alertHistory.length > 100) { this.alertHistory = this.alertHistory.slice(-50); } this.emit('alert', alert); this.emitStabilityEvent('alert_triggered', level, { message }); } handleMonitoringError(error) { console.error('โŒ V2 Stability monitoring error:', error); this.emitStabilityEvent('monitoring_error', 'warning', { error: error.message, stack: error.stack }); } /** * Graceful shutdown */ async shutdown() { console.error('๐Ÿ›‘ V2 Stability Monitor shutting down...'); if (this.monitoringInterval) { clearInterval(this.monitoringInterval); } if (this.healthCheckInterval) { clearInterval(this.healthCheckInterval); } this.emitStabilityEvent('shutdown', 'info', { timestamp: Date.now() }); console.error('โœ… V2 Stability Monitor shutdown complete'); } } //# sourceMappingURL=v2-stability-monitor.js.map