@hivetechs/hive-ai/dist/lib/tools/health-monitor.js

Real-time streaming AI consensus platform with HTTP+SSE MCP integration for Claude Code, VS Code, Cursor, and Windsurf - powered by OpenRouter's unified API

/**
 * Health Check and Monitoring System
 *
 * Proactive monitoring of OpenRouter services, model availability,
 * and system health with automatic degradation and recovery.
 */
import { globalErrorHandler } from './error-handling.js';
import { structuredLogger } from './structured-logger.js';
export class HealthMonitor {
    constructor(checkInterval = 300000) {
        this.checkInterval = checkInterval;
        this.healthResults = new Map();
        this.modelHealth = new Map();
        this.monitoringInterval = null;
        this.testModels = [
            'openai/gpt-4o-mini',
            'anthropic/claude-3-haiku',
            'google/gemini-pro'
        ];
        this.startMonitoring();
    }
    /**
     * Start continuous health monitoring
     */
    startMonitoring() {
        if (this.monitoringInterval) {
            clearInterval(this.monitoringInterval);
        }
        // Run initial checks
        this.runHealthChecks();
        // Schedule periodic checks
        this.monitoringInterval = setInterval(() => {
            this.runHealthChecks();
        }, this.checkInterval);
        console.log('🏥 Health monitoring started');
    }
    /**
     * Stop health monitoring
     */
    stopMonitoring() {
        if (this.monitoringInterval) {
            clearInterval(this.monitoringInterval);
            this.monitoringInterval = null;
        }
        console.log('🏥 Health monitoring stopped');
    }
    /**
     * Run all health checks
     */
    async runHealthChecks() {
        try {
            await Promise.allSettled([
                this.checkOpenRouterHealth(),
                this.checkDatabaseHealth(),
                this.checkMCPServerHealth(),
                this.checkModelHealth()
            ]);
        }
        catch (error) {
            console.error('❌ Error running health checks:', error);
        }
    }
    /**
     * Check OpenRouter API health
     */
    async checkOpenRouterHealth() {
        const startTime = Date.now();
        const serviceName = 'openrouter-api';
        let current = this.healthResults.get(serviceName);
        try {
            // Test OpenRouter API with a simple models list call
            const response = await fetch('https://openrouter.ai/api/v1/models', {
                headers: {
                    'Authorization': `Bearer ${process.env.OPENROUTER_API_KEY}`,
                    'HTTP-Referer': 'https://hivetechs.io',
                    'X-Title': 'hive-ai'
                },
                signal: AbortSignal.timeout(10000) // 10 second timeout
            });
            const responseTime = Date.now() - startTime;
            if (response.ok) {
                const data = await response.json();
                const healthResult = {
                    service: serviceName,
                    status: 'healthy',
                    responseTime,
                    lastChecked: new Date(),
                    consecutiveFailures: 0,
                    metadata: {
                        modelsCount: data.data?.length || 0,
                        statusCode: response.status
                    }
                };
                structuredLogger.healthCheck(serviceName, 'healthy', responseTime, {
                    modelsCount: data.data?.length || 0
                });
                this.updateHealthResult(serviceName, healthResult);
            }
            else {
                throw new Error(`OpenRouter API returned ${response.status}: ${response.statusText}`);
            }
        }
        catch (error) {
            const responseTime = Date.now() - startTime;
            const consecutiveFailures = (current?.consecutiveFailures || 0) + 1;
            this.updateHealthResult(serviceName, {
                service: serviceName,
                status: consecutiveFailures > 3 ? 'unhealthy' : 'degraded',
                responseTime,
                lastChecked: new Date(),
                consecutiveFailures,
                lastError: error.message,
                metadata: {
                    timeout: responseTime > 10000
                }
            });
        }
    }
    /**
     * Check database health
     */
    async checkDatabaseHealth() {
        const startTime = Date.now();
        const serviceName = 'database';
        let current = this.healthResults.get(serviceName);
        try {
            // Test database with a simple query
            const { getDatabase } = await import('../storage/unified-database.js');
            const database = await getDatabase();
            // Test basic database operations
            await database.get('SELECT 1 as test');
            const profileCount = await database.get('SELECT COUNT(*) as count FROM pipeline_profiles');
            const responseTime = Date.now() - startTime;
            this.updateHealthResult(serviceName, {
                service: serviceName,
                status: 'healthy',
                responseTime,
                lastChecked: new Date(),
                consecutiveFailures: 0,
                metadata: {
                    profileCount: profileCount.count
                }
            });
        }
        catch (error) {
            const responseTime = Date.now() - startTime;
            const consecutiveFailures = (current?.consecutiveFailures || 0) + 1;
            this.updateHealthResult(serviceName, {
                service: serviceName,
                status: consecutiveFailures > 2 ? 'unhealthy' : 'degraded',
                responseTime,
                lastChecked: new Date(),
                consecutiveFailures,
                lastError: error.message
            });
        }
    }
    /**
     * Check MCP server health
     */
    async checkMCPServerHealth() {
        const startTime = Date.now();
        const serviceName = 'mcp-server';
        let current = this.healthResults.get(serviceName);
        try {
            // Import port manager to get current MCP server configuration
            const { MCPPortManager } = await import('./mcp-port-manager.js');
            const portManager = new MCPPortManager();
            const currentPort = await portManager.getConfiguredPort() || 3000;
            // Test MCP server health endpoint
            const healthUrl = `http://localhost:${currentPort}/health`;
            const response = await fetch(healthUrl, {
                signal: AbortSignal.timeout(5000) // 5 second timeout
            });
            const responseTime = Date.now() - startTime;
            if (response.ok) {
                const healthData = await response.json();
                this.updateHealthResult(serviceName, {
                    service: serviceName,
                    status: 'healthy',
                    responseTime,
                    lastChecked: new Date(),
                    consecutiveFailures: 0,
                    metadata: {
                        port: currentPort,
                        transport: healthData.transport,
                        version: healthData.version,
                        sessions: healthData.sessions,
                        statusCode: response.status
                    }
                });
                structuredLogger.healthCheck(serviceName, 'healthy', responseTime, {
                    port: currentPort,
                    transport: healthData.transport
                });
            }
            else {
                throw new Error(`MCP server health check failed: ${response.status} ${response.statusText}`);
            }
        }
        catch (error) {
            const responseTime = Date.now() - startTime;
            const consecutiveFailures = (current?.consecutiveFailures || 0) + 1;
            // If the error is connection refused, the server is likely not running
            const isConnectionError = error.message.includes('ECONNREFUSED') ||
                error.message.includes('fetch failed');
            this.updateHealthResult(serviceName, {
                service: serviceName,
                status: isConnectionError && consecutiveFailures === 1 ? 'degraded' :
                    consecutiveFailures > 3 ? 'unhealthy' : 'degraded',
                responseTime,
                lastChecked: new Date(),
                consecutiveFailures,
                lastError: error.message,
                metadata: {
                    connectionError: isConnectionError,
                    timeout: responseTime > 5000
                }
            });
        }
    }
    /**
     * Check model health by testing key models
     */
    async checkModelHealth() {
        const testPromises = this.testModels.map(model => this.testModelHealth(model));
        await Promise.allSettled(testPromises);
    }
    /**
     * Test individual model health
     */
    async testModelHealth(modelId) {
        const [provider, model] = modelId.split('/');
        const key = `${provider}:${model}`;
        const startTime = Date.now();
        try {
            // Simple test prompt
            const response = await fetch('https://openrouter.ai/api/v1/chat/completions', {
                method: 'POST',
                headers: {
                    'Authorization': `Bearer ${process.env.OPENROUTER_API_KEY}`,
                    'Content-Type': 'application/json',
                    'HTTP-Referer': 'https://hivetechs.io',
                    'X-Title': 'hive-ai'
                },
                body: JSON.stringify({
                    model: modelId,
                    messages: [{ role: 'user', content: 'Test' }],
                    max_tokens: 5,
                    temperature: 0
                }),
                signal: AbortSignal.timeout(30000) // 30 second timeout
            });
            const responseTime = Date.now() - startTime;
            let current = this.modelHealth.get(key);
            if (response.ok) {
                // Update success metrics
                const successRate = current ?
                    Math.min(100, (current.successRate * 0.9) + 10) : // Weighted average favoring recent success
                    100;
                const avgResponseTime = current ?
                    (current.avgResponseTime * 0.7) + (responseTime * 0.3) : // Weighted average
                    responseTime;
                this.modelHealth.set(key, {
                    provider,
                    model,
                    status: 'available',
                    avgResponseTime,
                    successRate,
                    lastTested: new Date(),
                    errorCount: 0
                });
            }
            else {
                throw new Error(`Model test failed: ${response.status} ${response.statusText}`);
            }
        }
        catch (error) {
            const responseTime = Date.now() - startTime;
            let current = this.modelHealth.get(key);
            const errorCount = (current?.errorCount || 0) + 1;
            // Decrease success rate on failure
            const successRate = current ?
                Math.max(0, current.successRate - 20) :
                0;
            const avgResponseTime = current ?
                (current.avgResponseTime * 0.8) + (responseTime * 0.2) :
                responseTime;
            this.modelHealth.set(key, {
                provider,
                model,
                status: errorCount > 3 ? 'unavailable' : 'degraded',
                avgResponseTime,
                successRate,
                lastTested: new Date(),
                errorCount
            });
        }
    }
    /**
     * Update health result with tracking
     */
    updateHealthResult(serviceName, result) {
        this.healthResults.set(serviceName, result);
        // Log significant status changes
        const previous = this.healthResults.get(serviceName);
        if (!previous || previous.status !== result.status) {
            const emoji = result.status === 'healthy' ? '✅' : result.status === 'degraded' ? '⚠️' : '❌';
            console.log(`${emoji} Health status changed: ${serviceName} is now ${result.status}`);
        }
    }
    /**
     * Get current system health
     */
    getSystemHealth() {
        const openrouter = this.healthResults.get('openrouter-api');
        const database = this.healthResults.get('database');
        const mcpServer = this.healthResults.get('mcp-server');
        const models = Array.from(this.modelHealth.values());
        // Determine overall health
        let overall = 'healthy';
        if (!openrouter || openrouter.status === 'unhealthy' ||
            !database || database.status === 'unhealthy' ||
            !mcpServer || mcpServer.status === 'unhealthy') {
            overall = 'unhealthy';
        }
        else if (openrouter.status === 'degraded' ||
            database.status === 'degraded' ||
            mcpServer.status === 'degraded' ||
            models.some(m => m.status === 'unavailable')) {
            overall = 'degraded';
        }
        return {
            overall,
            openrouter: openrouter || {
                service: 'openrouter-api',
                status: 'unhealthy',
                responseTime: 0,
                lastChecked: new Date(),
                consecutiveFailures: 999,
                lastError: 'Never checked'
            },
            database: database || {
                service: 'database',
                status: 'unhealthy',
                responseTime: 0,
                lastChecked: new Date(),
                consecutiveFailures: 999,
                lastError: 'Never checked'
            },
            mcpServer: mcpServer || {
                service: 'mcp-server',
                status: 'unhealthy',
                responseTime: 0,
                lastChecked: new Date(),
                consecutiveFailures: 999,
                lastError: 'Never checked'
            },
            models,
            timestamp: new Date()
        };
    }
    /**
     * Get health status for a specific model
     */
    getModelHealth(provider, model) {
        return this.modelHealth.get(`${provider}:${model}`) || null;
    }
    /**
     * Get list of healthy models for fallback
     */
    getHealthyModels() {
        return Array.from(this.modelHealth.values())
            .filter(health => health.status === 'available' && health.successRate > 70)
            .sort((a, b) => b.successRate - a.successRate) // Sort by success rate
            .map(health => `${health.provider}/${health.model}`);
    }
    /**
     * Check if a specific service is healthy
     */
    isServiceHealthy(serviceName) {
        const result = this.healthResults.get(serviceName);
        return result?.status === 'healthy';
    }
    /**
     * Check if a specific model is available
     */
    isModelAvailable(provider, model) {
        const health = this.modelHealth.get(`${provider}:${model}`);
        return health?.status === 'available';
    }
    /**
     * Add custom test model
     */
    addTestModel(modelId) {
        if (!this.testModels.includes(modelId)) {
            this.testModels.push(modelId);
        }
    }
    /**
     * Remove test model
     */
    removeTestModel(modelId) {
        this.testModels = this.testModels.filter(id => id !== modelId);
    }
    /**
     * Get detailed health report
     */
    getDetailedHealthReport() {
        return {
            summary: this.getSystemHealth(),
            errorStats: globalErrorHandler.getErrorStats(),
            circuitBreakers: globalErrorHandler.getCircuitBreakerStatus(),
            uptime: process.uptime()
        };
    }
}
// Singleton instance for global use
export const globalHealthMonitor = new HealthMonitor();
/**
 * Express middleware for health check endpoint
 */
export function healthCheckMiddleware(req, res, next) {
    if (req.path === '/health') {
        const health = globalHealthMonitor.getSystemHealth();
        const statusCode = health.overall === 'healthy' ? 200 :
            health.overall === 'degraded' ? 200 : 503;
        res.status(statusCode).json(health);
        return;
    }
    if (req.path === '/health/detailed') {
        const report = globalHealthMonitor.getDetailedHealthReport();
        res.json(report);
        return;
    }
    next();
}
/**
 * CLI command for health status
 */
export async function displayHealthStatus() {
    const health = globalHealthMonitor.getSystemHealth();
    console.log('\n🏥 System Health Status\n');
    // Overall status
    const overallEmoji = health.overall === 'healthy' ? '✅' :
        health.overall === 'degraded' ? '⚠️' : '❌';
    console.log(`${overallEmoji} Overall: ${health.overall.toUpperCase()}`);
    // Service details
    console.log('\n📊 Services:');
    console.log(`  🔗 OpenRouter: ${health.openrouter.status} (${health.openrouter.responseTime}ms)`);
    console.log(`  💾 Database: ${health.database.status} (${health.database.responseTime}ms)`);
    console.log(`  🔌 MCP Server: ${health.mcpServer.status} (${health.mcpServer.responseTime}ms)${health.mcpServer.metadata?.port ? ` - Port ${health.mcpServer.metadata.port}` : ''}`);
    // Model health
    if (health.models.length > 0) {
        console.log('\n🤖 Models:');
        health.models.forEach(model => {
            const emoji = model.status === 'available' ? '✅' :
                model.status === 'degraded' ? '⚠️' : '❌';
            console.log(`  ${emoji} ${model.provider}/${model.model}: ${model.status} (${model.successRate}% success, ${model.avgResponseTime}ms avg)`);
        });
    }
    console.log(`\n⏰ Last updated: ${health.timestamp.toLocaleString()}\n`);
    // Show recent errors if any
    const errorStats = globalErrorHandler.getErrorStats(3600000); // Last hour
    if (errorStats.totalErrors > 0) {
        console.log(`⚠️  ${errorStats.totalErrors} errors in the last hour`);
        console.log('Recent error types:', Object.entries(errorStats.errorsByType)
            .map(([type, count]) => `${type}: ${count}`)
            .join(', '));
    }
}