@hivetechs/hive-ai
Version:
Real-time streaming AI consensus platform with HTTP+SSE MCP integration for Claude Code, VS Code, Cursor, and Windsurf - powered by OpenRouter's unified API
532 lines (528 loc) • 22.2 kB
JavaScript
/**
* Performance Monitor - Expert-level performance monitoring and analysis
*
* Provides comprehensive performance metrics collection, analysis, and optimization
* recommendations for the consensus pipeline and OpenRouter integrations.
*/
import { structuredLogger } from './structured-logger.js';
export class PerformanceMonitor {
constructor() {
this.metricsBuffer = [];
this.BUFFER_SIZE = 1000;
this.COLLECTION_INTERVAL = 30000; // 30 seconds
this.isMonitoring = false;
this.startCollection();
}
/**
* Start performance metrics collection
*/
startCollection() {
if (this.isMonitoring)
return;
this.isMonitoring = true;
structuredLogger.info('Performance monitoring started');
this.collectionTimer = setInterval(() => {
this.collectMetrics();
}, this.COLLECTION_INTERVAL);
}
/**
* Stop performance metrics collection
*/
stopCollection() {
if (!this.isMonitoring)
return;
this.isMonitoring = false;
if (this.collectionTimer) {
clearInterval(this.collectionTimer);
this.collectionTimer = undefined;
}
structuredLogger.info('Performance monitoring stopped');
}
/**
* Collect real-time performance metrics
*/
async collectMetrics() {
try {
const metrics = {
timestamp: new Date().toISOString(),
pipeline: await this.collectPipelineMetrics(),
openRouter: await this.collectOpenRouterMetrics(),
resources: await this.collectResourceMetrics(),
quality: await this.collectQualityMetrics(),
cost: await this.collectCostMetrics()
};
this.addMetrics(metrics);
}
catch (error) {
structuredLogger.error('Performance metrics collection failed', {}, error);
}
}
/**
* Add metrics to buffer
*/
addMetrics(metrics) {
this.metricsBuffer.push(metrics);
// Trim buffer to maintain size limit
if (this.metricsBuffer.length > this.BUFFER_SIZE) {
this.metricsBuffer = this.metricsBuffer.slice(-this.BUFFER_SIZE);
}
}
/**
* Record conversation performance
*/
async recordConversationPerformance(conversationId, stages, totalDuration, totalCost, qualityScore) {
try {
const metrics = {
timestamp: new Date().toISOString(),
conversationId,
pipeline: {
totalDuration,
stageBreakdown: stages,
overallEfficiency: this.calculateOverallEfficiency(stages),
bottleneckStage: this.identifyBottleneckStage(stages)
},
openRouter: await this.collectOpenRouterMetrics(),
resources: await this.collectResourceMetrics(),
quality: {
averageQualityScore: qualityScore || 0,
improvementRate: 0,
consensusEffectiveness: 0
},
cost: {
totalCost,
costPerRequest: totalCost,
budgetUtilization: 0,
costEfficiency: qualityScore ? (qualityScore / totalCost) : 0
}
};
this.addMetrics(metrics);
await this.saveMetricsToDatabase(metrics);
}
catch (error) {
structuredLogger.error('Failed to record conversation performance', { conversationId }, error);
}
}
/**
* Generate comprehensive performance report
*/
async generatePerformanceReport(timeframe = '24h') {
try {
const timeframeMs = this.parseTimeframe(timeframe);
const since = new Date(Date.now() - timeframeMs);
// Get metrics from both buffer and database
const bufferMetrics = this.metricsBuffer.filter(m => new Date(m.timestamp) >= since);
const dbMetrics = await this.getMetricsFromDatabase(timeframe);
// Combine and deduplicate metrics
const allMetrics = [...bufferMetrics, ...dbMetrics];
const relevantMetrics = this.deduplicateMetrics(allMetrics).filter(m => new Date(m.timestamp) >= since);
if (relevantMetrics.length === 0) {
throw new Error(`No performance data available for timeframe: ${timeframe}`);
}
const summary = this.calculateSummary(relevantMetrics);
const trends = this.analyzeTrends(relevantMetrics);
const bottlenecks = this.analyzeBottlenecks(relevantMetrics);
const recommendations = this.generateRecommendations(relevantMetrics, bottlenecks);
return {
timeframe,
summary,
trends,
bottlenecks,
recommendations
};
}
catch (error) {
structuredLogger.error('Failed to generate performance report', { timeframe }, error);
throw error;
}
}
/**
* Identify performance bottlenecks
*/
async identifyBottlenecks(timeframe = '1h') {
const report = await this.generatePerformanceReport(timeframe);
return report.bottlenecks.filter(b => b.impact === 'high' || b.impact === 'critical');
}
/**
* Get optimization recommendations
*/
async getOptimizationRecommendations(focus) {
const report = await this.generatePerformanceReport('24h');
if (focus) {
return report.recommendations.filter(r => r.category === focus);
}
return report.recommendations.sort((a, b) => {
const priorityOrder = { high: 3, medium: 2, low: 1 };
return priorityOrder[b.priority] - priorityOrder[a.priority];
});
}
/**
* Compare performance across time periods
*/
async comparePerformance(baselinePeriod, currentPeriod) {
const baseline = await this.generatePerformanceReport(baselinePeriod);
const current = await this.generatePerformanceReport(currentPeriod);
const performanceChange = ((current.summary.averageResponseTime - baseline.summary.averageResponseTime) / baseline.summary.averageResponseTime) * 100;
const costChange = ((current.summary.costEfficiency - baseline.summary.costEfficiency) / baseline.summary.costEfficiency) * 100;
const qualityChange = 0; // Would be calculated from quality metrics
const summary = this.generateComparisonSummary(performanceChange, costChange, qualityChange);
return {
baseline,
current,
comparison: {
performanceChange,
costChange,
qualityChange,
summary
}
};
}
/**
* Export metrics for external monitoring tools
*/
async exportMetrics(format = 'json') {
try {
switch (format) {
case 'json':
return JSON.stringify(this.metricsBuffer, null, 2);
case 'csv':
return this.formatMetricsAsCSV(this.metricsBuffer);
case 'prometheus':
return this.formatMetricsAsPrometheus(this.metricsBuffer);
default:
throw new Error(`Unsupported export format: ${format}`);
}
}
catch (error) {
structuredLogger.error('Failed to export metrics', { format }, error);
throw error;
}
}
/**
* Private helper methods
*/
async collectPipelineMetrics() {
// This would collect real pipeline metrics from recent conversations
return {
totalDuration: 0,
stageBreakdown: {
generator: { duration: 0, tokensPerSecond: 0, cost: 0, retries: 0, modelUsed: '', latency: 0, efficiency: 0 },
refiner: { duration: 0, tokensPerSecond: 0, cost: 0, retries: 0, modelUsed: '', latency: 0, efficiency: 0 },
validator: { duration: 0, tokensPerSecond: 0, cost: 0, retries: 0, modelUsed: '', latency: 0, efficiency: 0 },
curator: { duration: 0, tokensPerSecond: 0, cost: 0, retries: 0, modelUsed: '', latency: 0, efficiency: 0 }
},
overallEfficiency: 0,
bottleneckStage: 'none'
};
}
async collectOpenRouterMetrics() {
// Collect OpenRouter API performance metrics
const { globalHealthMonitor } = await import('./health-monitor.js');
const health = globalHealthMonitor.getSystemHealth();
return {
averageLatency: health.openrouter.averageLatency || 0,
successRate: health.openrouter.status === 'healthy' ? 0.99 : 0.8,
retryCount: 0,
circuitBreakerTrips: 0,
providerBreakdown: {}
};
}
async collectResourceMetrics() {
// Collect system resource metrics
const memUsage = process.memoryUsage();
return {
memoryUsage: memUsage.heapUsed,
cpuUsage: 0, // Would use a CPU monitoring library
activeConnections: 0,
databaseQueries: 0,
queryLatency: 0
};
}
async collectQualityMetrics() {
// Collect quality metrics from recent conversations
return {
averageQualityScore: 0,
improvementRate: 0,
consensusEffectiveness: 0
};
}
async collectCostMetrics() {
// Collect cost metrics from recent conversations
return {
totalCost: 0,
costPerRequest: 0,
budgetUtilization: 0,
costEfficiency: 0
};
}
calculateOverallEfficiency(stages) {
const stageValues = Object.values(stages);
if (stageValues.length === 0)
return 0;
const totalTokens = stageValues.reduce((sum, stage) => sum + (stage.tokensPerSecond * (stage.duration / 1000)), 0);
const totalTime = stageValues.reduce((sum, stage) => sum + stage.duration, 0);
return totalTime > 0 ? (totalTokens / (totalTime / 1000)) : 0;
}
identifyBottleneckStage(stages) {
let slowestStage = 'none';
let maxDuration = 0;
Object.entries(stages).forEach(([stageName, stage]) => {
if (stage.duration > maxDuration) {
maxDuration = stage.duration;
slowestStage = stageName;
}
});
return slowestStage;
}
calculateSummary(metrics) {
const totalRequests = metrics.length;
const averageResponseTime = metrics.reduce((sum, m) => sum + m.pipeline.totalDuration, 0) / totalRequests;
const overallEfficiency = metrics.reduce((sum, m) => sum + m.pipeline.overallEfficiency, 0) / totalRequests;
const costEfficiency = metrics.reduce((sum, m) => sum + m.cost.costEfficiency, 0) / totalRequests;
return {
totalRequests,
averageResponseTime,
overallEfficiency,
costEfficiency
};
}
analyzeTrends(metrics) {
// Simple trend analysis - would be more sophisticated in practice
const recentMetrics = metrics.slice(-10);
const olderMetrics = metrics.slice(0, 10);
const recentPerf = recentMetrics.reduce((sum, m) => sum + m.pipeline.totalDuration, 0) / recentMetrics.length;
const olderPerf = olderMetrics.reduce((sum, m) => sum + m.pipeline.totalDuration, 0) / olderMetrics.length;
return {
performanceTrend: recentPerf < olderPerf ? 'improving' : recentPerf > olderPerf ? 'degrading' : 'stable',
costTrend: 'stable',
qualityTrend: 'stable'
};
}
analyzeBottlenecks(metrics) {
const bottlenecks = [];
// Analyze stage bottlenecks
const stageDelays = {
generator: [],
refiner: [],
validator: [],
curator: []
};
metrics.forEach(metric => {
Object.entries(metric.pipeline.stageBreakdown).forEach(([stage, perf]) => {
if (perf.duration > 5000) { // More than 5 seconds
stageDelays[stage].push(perf.duration);
}
});
});
Object.entries(stageDelays).forEach(([stage, delays]) => {
if (delays.length > 0) {
const avgDelay = delays.reduce((sum, d) => sum + d, 0) / delays.length;
const frequency = delays.length / metrics.length;
bottlenecks.push({
component: `${stage}_stage`,
issue: `High latency in ${stage} stage`,
impact: avgDelay > 10000 ? 'critical' : avgDelay > 7000 ? 'high' : 'medium',
frequency,
avgDelayMs: avgDelay,
recommendation: `Optimize ${stage} stage performance or consider faster model`
});
}
});
return bottlenecks;
}
generateRecommendations(metrics, bottlenecks) {
const recommendations = [];
// Generate recommendations based on bottlenecks
bottlenecks.forEach(bottleneck => {
if (bottleneck.impact === 'high' || bottleneck.impact === 'critical') {
recommendations.push({
category: 'performance',
priority: bottleneck.impact === 'critical' ? 'high' : 'medium',
description: `Address ${bottleneck.component} performance issues`,
expectedImprovement: `${Math.round(bottleneck.avgDelayMs / 1000)}s reduction in response time`,
implementation: bottleneck.recommendation,
estimatedEffort: '2-4 hours'
});
}
});
// Add general recommendations
const avgResponseTime = metrics.reduce((sum, m) => sum + m.pipeline.totalDuration, 0) / metrics.length;
if (avgResponseTime > 15000) {
recommendations.push({
category: 'performance',
priority: 'high',
description: 'Overall response time is too high',
expectedImprovement: '30-50% reduction in response time',
implementation: 'Consider using faster models or parallel stage execution',
estimatedEffort: '1-2 days'
});
}
return recommendations;
}
generateComparisonSummary(perfChange, costChange, qualityChange) {
const changes = [];
if (Math.abs(perfChange) > 5) {
changes.push(`Performance ${perfChange > 0 ? 'degraded' : 'improved'} by ${Math.abs(perfChange).toFixed(1)}%`);
}
if (Math.abs(costChange) > 5) {
changes.push(`Cost efficiency ${costChange > 0 ? 'improved' : 'declined'} by ${Math.abs(costChange).toFixed(1)}%`);
}
if (changes.length === 0) {
return 'Performance metrics remained stable across both periods';
}
return changes.join('; ');
}
parseTimeframe(timeframe) {
const match = timeframe.match(/^(\d+)([hmdy])$/);
if (!match) {
throw new Error(`Invalid timeframe format: ${timeframe}`);
}
const value = parseInt(match[1], 10);
const unit = match[2];
const multipliers = {
h: 60 * 60 * 1000,
d: 24 * 60 * 60 * 1000,
m: 30 * 24 * 60 * 60 * 1000,
y: 365 * 24 * 60 * 60 * 1000
};
return value * multipliers[unit];
}
formatMetricsAsCSV(metrics) {
const headers = ['timestamp', 'totalDuration', 'overallEfficiency', 'totalCost', 'averageLatency'];
const rows = metrics.map(m => [
m.timestamp,
m.pipeline.totalDuration.toString(),
m.pipeline.overallEfficiency.toString(),
m.cost.totalCost.toString(),
m.openRouter.averageLatency.toString()
]);
return [headers.join(','), ...rows.map(row => row.join(','))].join('\n');
}
formatMetricsAsPrometheus(metrics) {
const latest = metrics[metrics.length - 1];
if (!latest)
return '';
return `
# HELP hive_ai_response_time_ms Average response time in milliseconds
# TYPE hive_ai_response_time_ms gauge
hive_ai_response_time_ms ${latest.pipeline.totalDuration}
# HELP hive_ai_cost_per_request Cost per request in USD
# TYPE hive_ai_cost_per_request gauge
hive_ai_cost_per_request ${latest.cost.costPerRequest}
# HELP hive_ai_openrouter_latency_ms OpenRouter API latency in milliseconds
# TYPE hive_ai_openrouter_latency_ms gauge
hive_ai_openrouter_latency_ms ${latest.openRouter.averageLatency}
# HELP hive_ai_memory_usage_bytes Memory usage in bytes
# TYPE hive_ai_memory_usage_bytes gauge
hive_ai_memory_usage_bytes ${latest.resources.memoryUsage}
`.trim();
}
/**
* Get metrics from database
*/
async getMetricsFromDatabase(timeframe) {
try {
const { getDatabase } = await import('../storage/unified-database.js');
const db = await getDatabase();
const timeframeMs = this.parseTimeframe(timeframe);
const since = new Date(Date.now() - timeframeMs).toISOString();
const dbRows = await db.all(`
SELECT * FROM performance_metrics
WHERE timestamp >= ?
ORDER BY timestamp DESC
`, [since]);
return dbRows.map(row => {
try {
const baseMetrics = JSON.parse(row.metrics_data || '{}');
return {
timestamp: row.timestamp,
conversationId: row.conversation_id,
pipeline: {
totalDuration: row.total_duration,
stageBreakdown: baseMetrics.pipeline?.stageBreakdown || {},
overallEfficiency: row.overall_efficiency,
bottleneckStage: baseMetrics.pipeline?.bottleneckStage || 'none'
},
openRouter: {
averageLatency: row.openrouter_latency,
successRate: baseMetrics.openRouter?.successRate || 0.95,
retryCount: baseMetrics.openRouter?.retryCount || 0,
circuitBreakerTrips: baseMetrics.openRouter?.circuitBreakerTrips || 0,
providerBreakdown: baseMetrics.openRouter?.providerBreakdown || {}
},
resources: {
memoryUsage: row.memory_usage,
cpuUsage: baseMetrics.resources?.cpuUsage || 0,
activeConnections: baseMetrics.resources?.activeConnections || 0,
databaseQueries: baseMetrics.resources?.databaseQueries || 0,
queryLatency: baseMetrics.resources?.queryLatency || 0
},
quality: {
averageQualityScore: row.quality_score,
improvementRate: baseMetrics.quality?.improvementRate || 0,
consensusEffectiveness: baseMetrics.quality?.consensusEffectiveness || 0
},
cost: {
totalCost: row.total_cost,
costPerRequest: baseMetrics.cost?.costPerRequest || row.total_cost,
budgetUtilization: baseMetrics.cost?.budgetUtilization || 0,
costEfficiency: baseMetrics.cost?.costEfficiency || 0
}
};
}
catch (parseError) {
structuredLogger.warn('Failed to parse metrics data', { error: parseError });
return null;
}
}).filter(metrics => metrics !== null);
}
catch (error) {
structuredLogger.warn('Failed to fetch metrics from database', { error: error.message });
return [];
}
}
/**
* Deduplicate metrics based on timestamp and conversation ID
*/
deduplicateMetrics(metrics) {
const seen = new Set();
return metrics.filter(metric => {
const key = `${metric.timestamp}-${metric.conversationId || 'no-id'}`;
if (seen.has(key)) {
return false;
}
seen.add(key);
return true;
});
}
async saveMetricsToDatabase(metrics) {
try {
const { getDatabase } = await import('../storage/unified-database.js');
const db = await getDatabase();
await db.run(`
INSERT OR IGNORE INTO performance_metrics (
timestamp, conversation_id, total_duration, overall_efficiency,
openrouter_latency, memory_usage, total_cost, quality_score,
metrics_data
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
`, [
metrics.timestamp,
metrics.conversationId || null,
metrics.pipeline.totalDuration,
metrics.pipeline.overallEfficiency,
metrics.openRouter.averageLatency,
metrics.resources.memoryUsage,
metrics.cost.totalCost,
metrics.quality.averageQualityScore,
JSON.stringify(metrics)
]);
}
catch (error) {
structuredLogger.warn('Failed to save metrics to database', { error: error.message });
// Don't throw - metrics collection should continue even if DB save fails
}
}
}
/**
* Global performance monitor instance
*/
export const globalPerformanceMonitor = new PerformanceMonitor();