quantum-cli-core
Version:
Quantum CLI Core - Multi-LLM Collaboration System
458 lines • 17.2 kB
JavaScript
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { EventEmitter } from 'events';
import { ModelCharacteristicsService } from './model-characteristics.js';
export class RealTimeMetricsCollector extends EventEmitter {
performanceTracker;
activeRequests = new Map();
metricStreams = new Map();
healthChecks = new Map();
// Real-time tracking
requestCounters = new Map();
costAccumulators = new Map();
errorCounters = new Map();
// Sliding window metrics
windowSize = 60000; // 1 minute
slidingWindows = new Map();
// Monitoring intervals
dashboardInterval = null;
healthCheckInterval = null;
cleanupInterval = null;
constructor(performanceTracker) {
super();
this.performanceTracker = performanceTracker;
this.setupEventListeners();
this.startRealTimeMonitoring();
}
/**
* Start tracking a request
*/
startRequest(requestId, modelId, query) {
this.activeRequests.set(requestId, {
startTime: new Date(),
modelId,
query,
});
// Increment request counter
const currentCount = this.requestCounters.get(modelId) || 0;
this.requestCounters.set(modelId, currentCount + 1);
// Emit real-time event
this.emit('metric_event', {
type: 'request_start',
timestamp: new Date(),
modelId,
data: { requestId, query },
});
}
/**
* Complete a request and record metrics
*/
completeRequest(requestId, result) {
const request = this.activeRequests.get(requestId);
if (!request) {
console.warn(`Request ${requestId} not found in active requests`);
return;
}
const endTime = new Date();
const latency = endTime.getTime() - request.startTime.getTime();
// Create performance metric
const metric = {
modelId: request.modelId,
timestamp: endTime,
latency,
tokens: {
input: result.inputTokens,
output: result.outputTokens,
},
cost: {
input: result.cost *
(result.inputTokens / (result.inputTokens + result.outputTokens)),
output: result.cost *
(result.outputTokens / (result.inputTokens + result.outputTokens)),
total: result.cost,
},
queryType: result.queryType,
success: true,
userRating: result.userRating,
};
// Record in performance tracker
this.performanceTracker.recordMetric(metric);
// Update sliding window
this.updateSlidingWindow(request.modelId, metric);
// Update cost accumulator
const currentCost = this.costAccumulators.get(request.modelId) || 0;
this.costAccumulators.set(request.modelId, currentCost + result.cost);
// Clean up
this.activeRequests.delete(requestId);
// Emit real-time event
this.emit('metric_event', {
type: 'request_complete',
timestamp: endTime,
modelId: request.modelId,
data: {
requestId,
latency,
cost: result.cost,
tokens: result.inputTokens + result.outputTokens,
},
});
}
/**
* Record a failed request
*/
failRequest(requestId, error) {
const request = this.activeRequests.get(requestId);
if (!request) {
console.warn(`Request ${requestId} not found in active requests`);
return;
}
const endTime = new Date();
const latency = endTime.getTime() - request.startTime.getTime();
// Create error metric
const metric = {
modelId: request.modelId,
timestamp: endTime,
latency,
tokens: { input: 0, output: 0 },
cost: { input: 0, output: 0, total: 0 },
success: false,
errorType: error.name || 'UnknownError',
};
// Record in performance tracker
this.performanceTracker.recordMetric(metric);
// Update error counter
const currentErrors = this.errorCounters.get(request.modelId) || 0;
this.errorCounters.set(request.modelId, currentErrors + 1);
// Clean up
this.activeRequests.delete(requestId);
// Emit real-time event
this.emit('metric_event', {
type: 'request_error',
timestamp: endTime,
modelId: request.modelId,
data: { requestId, error: error.message, latency },
});
}
/**
* Get current real-time dashboard data
*/
getLiveDashboardData() {
const now = new Date();
const models = ModelCharacteristicsService.getAllModels();
const modelData = models.map((model) => {
const slidingWindow = this.slidingWindows.get(model.id) || [];
const recentMetrics = slidingWindow.filter((m) => now.getTime() - m.timestamp.getTime() < 60000);
const successfulMetrics = recentMetrics.filter((m) => m.success);
const currentRps = recentMetrics.length / 60; // requests per minute / 60
const averageLatency = successfulMetrics.length > 0
? successfulMetrics.reduce((sum, m) => sum + m.latency, 0) /
successfulMetrics.length
: 0;
const successRate = recentMetrics.length > 0
? successfulMetrics.length / recentMetrics.length
: 1;
const costPerHour = (this.costAccumulators.get(model.id) || 0) * 60; // extrapolate from minute
const queueSize = Array.from(this.activeRequests.values()).filter((req) => req.modelId === model.id).length;
const healthCheck = this.healthChecks.get(model.id);
let status = 'healthy';
if (!healthCheck || !healthCheck.isHealthy) {
status = 'error';
}
else if (successRate < 0.95) {
status = 'warning';
}
else if (averageLatency > 5000) {
status = 'warning';
}
const lastError = recentMetrics
.filter((m) => !m.success)
.sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime())[0]?.errorType;
return {
modelId: model.id,
name: model.name,
status,
currentRps,
averageLatency,
successRate,
costPerHour,
queueSize,
lastError,
};
});
// Calculate system metrics
const allMetrics = Array.from(this.slidingWindows.values()).flat();
const recentSystemMetrics = allMetrics.filter((m) => now.getTime() - m.timestamp.getTime() < 60000);
const systemMetrics = {
totalRequests: recentSystemMetrics.length,
totalCost: recentSystemMetrics.reduce((sum, m) => sum + m.cost.total, 0),
totalErrors: recentSystemMetrics.filter((m) => !m.success).length,
averageResponseTime: recentSystemMetrics.length > 0
? recentSystemMetrics.reduce((sum, m) => sum + m.latency, 0) /
recentSystemMetrics.length
: 0,
};
// Generate alerts
const alerts = this.generateCurrentAlerts(modelData, systemMetrics);
return {
timestamp: now,
models: modelData,
systemMetrics,
alerts,
};
}
/**
* Get metrics for a specific model in real-time
*/
getModelMetrics(modelId, windowSizeMs = 300000) {
const windowStart = new Date(Date.now() - windowSizeMs);
return this.performanceTracker.getAggregatedMetrics(modelId, windowStart);
}
/**
* Start streaming metrics for a model
*/
startMetricStream(modelId, windowSize = 60000, updateInterval = 5000) {
const stream = {
modelId,
metrics: [],
windowSize,
updateInterval,
};
this.metricStreams.set(modelId, stream);
// Emit initial data
this.emitStreamUpdate(stream);
}
/**
* Stop streaming metrics for a model
*/
stopMetricStream(modelId) {
this.metricStreams.delete(modelId);
}
/**
* Perform health check on a model
*/
async performHealthCheck(modelId) {
const startTime = Date.now();
let isHealthy = true;
let error;
try {
// Simple health check - could be expanded to actual API call
const characteristics = ModelCharacteristicsService.getModelCharacteristics(modelId);
if (!characteristics) {
throw new Error('Model not found');
}
// Check recent error rate
const recentMetrics = this.slidingWindows.get(modelId) || [];
const last5Minutes = recentMetrics.filter((m) => Date.now() - m.timestamp.getTime() < 300000);
if (last5Minutes.length > 10) {
const errorRate = last5Minutes.filter((m) => !m.success).length / last5Minutes.length;
if (errorRate > 0.1) {
isHealthy = false;
error = `High error rate: ${(errorRate * 100).toFixed(1)}%`;
}
}
}
catch (err) {
isHealthy = false;
error = err instanceof Error ? err.message : 'Unknown error';
}
const latency = Date.now() - startTime;
const healthCheck = {
modelId,
isHealthy,
latency,
error,
lastChecked: new Date(),
};
this.healthChecks.set(modelId, healthCheck);
return healthCheck;
}
/**
* Clean up resources
*/
destroy() {
if (this.dashboardInterval) {
clearInterval(this.dashboardInterval);
}
if (this.healthCheckInterval) {
clearInterval(this.healthCheckInterval);
}
if (this.cleanupInterval) {
clearInterval(this.cleanupInterval);
}
this.removeAllListeners();
this.activeRequests.clear();
this.metricStreams.clear();
this.slidingWindows.clear();
}
setupEventListeners() {
// Listen to performance tracker events
this.performanceTracker.on('metric_recorded', (metric) => {
this.updateSlidingWindow(metric.modelId, metric);
});
this.performanceTracker.on('cost_alert', (alert) => {
this.emit('metric_event', {
type: 'cost_alert',
timestamp: new Date(),
modelId: alert.modelId,
data: alert,
});
});
this.performanceTracker.on('performance_update', (update) => {
this.emit('metric_event', {
type: 'performance_degradation',
timestamp: new Date(),
modelId: update.modelId,
data: update,
});
});
}
startRealTimeMonitoring() {
// Dashboard data updates every 5 seconds
this.dashboardInterval = setInterval(() => {
const dashboardData = this.getLiveDashboardData();
this.emit('dashboard_update', dashboardData);
// Update metric streams
this.metricStreams.forEach((stream) => {
this.emitStreamUpdate(stream);
});
}, 5000);
// Health checks every 30 seconds
this.healthCheckInterval = setInterval(async () => {
const models = ModelCharacteristicsService.getAllModels();
for (const model of models) {
try {
await this.performHealthCheck(model.id);
}
catch (error) {
console.warn(`Health check failed for ${model.id}:`, error);
}
}
}, 30000);
// Cleanup old data every 5 minutes
this.cleanupInterval = setInterval(() => {
this.cleanupOldData();
}, 300000);
}
updateSlidingWindow(modelId, metric) {
if (!this.slidingWindows.has(modelId)) {
this.slidingWindows.set(modelId, []);
}
const window = this.slidingWindows.get(modelId);
window.push(metric);
// Remove old metrics outside the window
const cutoff = Date.now() - this.windowSize;
const filtered = window.filter((m) => m.timestamp.getTime() > cutoff);
this.slidingWindows.set(modelId, filtered);
}
emitStreamUpdate(stream) {
const windowStart = new Date(Date.now() - stream.windowSize);
const metrics = this.performanceTracker.getAggregatedMetrics(stream.modelId, windowStart);
if (metrics) {
this.emit('stream_update', {
modelId: stream.modelId,
metrics,
timestamp: new Date(),
});
}
}
generateCurrentAlerts(modelData, systemMetrics) {
const alerts = [];
const now = new Date();
// Model-specific alerts
modelData.forEach((model) => {
if (model.status === 'error') {
alerts.push({
severity: 'high',
message: `${model.name} is experiencing errors: ${model.lastError || 'Unknown error'}`,
modelId: model.modelId,
timestamp: now,
});
}
else if (model.status === 'warning') {
if (model.successRate < 0.95) {
alerts.push({
severity: 'medium',
message: `${model.name} success rate is low: ${(model.successRate * 100).toFixed(1)}%`,
modelId: model.modelId,
timestamp: now,
});
}
if (model.averageLatency > 5000) {
alerts.push({
severity: 'medium',
message: `${model.name} is responding slowly: ${model.averageLatency.toFixed(0)}ms`,
modelId: model.modelId,
timestamp: now,
});
}
}
// Cost alerts
if (model.costPerHour > 10) {
alerts.push({
severity: 'low',
message: `${model.name} cost is high: $${model.costPerHour.toFixed(2)}/hour`,
modelId: model.modelId,
timestamp: now,
});
}
// Queue size alerts
if (model.queueSize > 10) {
alerts.push({
severity: 'medium',
message: `${model.name} has a large queue: ${model.queueSize} pending requests`,
modelId: model.modelId,
timestamp: now,
});
}
});
// System-wide alerts
if (systemMetrics.totalErrors > systemMetrics.totalRequests * 0.1) {
alerts.push({
severity: 'high',
message: `High system error rate: ${systemMetrics.totalErrors}/${systemMetrics.totalRequests} requests failed`,
timestamp: now,
});
}
if (systemMetrics.averageResponseTime > 10000) {
alerts.push({
severity: 'medium',
message: `System response time is high: ${systemMetrics.averageResponseTime.toFixed(0)}ms`,
timestamp: now,
});
}
return alerts;
}
cleanupOldData() {
const cutoff = Date.now() - 3600000; // 1 hour
// Clean up sliding windows
this.slidingWindows.forEach((window, modelId) => {
const filtered = window.filter((m) => m.timestamp.getTime() > cutoff);
this.slidingWindows.set(modelId, filtered);
});
// Reset hourly counters
this.requestCounters.clear();
this.costAccumulators.clear();
this.errorCounters.clear();
}
getModelIds() {
// private metrics 접근: 타입 단언 사용
const metrics = this.performanceTracker.metrics;
return Array.from(new Set(metrics.map((m) => m.modelId)));
}
getAggregatedMetricsForModel(modelId, windowMs = 300000) {
return this.performanceTracker.getAggregatedMetrics(modelId, new Date(Date.now() - windowMs));
}
getModelHealthMap() {
// private healthChecks 접근: 타입 단언 사용
const healthChecks = this.healthChecks;
const result = {};
for (const [modelId, health] of healthChecks.entries()) {
result[modelId] = health;
}
return result;
}
}
//# sourceMappingURL=real-time-metrics.js.map