@boundless-oss/atlas
Version:
Atlas - MCP Server for comprehensive startup project management
600 lines • 22.6 kB
JavaScript
/**
* Performance Monitor
* Implements MCP Design Guide Section 5.3 principles for performance measurement and observability
*/
import { ensureDatabaseReady } from '../storage/sqlite-manager.js';
import { randomUUID } from 'crypto';
import { emitPerformanceUpdate } from '../web-dashboard/utils/database-events.js';
/**
* Comprehensive performance monitoring and observability system
*/
export class PerformanceMonitor {
static instance;
metrics = [];
qualityEvaluations = [];
activeTraces = new Map();
constructor() { }
static getInstance() {
if (!PerformanceMonitor.instance) {
PerformanceMonitor.instance = new PerformanceMonitor();
}
return PerformanceMonitor.instance;
}
/**
* Start tracking a tool execution
*/
startTrace(traceId, toolName, context) {
this.activeTraces.set(traceId, {
startTime: Date.now(),
toolName,
context
});
}
/**
* End tracking and record metrics
*/
endTrace(traceId, success, errorType, additionalMetrics) {
const trace = this.activeTraces.get(traceId);
if (!trace) {
console.warn(`No active trace found for ID: ${traceId}`);
return;
}
const executionTime = Date.now() - trace.startTime;
const metric = {
toolName: trace.toolName,
executionTime,
success,
errorType,
tokenCount: additionalMetrics?.tokenCount,
cost: additionalMetrics?.cost,
timestamp: Date.now(),
userId: trace.context.userId,
sessionId: trace.context.sessionId,
parameters: this.sanitizeParameters(trace.context.parameters)
};
this.metrics.push(metric);
this.activeTraces.delete(traceId);
// Keep only last 10,000 metrics to prevent memory issues
if (this.metrics.length > 10000) {
this.metrics = this.metrics.slice(-10000);
}
// Persist metric to database
this.persistMetric(metric).catch(err => {
console.error('[Performance] Failed to persist metric:', err);
});
// Emit real-time update
emitPerformanceUpdate(metric);
// Log performance issues
if (!success) {
console.error(`[Performance] Tool ${trace.toolName} failed after ${executionTime}ms`, {
errorType,
context: trace.context
});
}
else if (executionTime > 5000) { // Slow execution warning
console.warn(`[Performance] Tool ${trace.toolName} took ${executionTime}ms`, {
context: trace.context
});
}
}
/**
* Start monitoring (no-op for compatibility)
*/
startMonitoring() {
console.log('📊 Performance monitoring started');
}
/**
* Stop monitoring (no-op for compatibility)
*/
stopMonitoring() {
console.log('📊 Performance monitoring stopped');
}
/**
* Record tool execution for simple use cases
*/
recordToolExecution(toolName, executionTime, success = true) {
const metric = {
toolName,
executionTime,
success,
timestamp: Date.now(),
sessionId: 'default'
};
this.metrics.push(metric);
// Keep only last 10,000 metrics to prevent memory issues
if (this.metrics.length > 10000) {
this.metrics = this.metrics.slice(-10000);
}
// Persist metric to database
this.persistMetric(metric).catch(err => {
console.error('[Performance] Failed to persist metric:', err);
});
// Emit real-time update
emitPerformanceUpdate(metric);
}
/**
* Record LLM-as-a-Judge evaluation
*/
recordQualityEvaluation(evaluation) {
this.qualityEvaluations.push({
...evaluation,
timestamp: Date.now()
});
// Keep only last 1000 evaluations
if (this.qualityEvaluations.length > 1000) {
this.qualityEvaluations = this.qualityEvaluations.slice(-1000);
}
}
/**
* Generate comprehensive metrics for a specific tool
*/
getToolMetrics(toolName, timeRange) {
let toolMetrics = this.metrics.filter(m => m.toolName === toolName);
if (timeRange) {
toolMetrics = toolMetrics.filter(m => m.timestamp >= timeRange.start && m.timestamp <= timeRange.end);
}
if (toolMetrics.length === 0) {
return null;
}
const totalCalls = toolMetrics.length;
const successfulCalls = toolMetrics.filter(m => m.success).length;
const successRate = successfulCalls / totalCalls;
const executionTimes = toolMetrics.map(m => m.executionTime).sort((a, b) => a - b);
const averageExecutionTime = executionTimes.reduce((a, b) => a + b, 0) / executionTimes.length;
const medianExecutionTime = executionTimes[Math.floor(executionTimes.length / 2)];
const p95ExecutionTime = executionTimes[Math.floor(executionTimes.length * 0.95)];
const errorTypes = {};
toolMetrics
.filter(m => !m.success && m.errorType)
.forEach(m => {
errorTypes[m.errorType] = (errorTypes[m.errorType] || 0) + 1;
});
const totalTokens = toolMetrics
.filter(m => m.tokenCount)
.reduce((sum, m) => sum + (m.tokenCount || 0), 0);
const totalCost = toolMetrics
.filter(m => m.cost)
.reduce((sum, m) => sum + (m.cost || 0), 0);
const lastCalled = Math.max(...toolMetrics.map(m => m.timestamp));
return {
toolName,
totalCalls,
successRate,
averageExecutionTime,
medianExecutionTime,
p95ExecutionTime,
errorTypes,
totalTokens,
totalCost,
lastCalled
};
}
/**
* Generate system-wide performance metrics
*/
getSystemMetrics(timeRange) {
let metrics = this.metrics;
if (timeRange) {
metrics = metrics.filter(m => m.timestamp >= timeRange.start && m.timestamp <= timeRange.end);
}
const totalCalls = metrics.length;
const successfulCalls = metrics.filter(m => m.success).length;
const overallSuccessRate = totalCalls > 0 ? successfulCalls / totalCalls : 0;
const averageLatency = totalCalls > 0
? metrics.reduce((sum, m) => sum + m.executionTime, 0) / totalCalls
: 0;
const totalTokensUsed = metrics
.filter(m => m.tokenCount)
.reduce((sum, m) => sum + (m.tokenCount || 0), 0);
const totalCost = metrics
.filter(m => m.cost)
.reduce((sum, m) => sum + (m.cost || 0), 0);
// Group metrics by tool
const toolGroups = {};
metrics.forEach(m => {
if (!toolGroups[m.toolName]) {
toolGroups[m.toolName] = [];
}
toolGroups[m.toolName].push(m);
});
const totalTools = Object.keys(toolGroups).length;
// Calculate per-tool metrics
const toolStats = Object.entries(toolGroups).map(([toolName, toolMetrics]) => {
const calls = toolMetrics.length;
const successes = toolMetrics.filter(m => m.success).length;
const successRate = calls > 0 ? successes / calls : 0;
const avgTime = calls > 0
? toolMetrics.reduce((sum, m) => sum + m.executionTime, 0) / calls
: 0;
const errors = toolMetrics.filter(m => !m.success).length;
const errorRate = calls > 0 ? errors / calls : 0;
return {
tool: toolName,
calls,
successRate,
avgTime,
errorRate
};
});
// Sort and get top/bottom performers
const topPerformingTools = toolStats
.filter(t => t.calls >= 5) // Minimum calls for statistical significance
.sort((a, b) => b.successRate - a.successRate)
.slice(0, 5)
.map(t => ({ tool: t.tool, successRate: t.successRate, calls: t.calls }));
const slowestTools = toolStats
.filter(t => t.calls >= 5)
.sort((a, b) => b.avgTime - a.avgTime)
.slice(0, 5)
.map(t => ({ tool: t.tool, avgTime: t.avgTime, calls: t.calls }));
const mostErrorProneTools = toolStats
.filter(t => t.calls >= 5)
.sort((a, b) => b.errorRate - a.errorRate)
.slice(0, 5)
.map(t => ({ tool: t.tool, errorRate: t.errorRate, calls: t.calls }));
// Error breakdown
const errorBreakdown = {};
metrics
.filter(m => !m.success && m.errorType)
.forEach(m => {
errorBreakdown[m.errorType] = (errorBreakdown[m.errorType] || 0) + 1;
});
const timestamps = metrics.map(m => m.timestamp);
const timeRangeActual = {
start: timestamps.length > 0 ? Math.min(...timestamps) : Date.now(),
end: timestamps.length > 0 ? Math.max(...timestamps) : Date.now()
};
return {
totalTools,
totalCalls,
overallSuccessRate,
averageLatency,
totalTokensUsed,
totalCost,
topPerformingTools,
slowestTools,
mostErrorProneTools,
errorBreakdown,
timeRange: timeRangeActual
};
}
/**
* Get quality evaluation metrics
*/
getQualityMetrics(toolName) {
let evaluations = this.qualityEvaluations;
if (toolName) {
evaluations = evaluations.filter(e => e.toolName === toolName);
}
if (evaluations.length === 0) {
return {
averageQuality: 0,
averageAppropriateness: 0,
averageCompleteness: 0,
averageAccuracy: 0,
totalEvaluations: 0,
evaluationTrend: []
};
}
const averageQuality = evaluations.reduce((sum, e) => sum + e.outputQuality, 0) / evaluations.length;
const averageAppropriateness = evaluations.reduce((sum, e) => sum + e.appropriatenessScore, 0) / evaluations.length;
const averageCompleteness = evaluations.reduce((sum, e) => sum + e.completenessScore, 0) / evaluations.length;
const averageAccuracy = evaluations.reduce((sum, e) => sum + e.accuracyScore, 0) / evaluations.length;
const evaluationTrend = evaluations
.sort((a, b) => a.timestamp - b.timestamp)
.map(e => ({ timestamp: e.timestamp, quality: e.outputQuality }));
return {
averageQuality,
averageAppropriateness,
averageCompleteness,
averageAccuracy,
totalEvaluations: evaluations.length,
evaluationTrend
};
}
/**
* Generate performance alerts
*/
generateAlerts() {
const alerts = [];
const systemMetrics = this.getSystemMetrics();
// Success rate alerts
if (systemMetrics.overallSuccessRate < 0.8) {
alerts.push({
type: 'success_rate',
message: `Overall success rate is ${(systemMetrics.overallSuccessRate * 100).toFixed(1)}% (below 80% threshold)`,
severity: 'high'
});
}
// Latency alerts
if (systemMetrics.averageLatency > 3000) {
alerts.push({
type: 'latency',
message: `Average response time is ${systemMetrics.averageLatency.toFixed(0)}ms (above 3s threshold)`,
severity: 'medium'
});
}
// Error-prone tools
systemMetrics.mostErrorProneTools.forEach(tool => {
if (tool.errorRate > 0.2) {
alerts.push({
type: 'tool_errors',
message: `Tool ${tool.tool} has ${(tool.errorRate * 100).toFixed(1)}% error rate`,
severity: 'high'
});
}
});
// Slow tools
systemMetrics.slowestTools.forEach(tool => {
if (tool.avgTime > 5000) {
alerts.push({
type: 'slow_tools',
message: `Tool ${tool.tool} averages ${tool.avgTime.toFixed(0)}ms response time`,
severity: 'medium'
});
}
});
// Cost alerts
if (systemMetrics.totalCost > 100) { // Example threshold
alerts.push({
type: 'cost',
message: `Total cost is $${systemMetrics.totalCost.toFixed(2)} (review usage patterns)`,
severity: 'low'
});
}
return alerts;
}
/**
* Export metrics for external analysis
*/
exportMetrics(format = 'json') {
if (format === 'csv') {
const headers = [
'toolName', 'executionTime', 'success', 'errorType',
'tokenCount', 'cost', 'timestamp', 'userId', 'sessionId'
];
const csvData = this.metrics.map(m => [
m.toolName,
m.executionTime,
m.success,
m.errorType || '',
m.tokenCount || '',
m.cost || '',
new Date(m.timestamp).toISOString(),
m.userId || '',
m.sessionId
]);
return [headers, ...csvData].map(row => row.join(',')).join('\n');
}
return JSON.stringify({
systemMetrics: this.getSystemMetrics(),
qualityMetrics: this.getQualityMetrics(),
alerts: this.generateAlerts(),
exportTimestamp: new Date().toISOString()
}, null, 2);
}
/**
* Get module-specific metrics
*/
getModuleMetrics(moduleName) {
const moduleMetrics = {};
// Group metrics by module (based on tool name prefixes)
this.metrics.forEach(metric => {
const modulePrefix = metric.toolName.split('_')[0];
if (!moduleName || modulePrefix === moduleName) {
if (!moduleMetrics[modulePrefix]) {
moduleMetrics[modulePrefix] = {
calls: 0,
avgTime: 0,
successRate: 0,
totalTime: 0,
successCount: 0
};
}
const module = moduleMetrics[modulePrefix];
module.calls++;
module.totalTime += metric.executionTime;
if (metric.success)
module.successCount++;
}
});
// Calculate averages
Object.values(moduleMetrics).forEach((module) => {
module.avgTime = module.totalTime / module.calls;
module.successRate = (module.successCount / module.calls) * 100;
delete module.totalTime;
delete module.successCount;
});
return moduleMetrics;
}
/**
* Get all tool metrics as a summary
*/
getAllToolMetrics() {
const toolNames = [...new Set(this.metrics.map(m => m.toolName))];
const allMetrics = {};
toolNames.forEach(toolName => {
const metrics = this.getToolMetrics(toolName);
if (metrics) {
allMetrics[toolName] = metrics;
}
});
return allMetrics;
}
/**
* Analyze performance bottlenecks
*/
analyzeBottlenecks(options = {}) {
const threshold = options.threshold || 500;
const toolStats = this.getAllToolMetrics();
const slowTools = Object.entries(toolStats).filter(([, stats]) => stats.averageExecutionTime > threshold);
const highMemoryUsage = ['data_processing', 'large_file_analysis']; // Mock data
return {
slowestTools: slowTools.map(([tool]) => tool),
highMemoryUsage,
recommendations: slowTools.map(([tool]) => `Consider optimizing ${tool} - average execution time: ${toolStats[tool].averageExecutionTime}ms`)
};
}
/**
* Generate performance report
*/
generatePerformanceReport(options = {}) {
const systemMetrics = this.getSystemMetrics();
const qualityMetrics = this.getQualityMetrics();
const bottlenecks = this.analyzeBottlenecks();
const report = {
summary: `System performing ${systemMetrics.overallSuccessRate > 90 ? 'well' : 'with issues'}`,
metrics: {
system: systemMetrics,
quality: qualityMetrics
},
recommendations: options.includeRecommendations ? bottlenecks.recommendations : []
};
return options.format === 'csv' ? this.convertReportToCSV(report) : report;
}
/**
* Clear metrics data
*/
clearMetrics(options) {
if (options?.before) {
const beforeTimestamp = new Date(options.before).getTime();
this.metrics = this.metrics.filter(m => m.timestamp >= beforeTimestamp);
this.qualityEvaluations = this.qualityEvaluations.filter(e => e.timestamp >= beforeTimestamp);
}
else {
this.metrics = [];
this.qualityEvaluations = [];
}
return { cleared: true, remaining: this.metrics.length };
}
convertReportToCSV(report) {
// Simple CSV conversion for demo
const lines = [
'Metric,Value',
`Total Tools,${report.metrics.system.totalTools}`,
`Total Calls,${report.metrics.system.totalCalls}`,
`Success Rate,${report.metrics.system.overallSuccessRate}%`,
`Average Latency,${report.metrics.system.averageLatency}ms`
];
return lines.join('\n');
}
sanitizeParameters(parameters) {
if (!parameters)
return undefined;
const sensitiveKeys = ['password', 'token', 'secret', 'key', 'auth'];
const sanitized = JSON.parse(JSON.stringify(parameters));
const sanitizeObject = (obj) => {
if (typeof obj !== 'object' || obj === null)
return;
Object.keys(obj).forEach(key => {
if (sensitiveKeys.some(sensitive => key.toLowerCase().includes(sensitive))) {
obj[key] = '[REDACTED]';
}
else if (typeof obj[key] === 'object') {
sanitizeObject(obj[key]);
}
});
};
sanitizeObject(sanitized);
return sanitized;
}
/**
* Persist a metric to the database
*/
async persistMetric(metric) {
try {
const db = await ensureDatabaseReady();
await db.run(`INSERT INTO performance_metrics_v2 (
id, tool_name, execution_time, success, error_type,
token_count, cost, user_id, session_id, request_id,
parameters, timestamp
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
randomUUID(),
metric.toolName,
metric.executionTime,
metric.success ? 1 : 0,
metric.errorType || null,
metric.tokenCount || null,
metric.cost || null,
metric.userId || null,
metric.sessionId,
randomUUID(), // Generate request_id
JSON.stringify(metric.parameters || {}),
Math.floor(metric.timestamp / 1000) // Convert to seconds
]);
}
catch (error) {
console.error('[Performance] Database error:', error);
throw error;
}
}
/**
* Load metrics from database for a time range
*/
async loadMetricsFromDB(timeRangeHours) {
try {
const db = await ensureDatabaseReady();
const cutoffTime = Math.floor((Date.now() - timeRangeHours * 60 * 60 * 1000) / 1000);
const result = await db.query(`SELECT
tool_name as toolName,
execution_time as executionTime,
success,
error_type as errorType,
token_count as tokenCount,
cost,
user_id as userId,
session_id as sessionId,
parameters,
timestamp * 1000 as timestamp
FROM performance_metrics_v2
WHERE timestamp > ?
ORDER BY timestamp DESC`, [cutoffTime]);
if (result.success && result.data) {
return result.data.map((row) => ({
...row,
success: row.success === 1,
parameters: row.parameters ? JSON.parse(row.parameters) : undefined
}));
}
return [];
}
catch (error) {
console.error('[Performance] Failed to load metrics from DB:', error);
return [];
}
}
}
/**
* Decorator for automatic performance monitoring
*/
export function monitorPerformance(target, propertyKey, descriptor) {
const originalMethod = descriptor.value;
descriptor.value = async function (...args) {
const monitor = PerformanceMonitor.getInstance();
const traceId = `${target.constructor.name}_${propertyKey}_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
const context = {
userId: args[0]?.userId,
sessionId: args[0]?.sessionId || 'unknown',
parameters: args[0]
};
monitor.startTrace(traceId, propertyKey, context);
try {
const result = await originalMethod.apply(this, args);
monitor.endTrace(traceId, true);
return result;
}
catch (error) {
const errorType = error instanceof Error ? error.constructor.name : 'UnknownError';
monitor.endTrace(traceId, false, errorType);
throw error;
}
};
return descriptor;
}
/**
* Create a unique trace ID for manual tracking
*/
export function createTraceId(toolName) {
return `${toolName}_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
}
//# sourceMappingURL=performance-monitor.js.map