codecrucible-synth
Version:
Production-Ready AI Development Platform with Multi-Voice Synthesis, Smithery MCP Integration, Enterprise Security, and Zero-Timeout Reliability
1,157 lines • 40.1 kB
JavaScript
/**
* Comprehensive Observability System for CodeCrucible Synth
* Production-ready monitoring, metrics collection, logging, and telemetry system
* with OpenTelemetry integration and performance analytics
*/
import { EventEmitter } from 'events';
import { Logger } from '../logger.js';
import { performance } from 'perf_hooks';
import { promises as fs } from 'fs';
// Enhanced: OpenTelemetry Integration - Fixed type declarations
let trace;
let metrics;
let logs;
let SpanStatusCode;
let SpanKind;
let context;
let openTelemetryAvailable;
try {
const otelApi = require('@opentelemetry/api');
trace = otelApi.trace;
metrics = otelApi.metrics;
logs = otelApi.logs;
SpanStatusCode = otelApi.SpanStatusCode;
SpanKind = otelApi.SpanKind;
context = otelApi.context;
openTelemetryAvailable = true;
}
catch (error) {
openTelemetryAvailable = false;
// Mock OpenTelemetry APIs when not available
trace = {
getTracer: () => ({
startSpan: () => ({ end: () => { }, setStatus: () => { }, setAttributes: () => { } }),
}),
};
SpanStatusCode = { OK: 1, ERROR: 2 };
SpanKind = { CLIENT: 3 };
metrics = {};
logs = {};
context = {};
}
// Main Observability System
export class ObservabilitySystem extends EventEmitter {
logger;
config;
metricsCollector;
tracingSystem;
healthMonitor;
alertManager;
performanceProfiler;
dataStorage;
isRunning = false;
systemStartTime = new Date();
constructor(config) {
super();
this.logger = new Logger('ObservabilitySystem');
this.config = config;
// Initialize components
this.metricsCollector = new MetricsCollector(config.metrics, this);
this.tracingSystem = new TracingSystem(config.tracing, this);
this.healthMonitor = new HealthMonitor(config.health, this);
this.alertManager = new AlertManager(config.alerting, this);
this.performanceProfiler = new PerformanceProfiler();
this.dataStorage = new ObservabilityStorage(config.storage);
}
/**
* Initialize and start the observability system
*/
async initialize() {
this.logger.info('Initializing Observability System...');
try {
// Initialize storage
await this.dataStorage.initialize();
// Initialize components
await this.metricsCollector.initialize();
await this.tracingSystem.initialize();
await this.healthMonitor.initialize();
await this.alertManager.initialize();
// Start monitoring
this.startSystemMonitoring();
this.isRunning = true;
this.logger.info('Observability System initialized successfully');
this.emit('system:initialized');
}
catch (error) {
this.logger.error('Failed to initialize observability system:', error);
throw error;
}
}
/**
* Record a metric
*/
recordMetric(name, value, tags, unit = 'count') {
const metric = {
name,
value,
timestamp: new Date(),
tags: tags || {},
unit,
type: 'gauge',
};
this.metricsCollector.record(metric);
}
/**
* Increment a counter
*/
incrementCounter(name, tags, value = 1) {
const metric = {
name,
value,
timestamp: new Date(),
tags: tags || {},
unit: 'count',
type: 'counter',
};
this.metricsCollector.record(metric);
}
/**
* Record a timer
*/
recordTimer(name, duration, tags) {
const metric = {
name,
value: duration,
timestamp: new Date(),
tags: tags || {},
unit: 'ms',
type: 'timer',
};
this.metricsCollector.record(metric);
}
/**
* Start a trace span
*/
startSpan(operationName, parentSpan) {
return this.tracingSystem.startSpan(operationName, parentSpan);
}
/**
* Finish a trace span
*/
finishSpan(span, tags) {
this.tracingSystem.finishSpan(span, tags);
}
/**
* Profile an operation
*/
async profileOperation(operationName, operation, metadata) {
return this.performanceProfiler.profile(operationName, operation, metadata);
}
/**
* Check system health
*/
async checkHealth() {
return this.healthMonitor.checkHealth();
}
/**
* Get metrics summary
*/
getMetricsSummary(timeRange) {
return this.metricsCollector.getSummary(timeRange);
}
/**
* Get performance profiles
*/
getPerformanceProfiles() {
return this.performanceProfiler.getProfiles();
}
/**
* Get active alerts
*/
getActiveAlerts() {
return this.alertManager.getActiveAlerts();
}
/**
* Create custom alert rule
*/
createAlertRule(rule) {
this.alertManager.addRule(rule);
}
/**
* Enhanced: OpenTelemetry integration - Trace model requests
*/
async traceModelRequest(operation, attributes, fn) {
if (!openTelemetryAvailable) {
// Fallback to built-in tracing
const span = this.startSpan(operation);
try {
const result = await fn();
this.finishSpan(span, { status: 'ok' });
return result;
}
catch (error) {
this.finishSpan(span, {
status: 'error',
error: error instanceof Error ? error.message : String(error),
});
throw error;
}
}
const tracer = trace.getTracer('codecrucible-synth', '4.0.7');
const span = tracer.startSpan(operation, {
kind: SpanKind.CLIENT,
attributes: attributes,
});
try {
const result = await fn();
span.setStatus({ code: SpanStatusCode.OK });
return result;
}
catch (error) {
span.setStatus({
code: SpanStatusCode.ERROR,
message: error instanceof Error ? error.message : String(error),
});
throw error;
}
finally {
span.end();
}
}
/**
* Enhanced: OpenTelemetry integration - Trace agent communication
*/
async traceAgentCommunication(attributes, fn) {
if (!openTelemetryAvailable) {
// Fallback to built-in tracing
const span = this.startSpan('agent_communication');
try {
const result = await fn();
this.finishSpan(span, { status: 'ok' });
return result;
}
catch (error) {
this.finishSpan(span, {
status: 'error',
error: error instanceof Error ? error.message : String(error),
});
throw error;
}
}
const tracer = trace.getTracer('codecrucible-synth', '4.0.7');
const span = tracer.startSpan('agent_communication', {
kind: SpanKind.CLIENT,
attributes: attributes,
});
try {
const result = await fn();
span.setStatus({ code: SpanStatusCode.OK });
return result;
}
catch (error) {
span.setStatus({
code: SpanStatusCode.ERROR,
message: error instanceof Error ? error.message : String(error),
});
throw error;
}
finally {
span.end();
}
}
/**
* Enhanced: Record tool execution metrics
*/
recordToolExecution(toolName, executionTime, success, errorType) {
// Record to built-in metrics system
this.recordMetric('codecrucible.tool.execution.duration', executionTime, {
tool: toolName,
success: success.toString(),
...(errorType && { error_type: errorType }),
}, 'milliseconds');
this.recordMetric('codecrucible.tool.execution.count', 1, {
tool: toolName,
success: success.toString(),
}, 'count');
}
/**
* Get system statistics
*/
getSystemStats() {
const uptime = Date.now() - this.systemStartTime.getTime();
return {
systemInfo: {
uptime,
version: '3.5.0',
nodeVersion: process.version,
platform: process.platform,
arch: process.arch,
},
metrics: this.metricsCollector.getStats(),
tracing: this.tracingSystem.getStats(),
health: this.healthMonitor.getStats(),
alerts: this.alertManager.getStats(),
performance: this.performanceProfiler.getStats(),
storage: this.dataStorage.getStats(),
};
}
/**
* Export observability data
*/
async exportData(format, timeRange) {
const data = {
metrics: this.metricsCollector.exportData(timeRange),
traces: this.tracingSystem.exportData(timeRange),
alerts: this.alertManager.exportData(timeRange),
};
switch (format) {
case 'json':
return JSON.stringify(data, null, 2);
case 'csv':
return this.convertToCSV(data);
case 'prometheus':
return this.convertToPrometheus(data);
default:
throw new Error(`Unsupported export format: ${format}`);
}
}
/**
* Shutdown the observability system
*/
async shutdown() {
this.logger.info('Shutting down observability system...');
this.isRunning = false;
// Shutdown components
await this.metricsCollector.shutdown();
await this.tracingSystem.shutdown();
await this.healthMonitor.shutdown();
await this.alertManager.shutdown();
await this.dataStorage.shutdown();
this.logger.info('Observability system shutdown completed');
}
/**
* Private Methods
*/
startSystemMonitoring() {
// Monitor system metrics every 30 seconds
setInterval(() => {
// TODO: Store interval ID and call clearInterval in cleanup
if (!this.isRunning)
return;
this.collectSystemMetrics();
}, 30000);
// Perform health checks every minute
setInterval(async () => {
// TODO: Store interval ID and call clearInterval in cleanup
if (!this.isRunning)
return;
try {
await this.healthMonitor.performHealthCheck();
}
catch (error) {
this.logger.error('Health check failed:', error);
}
}, 60000);
// Check alerts every 30 seconds
setInterval(() => {
// TODO: Store interval ID and call clearInterval in cleanup
if (!this.isRunning)
return;
this.alertManager.evaluateRules();
}, 30000);
}
collectSystemMetrics() {
const memUsage = process.memoryUsage();
const cpuUsage = process.cpuUsage();
// Memory metrics
this.recordMetric('system.memory.rss', memUsage.rss, {}, 'bytes');
this.recordMetric('system.memory.heap.used', memUsage.heapUsed, {}, 'bytes');
this.recordMetric('system.memory.heap.total', memUsage.heapTotal, {}, 'bytes');
this.recordMetric('system.memory.external', memUsage.external, {}, 'bytes');
// CPU metrics
this.recordMetric('system.cpu.user', cpuUsage.user / 1000, {}, 'ms');
this.recordMetric('system.cpu.system', cpuUsage.system / 1000, {}, 'ms');
// Event loop metrics
const eventLoopLag = this.measureEventLoopLag();
this.recordMetric('system.event_loop.lag', eventLoopLag, {}, 'ms');
// Uptime
const uptime = Date.now() - this.systemStartTime.getTime();
this.recordMetric('system.uptime', uptime, {}, 'ms');
}
measureEventLoopLag() {
const start = performance.now();
return new Promise(resolve => {
setImmediate(() => {
const lag = performance.now() - start;
resolve(lag);
});
});
}
convertToCSV(data) {
// Simple CSV conversion - in production would be more sophisticated
let csv = 'timestamp,type,name,value,tags\n';
for (const metric of data.metrics || []) {
const tags = Object.entries(metric.tags)
.map(([k, v]) => `${k}=${v}`)
.join(';');
csv += `${metric.timestamp},metric,${metric.name},${metric.value},"${tags}"\n`;
}
return csv;
}
convertToPrometheus(data) {
// Convert to Prometheus format
let prometheus = '';
for (const metric of data.metrics || []) {
const labels = Object.entries(metric.tags)
.map(([k, v]) => `${k}="${v}"`)
.join(',');
prometheus += `${metric.name.replace(/\./g, '_')}{${labels}} ${metric.value} ${metric.timestamp.getTime()}\n`;
}
return prometheus;
}
}
// Supporting Classes
class MetricsCollector {
config;
observabilitySystem;
metrics = [];
aggregatedMetrics = new Map();
exporters = [];
logger;
constructor(config, observabilitySystem) {
this.config = config;
this.observabilitySystem = observabilitySystem;
this.logger = new Logger('MetricsCollector');
this.exporters = config.exporters || [];
}
async initialize() {
this.logger.info('Initializing metrics collection...');
// Start periodic export
if (this.config.exportInterval > 0) {
setInterval(() => {
this.exportMetrics();
}, this.config.exportInterval);
}
}
record(metric) {
this.metrics.push(metric);
this.updateAggregatedMetrics(metric);
// Cleanup old metrics
this.cleanupOldMetrics();
}
getSummary(timeRange) {
let metricsToAnalyze = this.metrics;
if (timeRange) {
metricsToAnalyze = this.metrics.filter(m => m.timestamp >= timeRange.start && m.timestamp <= timeRange.end);
}
const summary = {
totalMetrics: metricsToAnalyze.length,
uniqueMetrics: new Set(metricsToAnalyze.map(m => m.name)).size,
timeRange: timeRange || {
start: new Date(Math.min(...metricsToAnalyze.map(m => m.timestamp.getTime()))),
end: new Date(Math.max(...metricsToAnalyze.map(m => m.timestamp.getTime()))),
},
topMetrics: this.getTopMetrics(metricsToAnalyze),
aggregations: this.getAggregations(metricsToAnalyze),
};
return summary;
}
exportData(timeRange) {
if (!timeRange)
return [...this.metrics];
return this.metrics.filter(m => m.timestamp >= timeRange.start && m.timestamp <= timeRange.end);
}
getStats() {
return {
totalCollected: this.metrics.length,
uniqueNames: new Set(this.metrics.map(m => m.name)).size,
aggregatedMetrics: this.aggregatedMetrics.size,
memoryUsage: this.estimateMemoryUsage(),
exporterStatus: this.exporters.map(e => ({ type: e.type, healthy: true })),
};
}
async shutdown() {
await this.exportMetrics();
this.logger.info('Metrics collector shutdown completed');
}
updateAggregatedMetrics(metric) {
const key = `${metric.name}:${JSON.stringify(metric.tags)}`;
if (!this.aggregatedMetrics.has(key)) {
this.aggregatedMetrics.set(key, {
name: metric.name,
tags: metric.tags,
count: 0,
sum: 0,
min: Number.MAX_VALUE,
max: Number.MIN_VALUE,
values: [],
});
}
const agg = this.aggregatedMetrics.get(key);
agg.count++;
agg.sum += metric.value;
agg.min = Math.min(agg.min, metric.value);
agg.max = Math.max(agg.max, metric.value);
agg.values.push(metric.value);
// Keep only recent values for percentile calculations
if (agg.values.length > 1000) {
agg.values = agg.values.slice(-500);
}
}
cleanupOldMetrics() {
if (this.metrics.length > 10000) {
const cutoff = new Date(Date.now() - this.config.retentionDays * 24 * 60 * 60 * 1000);
this.metrics = this.metrics.filter(m => m.timestamp > cutoff);
}
}
async exportMetrics() {
for (const exporter of this.exporters) {
try {
await this.exportToExporter(exporter);
}
catch (error) {
this.logger.error(`Failed to export to ${exporter.type}:`, error);
}
}
}
async exportToExporter(exporter) {
// Implementation would depend on exporter type
this.logger.debug(`Exporting metrics to ${exporter.type}`);
}
getTopMetrics(metrics) {
const metricCounts = new Map();
for (const metric of metrics) {
const existing = metricCounts.get(metric.name) || { count: 0, sum: 0 };
existing.count++;
existing.sum += metric.value;
metricCounts.set(metric.name, existing);
}
return Array.from(metricCounts.entries())
.map(([name, data]) => ({
name,
count: data.count,
avgValue: data.sum / data.count,
}))
.sort((a, b) => b.count - a.count)
.slice(0, 10);
}
getAggregations(metrics) {
const aggregations = {};
for (const [key, agg] of this.aggregatedMetrics.entries()) {
aggregations[key] = {
count: agg.count,
sum: agg.sum,
avg: agg.sum / agg.count,
min: agg.min,
max: agg.max,
p95: this.calculatePercentile(agg.values, 0.95),
p99: this.calculatePercentile(agg.values, 0.99),
};
}
return aggregations;
}
calculatePercentile(values, percentile) {
if (values.length === 0)
return 0;
const sorted = [...values].sort((a, b) => a - b);
const index = Math.ceil(sorted.length * percentile) - 1;
return sorted[Math.max(0, index)];
}
estimateMemoryUsage() {
return this.metrics.length * 100; // Rough estimate: 100 bytes per metric
}
}
class TracingSystem {
config;
observabilitySystem;
traces = new Map();
activeSpans = new Map();
logger;
constructor(config, observabilitySystem) {
this.config = config;
this.observabilitySystem = observabilitySystem;
this.logger = new Logger('TracingSystem');
}
async initialize() {
this.logger.info('Initializing tracing system...');
}
startSpan(operationName, parentSpan) {
const traceId = parentSpan?.traceId || this.generateTraceId();
const spanId = this.generateSpanId();
const span = {
traceId,
spanId,
parentSpanId: parentSpan?.spanId,
operationName,
startTime: new Date(),
tags: {},
logs: [],
status: 'ok',
};
this.activeSpans.set(spanId, span);
if (!this.traces.has(traceId)) {
this.traces.set(traceId, []);
}
this.traces.get(traceId).push(span);
return span;
}
finishSpan(span, tags) {
span.endTime = new Date();
span.duration = span.endTime.getTime() - span.startTime.getTime();
if (tags) {
span.tags = { ...span.tags, ...tags };
}
this.activeSpans.delete(span.spanId);
this.observabilitySystem.recordTimer(`span.duration.${span.operationName}`, span.duration, span.tags);
}
exportData(timeRange) {
const allSpans = [];
for (const spans of this.traces.values()) {
allSpans.push(...spans);
}
if (!timeRange)
return allSpans;
return allSpans.filter(s => s.startTime >= timeRange.start && (s.endTime || new Date()) <= timeRange.end);
}
getStats() {
const allSpans = this.exportData();
return {
totalTraces: this.traces.size,
totalSpans: allSpans.length,
activeSpans: this.activeSpans.size,
averageSpansPerTrace: allSpans.length / Math.max(1, this.traces.size),
averageDuration: this.calculateAverageDuration(allSpans),
};
}
async shutdown() {
this.logger.info('Tracing system shutdown completed');
}
generateTraceId() {
return Math.random().toString(36).substr(2, 16);
}
generateSpanId() {
return Math.random().toString(36).substr(2, 8);
}
calculateAverageDuration(spans) {
const completedSpans = spans.filter(s => s.duration !== undefined);
if (completedSpans.length === 0)
return 0;
const totalDuration = completedSpans.reduce((sum, s) => sum + (s.duration || 0), 0);
return totalDuration / completedSpans.length;
}
}
class HealthMonitor {
config;
observabilitySystem;
components = new Map();
logger;
constructor(config, observabilitySystem) {
this.config = config;
this.observabilitySystem = observabilitySystem;
this.logger = new Logger('HealthMonitor');
}
async initialize() {
this.logger.info('Initializing health monitoring...');
this.registerDefaultComponents();
}
async checkHealth() {
const componentHealths = [];
let totalScore = 0;
for (const [name, component] of this.components.entries()) {
try {
const health = await this.checkComponentHealth(name, component);
componentHealths.push(health);
// Calculate health score (healthy=1, degraded=0.5, critical=0, unknown=0.25)
const score = health.status === 'healthy'
? 1
: health.status === 'degraded'
? 0.5
: health.status === 'unknown'
? 0.25
: 0;
totalScore += score;
}
catch (error) {
this.logger.error(`Health check failed for ${name}:`, error);
componentHealths.push({
...component,
status: 'critical',
lastChecked: new Date(),
errorRate: 1.0,
});
}
}
const overallScore = componentHealths.length > 0 ? totalScore / componentHealths.length : 0;
const status = this.determineOverallStatus(componentHealths, overallScore);
return {
status,
components: componentHealths,
overallScore,
lastChecked: new Date(),
uptime: Date.now() - this.observabilitySystem.systemStartTime.getTime(),
version: '3.5.0',
};
}
async performHealthCheck() {
const health = await this.checkHealth();
this.observabilitySystem.recordMetric('system.health.score', health.overallScore);
this.observabilitySystem.recordMetric('system.health.components.total', health.components.length);
const healthyCount = health.components.filter(c => c.status === 'healthy').length;
this.observabilitySystem.recordMetric('system.health.components.healthy', healthyCount);
}
registerComponent(name, component) {
this.components.set(name, component);
}
getStats() {
return {
totalComponents: this.components.size,
healthyComponents: Array.from(this.components.values()).filter(c => c.status === 'healthy')
.length,
degradedComponents: Array.from(this.components.values()).filter(c => c.status === 'degraded')
.length,
criticalComponents: Array.from(this.components.values()).filter(c => c.status === 'critical')
.length,
lastHealthCheck: new Date(),
};
}
async shutdown() {
this.logger.info('Health monitor shutdown completed');
}
registerDefaultComponents() {
// Register core system components
this.registerComponent('memory', {
name: 'memory',
status: 'healthy',
metrics: {
cpu: 0,
memory: 0,
diskUsage: 0,
networkLatency: 0,
errorCount: 0,
requestCount: 0,
customMetrics: {},
},
dependencies: [],
lastChecked: new Date(),
errorRate: 0,
responseTime: 0,
});
this.registerComponent('event-loop', {
name: 'event-loop',
status: 'healthy',
metrics: {
cpu: 0,
memory: 0,
diskUsage: 0,
networkLatency: 0,
errorCount: 0,
requestCount: 0,
customMetrics: {},
},
dependencies: [],
lastChecked: new Date(),
errorRate: 0,
responseTime: 0,
});
}
async checkComponentHealth(name, component) {
switch (name) {
case 'memory':
return this.checkMemoryHealth(component);
case 'event-loop':
return this.checkEventLoopHealth(component);
default:
return this.checkGenericComponentHealth(component);
}
}
checkMemoryHealth(component) {
const memUsage = process.memoryUsage();
const heapUsedPercent = (memUsage.heapUsed / memUsage.heapTotal) * 100;
let status = 'healthy';
if (heapUsedPercent > 90) {
status = 'critical';
}
else if (heapUsedPercent > 75) {
status = 'degraded';
}
return {
...component,
status,
metrics: {
...component.metrics,
memory: heapUsedPercent,
},
lastChecked: new Date(),
};
}
checkEventLoopHealth(component) {
// Simple event loop lag check
const start = performance.now();
return new Promise(resolve => {
setImmediate(() => {
const lag = performance.now() - start;
let status = 'healthy';
if (lag > 100) {
status = 'critical';
}
else if (lag > 50) {
status = 'degraded';
}
resolve({
...component,
status,
responseTime: lag,
lastChecked: new Date(),
});
});
});
}
checkGenericComponentHealth(component) {
return {
...component,
lastChecked: new Date(),
};
}
determineOverallStatus(components, overallScore) {
const criticalComponents = components.filter(c => c.status === 'critical').length;
const degradedComponents = components.filter(c => c.status === 'degraded').length;
if (criticalComponents > 0)
return 'critical';
if (degradedComponents > 0 || overallScore < 0.8)
return 'degraded';
if (overallScore < 0.5)
return 'critical';
return 'healthy';
}
}
class AlertManager {
config;
observabilitySystem;
rules = new Map();
activeAlerts = new Map();
alertHistory = [];
logger;
constructor(config, observabilitySystem) {
this.config = config;
this.observabilitySystem = observabilitySystem;
this.logger = new Logger('AlertManager');
}
async initialize() {
this.logger.info('Initializing alert management...');
// Load default rules
if (this.config.rules) {
for (const rule of this.config.rules) {
this.addRule(rule);
}
}
}
addRule(rule) {
this.rules.set(rule.id, rule);
this.logger.debug(`Added alert rule: ${rule.name}`);
}
evaluateRules() {
for (const rule of this.rules.values()) {
if (!rule.enabled)
continue;
try {
this.evaluateRule(rule);
}
catch (error) {
this.logger.error(`Failed to evaluate rule ${rule.name}:`, error);
}
}
}
getActiveAlerts() {
return Array.from(this.activeAlerts.values());
}
acknowledgeAlert(alertId, acknowledgedBy) {
const alert = this.activeAlerts.get(alertId);
if (alert) {
alert.acknowledgedBy = acknowledgedBy;
alert.acknowledgedAt = new Date();
}
}
exportData(timeRange) {
let alerts = this.alertHistory;
if (timeRange) {
alerts = alerts.filter(a => a.triggeredAt >= timeRange.start && a.triggeredAt <= timeRange.end);
}
return alerts;
}
getStats() {
const recentAlerts = this.alertHistory.filter(a => a.triggeredAt > new Date(Date.now() - 24 * 60 * 60 * 1000));
return {
totalRules: this.rules.size,
activeAlerts: this.activeAlerts.size,
alertsLast24h: recentAlerts.length,
criticalAlerts: Array.from(this.activeAlerts.values()).filter(a => a.severity === 'critical')
.length,
resolvedAlertsLast24h: recentAlerts.filter(a => a.status === 'resolved').length,
};
}
async shutdown() {
this.logger.info('Alert manager shutdown completed');
}
evaluateRule(rule) {
// Simplified rule evaluation - in production would query actual metrics
const shouldTrigger = Math.random() < 0.01; // 1% chance for demo
if (shouldTrigger && !this.activeAlerts.has(rule.id)) {
this.triggerAlert(rule);
}
}
triggerAlert(rule) {
const alert = {
id: this.generateAlertId(),
ruleId: rule.id,
severity: rule.severity,
status: 'active',
triggeredAt: new Date(),
message: `Alert triggered: ${rule.name}`,
details: {
rule: rule.name,
description: rule.description,
},
};
this.activeAlerts.set(alert.id, alert);
this.alertHistory.push(alert);
// Execute alert actions
this.executeAlertActions(rule, alert);
this.observabilitySystem.emit('alert:triggered', alert);
this.logger.warn(`Alert triggered: ${rule.name} (${alert.id})`);
}
executeAlertActions(rule, alert) {
for (const action of rule.actions) {
if (!action.enabled)
continue;
try {
this.executeAction(action, alert);
}
catch (error) {
this.logger.error(`Failed to execute alert action ${action.type}:`, error);
}
}
}
executeAction(action, alert) {
switch (action.type) {
case 'log':
this.logger.error(`ALERT: ${alert.message}`, alert.details);
break;
case 'webhook':
// Would make HTTP request to webhook URL
this.logger.debug(`Would send webhook for alert ${alert.id}`);
break;
default:
this.logger.debug(`Action type ${action.type} not implemented`);
}
}
generateAlertId() {
return `alert_${Date.now()}_${Math.random().toString(36).substr(2, 8)}`;
}
}
class PerformanceProfiler {
profiles = new Map();
activeMeasurements = new Map();
async profile(operationName, operation, metadata) {
const startTime = performance.now();
const startMemory = process.memoryUsage();
let result;
let success = true;
try {
result = await operation();
}
catch (error) {
success = false;
throw error;
}
finally {
const endTime = performance.now();
const endMemory = process.memoryUsage();
const measurement = {
timestamp: new Date(),
duration: endTime - startTime,
memoryUsage: endMemory.heapUsed - startMemory.heapUsed,
cpuUsage: 0, // Would need more sophisticated CPU monitoring
success,
metadata: metadata || {},
};
this.recordMeasurement(operationName, measurement);
}
return result;
}
getProfiles() {
return Array.from(this.profiles.values());
}
getStats() {
const allProfiles = this.getProfiles();
const totalMeasurements = allProfiles.reduce((sum, p) => sum + p.measurements.length, 0);
return {
totalOperations: this.profiles.size,
totalMeasurements,
averageDuration: this.calculateOverallAverageDuration(allProfiles),
memoryEfficiency: this.calculateMemoryEfficiency(allProfiles),
};
}
recordMeasurement(operationName, measurement) {
if (!this.profiles.has(operationName)) {
this.profiles.set(operationName, {
operation: operationName,
measurements: [],
statistics: this.createEmptyStatistics(),
trends: [],
});
}
const profile = this.profiles.get(operationName);
profile.measurements.push(measurement);
// Keep only recent measurements
if (profile.measurements.length > 1000) {
profile.measurements = profile.measurements.slice(-500);
}
// Update statistics
profile.statistics = this.calculateStatistics(profile.measurements);
profile.trends = this.calculateTrends(profile.measurements);
}
calculateStatistics(measurements) {
if (measurements.length === 0)
return this.createEmptyStatistics();
const durations = measurements.map(m => m.duration);
const sortedDurations = [...durations].sort((a, b) => a - b);
const sum = durations.reduce((s, d) => s + d, 0);
const mean = sum / durations.length;
const median = sortedDurations[Math.floor(sortedDurations.length / 2)];
const variance = durations.reduce((v, d) => v + Math.pow(d - mean, 2), 0) / durations.length;
const stdDev = Math.sqrt(variance);
return {
count: measurements.length,
mean,
median,
p95: sortedDurations[Math.floor(sortedDurations.length * 0.95)],
p99: sortedDurations[Math.floor(sortedDurations.length * 0.99)],
min: Math.min(...durations),
max: Math.max(...durations),
stdDev,
};
}
calculateTrends(measurements) {
// Simplified trend calculation
if (measurements.length < 10)
return [];
const recentMeasurements = measurements.slice(-50);
const olderMeasurements = measurements.slice(-100, -50);
if (olderMeasurements.length === 0)
return [];
const recentAvg = recentMeasurements.reduce((sum, m) => sum + m.duration, 0) / recentMeasurements.length;
const olderAvg = olderMeasurements.reduce((sum, m) => sum + m.duration, 0) / olderMeasurements.length;
const changePercent = ((recentAvg - olderAvg) / olderAvg) * 100;
let direction = 'stable';
if (Math.abs(changePercent) > 5) {
direction = changePercent < 0 ? 'improving' : 'degrading';
}
return [
{
period: 'recent',
direction,
changePercent: Math.abs(changePercent),
significance: Math.min(1, Math.abs(changePercent) / 20),
},
];
}
createEmptyStatistics() {
return {
count: 0,
mean: 0,
median: 0,
p95: 0,
p99: 0,
min: 0,
max: 0,
stdDev: 0,
};
}
calculateOverallAverageDuration(profiles) {
const allDurations = profiles.flatMap(p => p.measurements.map(m => m.duration));
if (allDurations.length === 0)
return 0;
return allDurations.reduce((sum, d) => sum + d, 0) / allDurations.length;
}
calculateMemoryEfficiency(profiles) {
const allMeasurements = profiles.flatMap(p => p.measurements);
if (allMeasurements.length === 0)
return 1;
const avgMemoryUsage = allMeasurements.reduce((sum, m) => sum + m.memoryUsage, 0) / allMeasurements.length;
// Simple efficiency metric: lower memory usage = higher efficiency
return Math.max(0, Math.min(1, 1 - avgMemoryUsage / (100 * 1024 * 1024))); // Normalize to 100MB
}
}
class ObservabilityStorage {
config;
logger;
constructor(config) {
this.config = config;
this.logger = new Logger('ObservabilityStorage');
}
async initialize() {
this.logger.info('Initializing observability storage...');
// Ensure data directory exists
try {
await fs.mkdir(this.config.dataPath, { recursive: true });
}
catch (error) {
this.logger.error('Failed to create data directory:', error);
throw error;
}
}
getStats() {
return {
dataPath: this.config.dataPath,
totalSize: 0, // Would calculate actual size
compressionEnabled: this.config.compressionEnabled,
encryptionEnabled: this.config.encryptionEnabled,
};
}
async shutdown() {
this.logger.info('Observability storage shutdown completed');
}
}
/**
* Enhanced: Factory function for creating observability system with OpenTelemetry support
*/
export function getTelemetryProvider() {
// This would be the singleton instance - simplified for integration
return new ObservabilitySystem({
metrics: {
enabled: true,
retentionDays: 7,
exportInterval: 60000,
exporters: [{ type: 'prometheus', batchSize: 100, flushInterval: 5000 }],
},
tracing: {
enabled: true,
samplingRate: 1.0,
maxSpansPerTrace: 100,
exporters: [{ type: 'jaeger', batchSize: 100, flushInterval: 5000 }],
},
logging: {
level: 'info',
outputs: [{ type: 'console', format: 'structured', configuration: {} }],
structured: true,
includeStackTrace: true,
},
health: {
checkInterval: 30000,
timeoutMs: 5000,
retryAttempts: 3,
},
alerting: {
enabled: true,
rules: [],
defaultCooldown: 300000,
},
storage: {
dataPath: './observability-data',
maxFileSize: 104857600,
compressionEnabled: true,
encryptionEnabled: false,
},
});
}
//# sourceMappingURL=observability-system.js.map