@ahmedhegazee/nestjs-telescope
Version:
Advanced observability and monitoring solution for NestJS applications with ML-powered analytics, enterprise features, and production-ready scaling
551 lines • 24.3 kB
JavaScript
"use strict";
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var MLAnalyticsService_1;
Object.defineProperty(exports, "__esModule", { value: true });
exports.MLAnalyticsService = void 0;
const common_1 = require("@nestjs/common");
const rxjs_1 = require("rxjs");
const analytics_service_1 = require("./analytics.service");
const performance_correlation_service_1 = require("./performance-correlation.service");
class StatisticalAnalyzer {
static calculateMovingAverage(data, window) {
const result = [];
for (let i = 0; i < data.length; i++) {
const start = Math.max(0, i - window + 1);
const subset = data.slice(start, i + 1);
const avg = subset.reduce((a, b) => a + b, 0) / subset.length;
result.push(avg);
}
return result;
}
static calculateStandardDeviation(data) {
const mean = data.reduce((a, b) => a + b, 0) / data.length;
const variance = data.reduce((sum, val) => sum + Math.pow(val - mean, 2), 0) / data.length;
return Math.sqrt(variance);
}
static calculateZScore(value, mean, stdDev) {
return (value - mean) / stdDev;
}
static detectOutliers(data, threshold = 2.5) {
const mean = data.reduce((a, b) => a + b, 0) / data.length;
const stdDev = this.calculateStandardDeviation(data);
return data.filter((val) => Math.abs(this.calculateZScore(val, mean, stdDev)) > threshold);
}
static exponentialSmoothing(data, alpha = 0.3) {
const result = [data[0]];
for (let i = 1; i < data.length; i++) {
result.push(alpha * data[i] + (1 - alpha) * result[i - 1]);
}
return result;
}
static linearRegression(x, y) {
const n = x.length;
const sumX = x.reduce((a, b) => a + b, 0);
const sumY = y.reduce((a, b) => a + b, 0);
const sumXY = x.reduce((sum, xi, i) => sum + xi * y[i], 0);
const sumXX = x.reduce((sum, xi) => sum + xi * xi, 0);
const sumYY = y.reduce((sum, yi) => sum + yi * yi, 0);
const slope = (n * sumXY - sumX * sumY) / (n * sumXX - sumX * sumX);
const intercept = (sumY - slope * sumX) / n;
const yMean = sumY / n;
const ssRes = y.reduce((sum, yi, i) => sum + Math.pow(yi - (slope * x[i] + intercept), 2), 0);
const ssTot = y.reduce((sum, yi) => sum + Math.pow(yi - yMean, 2), 0);
const rSquared = 1 - ssRes / ssTot;
return { slope, intercept, rSquared };
}
}
let MLAnalyticsService = MLAnalyticsService_1 = class MLAnalyticsService {
constructor(analyticsService, performanceCorrelationService) {
this.analyticsService = analyticsService;
this.performanceCorrelationService = performanceCorrelationService;
this.logger = new common_1.Logger(MLAnalyticsService_1.name);
this.dataHistory = new Map();
this.anomalySubject = new rxjs_1.BehaviorSubject([]);
this.regressionSubject = new rxjs_1.BehaviorSubject([]);
this.optimizationSubject = new rxjs_1.BehaviorSubject([]);
this.predictionSubject = new rxjs_1.BehaviorSubject([]);
this.alertSubject = new rxjs_1.BehaviorSubject([]);
this.config = {
anomalyDetection: {
zScoreThreshold: 2.5,
windowSize: 50,
minDataPoints: 10,
confidenceThreshold: 0.7,
},
regressionAnalysis: {
windowSize: 100,
rSquaredThreshold: 0.5,
significantChangeThreshold: 0.1,
},
prediction: {
smoothingFactor: 0.3,
predictionHorizon: {
short: 6,
medium: 24,
long: 168,
},
},
};
}
async onModuleInit() {
this.logger.log('ML Analytics Service initialized');
this.startMLAnalysis();
}
startMLAnalysis() {
this.analyticsService.getAnalyticsStream().subscribe((data) => {
this.updateDataHistory(data);
this.performAnomalyDetection(data);
this.performRegressionAnalysis(data);
this.generatePredictiveInsights(data);
this.analyzeQueryOptimizationOpportunities(data);
});
(0, rxjs_1.interval)(300000).subscribe(() => {
this.performAdvancedAnalysis();
});
}
updateDataHistory(data) {
const metrics = {
response_time: data.overview.averageResponseTime,
error_rate: data.overview.errorRate,
throughput: data.overview.throughput,
active_users: data.overview.activeUsers,
db_connections: data.database.connectionHealth.activeConnections,
};
if (data.performance.resourceUsage.cpu.length > 0) {
const latestCpu = data.performance.resourceUsage.cpu[data.performance.resourceUsage.cpu.length - 1];
metrics['cpu_usage'] = latestCpu.value;
}
if (data.performance.resourceUsage.memory.length > 0) {
const latestMemory = data.performance.resourceUsage.memory[data.performance.resourceUsage.memory.length - 1];
metrics['memory_usage'] = latestMemory.value;
}
Object.entries(metrics).forEach(([metric, value]) => {
if (typeof value === 'number' && !isNaN(value)) {
if (!this.dataHistory.has(metric)) {
this.dataHistory.set(metric, []);
}
const history = this.dataHistory.get(metric);
history.push(value);
if (history.length > 1000) {
history.shift();
}
}
});
}
performAnomalyDetection(data) {
const anomalies = [];
this.dataHistory.forEach((history, metric) => {
if (history.length < this.config.anomalyDetection.minDataPoints) {
return;
}
const currentValue = history[history.length - 1];
const recentHistory = history.slice(-this.config.anomalyDetection.windowSize);
const mean = recentHistory.reduce((a, b) => a + b, 0) / recentHistory.length;
const stdDev = StatisticalAnalyzer.calculateStandardDeviation(recentHistory);
const zScore = StatisticalAnalyzer.calculateZScore(currentValue, mean, stdDev);
if (Math.abs(zScore) > this.config.anomalyDetection.zScoreThreshold) {
const anomaly = {
id: `anomaly_${Date.now()}_${metric}`,
timestamp: new Date(),
type: this.classifyAnomalyType(metric),
severity: this.calculateAnomalySeverity(Math.abs(zScore)),
component: this.getComponentFromMetric(metric),
metric,
value: currentValue,
baseline: mean,
deviation: Math.abs(currentValue - mean),
confidence: Math.min(Math.abs(zScore) / 5, 1),
description: this.generateAnomalyDescription(metric, currentValue, mean, zScore),
suggestedActions: this.generateAnomalySuggestions(metric, zScore > 0),
};
anomalies.push(anomaly);
this.logger.warn(`Anomaly detected: ${anomaly.description}`);
}
});
if (anomalies.length > 0) {
const currentAnomalies = this.anomalySubject.value;
this.anomalySubject.next([...currentAnomalies, ...anomalies]);
this.generateAlertsFromAnomalies(anomalies);
}
}
performRegressionAnalysis(data) {
const regressions = [];
this.dataHistory.forEach((history, metric) => {
if (history.length < this.config.regressionAnalysis.windowSize) {
return;
}
const recentHistory = history.slice(-this.config.regressionAnalysis.windowSize);
const xValues = recentHistory.map((_, i) => i);
const regression = StatisticalAnalyzer.linearRegression(xValues, recentHistory);
if (regression.rSquared > this.config.regressionAnalysis.rSquaredThreshold) {
const regressionRate = (regression.slope / recentHistory[0]) * 100;
if (Math.abs(regressionRate) >
this.config.regressionAnalysis.significantChangeThreshold * 100) {
const analysis = {
id: `regression_${Date.now()}_${metric}`,
timestamp: new Date(),
metric,
component: this.getComponentFromMetric(metric),
timeWindow: `${this.config.regressionAnalysis.windowSize} data points`,
trend: regressionRate > 0 ? 'degrading' : 'improving',
regressionRate,
confidence: regression.rSquared,
predictedValue: regression.slope * (recentHistory.length - 1) + regression.intercept,
actualValue: recentHistory[recentHistory.length - 1],
impactAssessment: this.assessRegressionImpact(metric, regressionRate),
};
regressions.push(analysis);
}
}
});
if (regressions.length > 0) {
const currentRegressions = this.regressionSubject.value;
this.regressionSubject.next([...currentRegressions, ...regressions]);
}
}
generatePredictiveInsights(data) {
const insights = [];
this.dataHistory.forEach((history, metric) => {
if (history.length < 50)
return;
const smoothed = StatisticalAnalyzer.exponentialSmoothing(history, this.config.prediction.smoothingFactor);
const trend = this.calculateTrend(smoothed.slice(-20));
const currentValue = history[history.length - 1];
const recentTrend = smoothed[smoothed.length - 1] - smoothed[smoothed.length - 2];
const predictedValue = currentValue + recentTrend * this.config.prediction.predictionHorizon.short;
const insight = {
id: `prediction_${Date.now()}_${metric}`,
timestamp: new Date(),
predictionType: this.getPredictionType(metric),
timeHorizon: '6h',
metric,
component: this.getComponentFromMetric(metric),
currentValue,
predictedValue,
confidence: this.calculatePredictionConfidence(history),
trend,
riskLevel: this.assessPredictionRisk(metric, predictedValue, currentValue),
recommendedActions: this.generatePredictionRecommendations(metric, trend, predictedValue),
thresholds: this.getMetricThresholds(metric),
};
insights.push(insight);
});
if (insights.length > 0) {
const currentInsights = this.predictionSubject.value;
this.predictionSubject.next([...currentInsights, ...insights]);
}
}
analyzeQueryOptimizationOpportunities(data) {
const suggestions = [];
data.database.slowQueries.forEach((query) => {
if (query.averageTime > 1000) {
const suggestion = {
id: `optimization_${Date.now()}_${query.query}`,
timestamp: new Date(),
queryHash: query.query,
query: query.query,
table: query.table,
currentPerformance: {
executionTime: query.averageTime,
ioOperations: 0,
cpuUsage: 0,
},
optimizationStrategy: this.suggestOptimizationStrategy(query),
};
suggestions.push(suggestion);
}
});
if (suggestions.length > 0) {
const currentSuggestions = this.optimizationSubject.value;
this.optimizationSubject.next([...currentSuggestions, ...suggestions]);
}
}
performAdvancedAnalysis() {
this.logger.debug('Performing advanced ML analysis...');
}
classifyAnomalyType(metric) {
if (metric.includes('response_time') || metric.includes('cpu') || metric.includes('memory')) {
return 'performance';
}
if (metric.includes('error'))
return 'error';
if (metric.includes('throughput') || metric.includes('users'))
return 'traffic';
if (metric.includes('cpu') || metric.includes('memory') || metric.includes('connections')) {
return 'resource';
}
return 'performance';
}
calculateAnomalySeverity(zScore) {
if (zScore > 4)
return 'critical';
if (zScore > 3)
return 'high';
if (zScore > 2.5)
return 'medium';
return 'low';
}
getComponentFromMetric(metric) {
if (metric.includes('db') || metric.includes('query'))
return 'database';
if (metric.includes('cache'))
return 'cache';
if (metric.includes('response') || metric.includes('throughput'))
return 'application';
if (metric.includes('memory') || metric.includes('cpu'))
return 'system';
return 'unknown';
}
generateAnomalyDescription(metric, value, baseline, zScore) {
const direction = zScore > 0 ? 'increased' : 'decreased';
const percentage = Math.abs(((value - baseline) / baseline) * 100).toFixed(1);
return `${metric} has ${direction} by ${percentage}% (current: ${value.toFixed(2)}, baseline: ${baseline.toFixed(2)})`;
}
generateAnomalySuggestions(metric, isIncrease) {
const suggestions = [];
if (metric.includes('response_time') && isIncrease) {
suggestions.push('Check for slow database queries', 'Review recent deployments', 'Monitor CPU and memory usage');
}
else if (metric.includes('error_rate') && isIncrease) {
suggestions.push('Check application logs', 'Review recent code changes', 'Verify external service availability');
}
else if (metric.includes('memory') && isIncrease) {
suggestions.push('Check for memory leaks', 'Review garbage collection settings', 'Monitor application memory usage');
}
return suggestions.length > 0 ? suggestions : ['Investigate the root cause', 'Monitor closely'];
}
assessRegressionImpact(metric, regressionRate) {
const severity = Math.abs(regressionRate) > 50
? 'critical'
: Math.abs(regressionRate) > 25
? 'high'
: Math.abs(regressionRate) > 10
? 'medium'
: 'low';
return {
severity,
affectedUsers: this.estimateAffectedUsers(metric, regressionRate),
estimatedLoss: this.estimateLoss(metric, regressionRate),
timeToRevert: this.estimateRevertTime(severity),
};
}
calculateTrend(data) {
const regression = StatisticalAnalyzer.linearRegression(data.map((_, i) => i), data);
if (Math.abs(regression.slope) < 0.01)
return 'stable';
if (regression.slope > 0.1)
return 'increasing';
if (regression.slope < -0.1)
return 'decreasing';
return 'volatile';
}
calculatePredictionConfidence(history) {
const recentData = history.slice(-20);
const stdDev = StatisticalAnalyzer.calculateStandardDeviation(recentData);
const mean = recentData.reduce((a, b) => a + b, 0) / recentData.length;
const coefficientOfVariation = stdDev / Math.abs(mean);
return Math.max(0, 1 - coefficientOfVariation);
}
getPredictionType(metric) {
if (metric.includes('throughput') || metric.includes('users'))
return 'load';
if (metric.includes('error'))
return 'failure';
if (metric.includes('response_time') || metric.includes('cpu'))
return 'performance';
return 'resource';
}
assessPredictionRisk(metric, predicted, current) {
const change = Math.abs((predicted - current) / current);
if (change > 0.5)
return 'critical';
if (change > 0.3)
return 'high';
if (change > 0.1)
return 'medium';
return 'low';
}
generatePredictionRecommendations(metric, trend, predictedValue) {
const recommendations = [];
if (trend === 'increasing' && metric.includes('response_time')) {
recommendations.push('Consider scaling infrastructure', 'Optimize database queries', 'Review caching strategy');
}
else if (trend === 'increasing' && metric.includes('error_rate')) {
recommendations.push('Investigate error patterns', 'Enhance error handling', 'Monitor dependencies');
}
return recommendations.length > 0
? recommendations
: ['Monitor closely', 'Review system health'];
}
getMetricThresholds(metric) {
if (metric.includes('response_time'))
return { warning: 500, critical: 1000 };
if (metric.includes('error_rate'))
return { warning: 0.01, critical: 0.05 };
if (metric.includes('cpu'))
return { warning: 0.7, critical: 0.9 };
if (metric.includes('memory'))
return { warning: 0.8, critical: 0.95 };
return { warning: 100, critical: 200 };
}
suggestOptimizationStrategy(query) {
let type = 'index';
let suggestion = 'Consider adding an index';
let estimatedImprovement = 30;
let confidence = 0.7;
let effort = 'low';
if (query.sql?.includes('SELECT *')) {
type = 'rewrite';
suggestion = 'Select only required columns instead of using SELECT *';
estimatedImprovement = 20;
effort = 'low';
}
else if (query.sql?.includes('ORDER BY') && !query.sql?.includes('INDEX')) {
type = 'index';
suggestion = 'Add an index on the ORDER BY column';
estimatedImprovement = 50;
}
else if (query.executionTime > 5000) {
type = 'cache';
suggestion = 'Consider caching this query result';
estimatedImprovement = 80;
effort = 'medium';
}
return { type, suggestion, estimatedImprovement, confidence, effort };
}
generateAlertsFromAnomalies(anomalies) {
const alerts = anomalies
.filter((anomaly) => anomaly.severity === 'high' || anomaly.severity === 'critical')
.map((anomaly) => ({
id: `alert_${Date.now()}_${anomaly.id}`,
timestamp: new Date(),
type: 'anomaly',
severity: anomaly.severity === 'critical' ? 'critical' : 'error',
title: `Anomaly Detected: ${anomaly.metric}`,
description: anomaly.description,
component: anomaly.component,
metric: anomaly.metric,
triggeredBy: {
value: anomaly.value,
threshold: anomaly.baseline,
confidence: anomaly.confidence,
},
actions: [
{
type: 'investigate',
description: 'Investigate root cause of anomaly',
priority: 1,
automated: false,
},
...(anomaly.severity === 'critical'
? [
{
type: 'alert',
description: 'Notify on-call engineer',
priority: 0,
automated: true,
},
]
: []),
],
relatedInsights: [],
}));
if (alerts.length > 0) {
const currentAlerts = this.alertSubject.value;
this.alertSubject.next([...currentAlerts, ...alerts]);
}
}
estimateAffectedUsers(metric, regressionRate) {
if (metric.includes('response_time')) {
return Math.floor(Math.abs(regressionRate) * 100);
}
return Math.floor(Math.abs(regressionRate) * 50);
}
estimateLoss(metric, regressionRate) {
const impact = Math.abs(regressionRate);
if (impact > 50)
return 'High - Significant user impact';
if (impact > 25)
return 'Medium - Noticeable degradation';
return 'Low - Minor impact';
}
estimateRevertTime(severity) {
switch (severity) {
case 'critical':
return '< 1 hour';
case 'high':
return '< 4 hours';
case 'medium':
return '< 24 hours';
default:
return '< 7 days';
}
}
getAnomalies() {
return this.anomalySubject.asObservable();
}
getRegressionAnalysis() {
return this.regressionSubject.asObservable();
}
getOptimizationSuggestions() {
return this.optimizationSubject.asObservable();
}
getPredictiveInsights() {
return this.predictionSubject.asObservable();
}
getMLAlerts() {
return this.alertSubject.asObservable();
}
getCurrentAnomalies() {
return this.anomalySubject.value;
}
getCurrentRegressions() {
return this.regressionSubject.value;
}
getCurrentOptimizations() {
return this.optimizationSubject.value;
}
getCurrentPredictions() {
return this.predictionSubject.value;
}
getCurrentAlerts() {
return this.alertSubject.value;
}
acknowledgeAlert(alertId) {
const currentAlerts = this.alertSubject.value;
const updatedAlerts = currentAlerts.filter((alert) => alert.id !== alertId);
this.alertSubject.next(updatedAlerts);
return currentAlerts.length !== updatedAlerts.length;
}
dismissAnomaly(anomalyId) {
const currentAnomalies = this.anomalySubject.value;
const updatedAnomalies = currentAnomalies.filter((anomaly) => anomaly.id !== anomalyId);
this.anomalySubject.next(updatedAnomalies);
return currentAnomalies.length !== updatedAnomalies.length;
}
getMLMetrics() {
return {
anomaliesDetected: this.anomalySubject.value.length,
regressionsAnalyzed: this.regressionSubject.value.length,
optimizationSuggestions: this.optimizationSubject.value.length,
predictiveInsights: this.predictionSubject.value.length,
activeAlerts: this.alertSubject.value.length,
dataHistorySize: Array.from(this.dataHistory.values()).reduce((sum, arr) => sum + arr.length, 0),
};
}
};
exports.MLAnalyticsService = MLAnalyticsService;
exports.MLAnalyticsService = MLAnalyticsService = MLAnalyticsService_1 = __decorate([
(0, common_1.Injectable)(),
__metadata("design:paramtypes", [analytics_service_1.AnalyticsService,
performance_correlation_service_1.PerformanceCorrelationService])
], MLAnalyticsService);
//# sourceMappingURL=ml-analytics.service.js.map