@ahmedhegazee/nestjs-telescope
Version:
Advanced observability and monitoring solution for NestJS applications with ML-powered analytics, enterprise features, and production-ready scaling
610 lines • 25.4 kB
JavaScript
;
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var AutomatedAlertingService_1;
Object.defineProperty(exports, "__esModule", { value: true });
exports.AutomatedAlertingService = void 0;
const common_1 = require("@nestjs/common");
const rxjs_1 = require("rxjs");
const operators_1 = require("rxjs/operators");
const ml_analytics_service_1 = require("./ml-analytics.service");
const analytics_service_1 = require("./analytics.service");
const memory_manager_service_1 = require("./memory-manager.service");
let AutomatedAlertingService = AutomatedAlertingService_1 = class AutomatedAlertingService {
constructor(mlAnalyticsService, analyticsService, memoryManager) {
this.mlAnalyticsService = mlAnalyticsService;
this.analyticsService = analyticsService;
this.memoryManager = memoryManager;
this.logger = new common_1.Logger(AutomatedAlertingService_1.name);
this.subscriptions = [];
this.alertChannels = new Map();
this.alertRules = new Map();
this.rateLimitTracker = new Map();
this.collectionIds = {
alertHistory: 'automated-alerting-history',
aggregations: 'automated-alerting-aggregations',
rateLimits: 'automated-alerting-rate-limits',
};
this.alertHistorySubject = new rxjs_1.Subject();
this.aggregationSubject = new rxjs_1.Subject();
this.alertHistory = [];
this.config = {
aggregationWindow: 5,
maxHistorySize: 10000,
defaultRateLimit: { maxAlerts: 10, timeWindow: 60 },
escalationDelay: 30,
autoAcknowledgeTimeout: 24 * 60,
retryAttempts: 3,
retryDelay: 300,
};
this.initializeDefaultChannels();
this.initializeDefaultRules();
}
async onModuleInit() {
this.logger.log('Automated Alerting Service initialized');
this.startAlertProcessing();
}
onModuleDestroy() {
this.subscriptions.forEach((sub) => sub.unsubscribe());
}
initializeDefaultChannels() {
this.alertChannels.set('default-webhook', {
id: 'default-webhook',
name: 'Default Webhook',
type: 'webhook',
enabled: true,
config: {
url: 'http://localhost:3000/telescope/webhook/alerts',
},
severityFilter: ['warning', 'error', 'critical'],
});
this.alertChannels.set('console-log', {
id: 'console-log',
name: 'Console Logger',
type: 'webhook',
enabled: true,
config: {
url: 'console',
},
severityFilter: ['info', 'warning', 'error', 'critical'],
});
}
initializeDefaultRules() {
this.alertRules.set('critical-performance', {
id: 'critical-performance',
name: 'Critical Performance Degradation',
description: 'Alert when response time increases significantly',
enabled: true,
priority: 1,
conditions: [
{
metric: 'response_time',
operator: '>',
threshold: 2000,
duration: 5,
component: 'application',
},
],
actions: {
channelIds: ['default-webhook', 'console-log'],
escalation: {
delayMinutes: 15,
channels: ['default-webhook'],
},
},
});
this.alertRules.set('high-error-rate', {
id: 'high-error-rate',
name: 'High Error Rate',
description: 'Alert when error rate exceeds threshold',
enabled: true,
priority: 2,
conditions: [
{
metric: 'error_rate',
operator: '>',
threshold: 0.05,
duration: 3,
},
],
actions: {
channelIds: ['default-webhook', 'console-log'],
},
});
this.alertRules.set('resource-exhaustion', {
id: 'resource-exhaustion',
name: 'Resource Exhaustion Warning',
description: 'Alert when system resources are running low',
enabled: true,
priority: 3,
conditions: [
{
metric: 'memory_usage',
operator: '>',
threshold: 0.9,
duration: 2,
component: 'system',
},
],
actions: {
channelIds: ['console-log'],
},
});
}
startAlertProcessing() {
const mlAlertSub = this.mlAnalyticsService
.getMLAlerts()
.pipe((0, operators_1.debounceTime)(1000))
.subscribe((alerts) => {
alerts.forEach((alert) => this.processAlert(alert));
});
const anomalySub = this.mlAnalyticsService
.getAnomalies()
.pipe((0, operators_1.distinctUntilChanged)((prev, curr) => prev.length === curr.length), (0, operators_1.debounceTime)(2000))
.subscribe((anomalies) => {
const newAnomalies = anomalies.slice(-10);
newAnomalies.forEach((anomaly) => this.processAnomalyAlert(anomaly));
});
const regressionSub = this.mlAnalyticsService
.getRegressionAnalysis()
.pipe((0, operators_1.distinctUntilChanged)((prev, curr) => prev.length === curr.length), (0, operators_1.debounceTime)(2000))
.subscribe((regressions) => {
const newRegressions = regressions.slice(-5);
newRegressions.forEach((regression) => this.processRegressionAlert(regression));
});
const predictionSub = this.mlAnalyticsService
.getPredictiveInsights()
.pipe((0, operators_1.filter)((insights) => insights.some((i) => i.riskLevel === 'high' || i.riskLevel === 'critical')), (0, operators_1.debounceTime)(5000))
.subscribe((insights) => {
const highRiskInsights = insights.filter((i) => i.riskLevel === 'high' || i.riskLevel === 'critical');
highRiskInsights.forEach((insight) => this.processPredictiveAlert(insight));
});
const cleanupSub = (0, rxjs_1.interval)(300000).subscribe(() => {
this.cleanupOldHistory();
this.processAlertAggregation();
this.autoAcknowledgeOldAlerts();
});
this.subscriptions.push(mlAlertSub, anomalySub, regressionSub, predictionSub, cleanupSub);
}
async processAlert(alert) {
this.logger.debug(`Processing alert: ${alert.title}`);
if (this.isRateLimited(alert)) {
this.logger.warn(`Alert rate limited: ${alert.id}`);
return;
}
const matchingRules = this.findMatchingRules(alert);
for (const rule of matchingRules) {
await this.executeAlertRule(alert, rule);
}
if (matchingRules.length === 0) {
await this.executeDefaultAlert(alert);
}
}
async processAnomalyAlert(anomaly) {
if (anomaly.severity === 'low')
return;
const alert = {
id: `anomaly_alert_${anomaly.id}`,
timestamp: new Date(),
type: 'anomaly',
severity: anomaly.severity === 'critical'
? 'critical'
: anomaly.severity === 'high'
? 'error'
: 'warning',
title: `Anomaly Detected: ${anomaly.metric}`,
description: anomaly.description,
component: anomaly.component,
metric: anomaly.metric,
triggeredBy: {
value: anomaly.value,
threshold: anomaly.baseline,
confidence: anomaly.confidence,
},
actions: anomaly.suggestedActions.map((action, index) => ({
type: 'investigate',
description: action,
priority: index,
automated: false,
})),
relatedInsights: [],
};
await this.processAlert(alert);
}
async processRegressionAlert(regression) {
if (regression.trend !== 'degrading' || regression.impactAssessment.severity === 'low') {
return;
}
const alert = {
id: `regression_alert_${regression.id}`,
timestamp: new Date(),
type: 'regression',
severity: regression.impactAssessment.severity === 'critical' ? 'critical' : 'warning',
title: `Performance Regression: ${regression.metric}`,
description: `${regression.metric} has been degrading at ${regression.regressionRate.toFixed(2)}% rate`,
component: regression.component,
metric: regression.metric,
triggeredBy: {
value: regression.actualValue,
threshold: regression.predictedValue,
confidence: regression.confidence,
},
actions: [
{
type: 'investigate',
description: 'Investigate performance regression',
priority: 1,
automated: false,
},
],
relatedInsights: [],
};
await this.processAlert(alert);
}
async processPredictiveAlert(insight) {
const alert = {
id: `prediction_alert_${insight.id}`,
timestamp: new Date(),
type: 'prediction',
severity: insight.riskLevel === 'critical' ? 'critical' : 'warning',
title: `Predictive Alert: ${insight.metric}`,
description: `Predicted ${insight.trend} trend for ${insight.metric} (${insight.timeHorizon})`,
component: insight.component,
metric: insight.metric,
triggeredBy: {
value: insight.predictedValue,
threshold: insight.thresholds.warning,
confidence: insight.confidence,
},
actions: insight.recommendedActions.map((action, index) => ({
type: 'investigate',
description: action,
priority: index,
automated: false,
})),
relatedInsights: [],
};
await this.processAlert(alert);
}
findMatchingRules(alert) {
return Array.from(this.alertRules.values()).filter((rule) => {
if (!rule.enabled)
return false;
return rule.conditions.some((condition) => {
if (condition.component && condition.component !== alert.component) {
return false;
}
if (condition.metric && condition.metric !== alert.metric) {
return false;
}
switch (condition.operator) {
case '>':
return alert.triggeredBy.value > condition.threshold;
case '<':
return alert.triggeredBy.value < condition.threshold;
case '>=':
return alert.triggeredBy.value >= condition.threshold;
case '<=':
return alert.triggeredBy.value <= condition.threshold;
case '==':
return alert.triggeredBy.value === condition.threshold;
case '!=':
return alert.triggeredBy.value !== condition.threshold;
default:
return false;
}
});
});
}
async executeAlertRule(alert, rule) {
this.logger.debug(`Executing alert rule: ${rule.name} for alert: ${alert.title}`);
if (!this.isWithinSchedule(rule)) {
this.logger.debug(`Alert rule ${rule.name} is outside active schedule`);
return;
}
for (const channelId of rule.actions.channelIds) {
const channel = this.alertChannels.get(channelId);
if (channel && channel.enabled) {
await this.sendAlert(alert, channel, rule);
}
}
if (rule.actions.escalation) {
setTimeout(() => {
this.escalateAlert(alert, rule);
}, rule.actions.escalation.delayMinutes * 60 * 1000);
}
if (rule.actions.autoRemediation?.enabled) {
this.executeAutoRemediation(alert, rule.actions.autoRemediation.actions);
}
}
async executeDefaultAlert(alert) {
const defaultChannel = this.alertChannels.get('console-log');
if (defaultChannel) {
await this.sendAlert(alert, defaultChannel);
}
}
async sendAlert(alert, channel, rule) {
const historyEntry = {
id: `history_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`,
alertId: alert.id,
ruleId: rule?.id,
timestamp: new Date(),
status: 'sent',
channel: channel.name,
recipient: undefined,
retryCount: 0,
error: undefined,
responseTime: undefined,
};
try {
const startTime = Date.now();
switch (channel.type) {
case 'webhook':
await this.sendWebhookAlert(alert, channel);
break;
case 'email':
await this.sendEmailAlert(alert, channel);
break;
case 'slack':
await this.sendSlackAlert(alert, channel);
break;
default:
throw new Error(`Unsupported channel type: ${channel.type}`);
}
historyEntry.status = 'sent';
historyEntry.responseTime = Date.now() - startTime;
this.logger.log(`Alert sent successfully: ${alert.title} via ${channel.name}`);
}
catch (error) {
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
const errorStack = error instanceof Error ? error.stack : undefined;
historyEntry.status = 'failed';
historyEntry.error = errorMessage;
this.logger.error(`Failed to send alert: ${errorMessage}`, errorStack);
if (historyEntry.retryCount < this.config.retryAttempts) {
setTimeout(() => {
this.retryAlert(alert, channel, historyEntry);
}, this.config.retryDelay * 1000);
}
}
this.alertHistory.push(historyEntry);
this.alertHistorySubject.next(historyEntry);
}
async sendWebhookAlert(alert, channel) {
if (channel.config.url === 'console') {
console.log(`🚨 TELESCOPE ALERT [${alert.severity.toUpperCase()}] ${alert.title}`);
console.log(` Component: ${alert.component}`);
console.log(` Description: ${alert.description}`);
console.log(` Timestamp: ${alert.timestamp.toISOString()}`);
if (alert.metric) {
console.log(` Metric: ${alert.metric} = ${alert.triggeredBy.value}`);
}
console.log(' Suggested Actions:');
alert.actions.forEach((action) => {
console.log(` - ${action.description}`);
});
console.log('---');
return;
}
const payload = {
alert_id: alert.id,
timestamp: alert.timestamp,
type: alert.type,
severity: alert.severity,
title: alert.title,
description: alert.description,
component: alert.component,
metric: alert.metric,
triggered_by: alert.triggeredBy,
actions: alert.actions,
};
this.logger.debug(`Would send webhook to: ${channel.config.url}`, payload);
}
async sendEmailAlert(alert, channel) {
this.logger.debug(`Would send email alert to: ${channel.config.email}`);
}
async sendSlackAlert(alert, channel) {
this.logger.debug(`Would send Slack alert to: ${channel.config.url}`);
}
async retryAlert(alert, channel, historyEntry) {
historyEntry.retryCount++;
historyEntry.timestamp = new Date();
try {
await this.sendAlert(alert, channel);
}
catch (error) {
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
this.logger.error(`Retry ${historyEntry.retryCount} failed for alert ${alert.id}: ${errorMessage}`);
}
}
escalateAlert(alert, rule) {
if (!rule.actions.escalation)
return;
this.logger.warn(`Escalating alert: ${alert.title}`);
rule.actions.escalation.channels.forEach((channelId) => {
const channel = this.alertChannels.get(channelId);
if (channel && channel.enabled) {
this.sendAlert(alert, channel, rule);
}
});
}
executeAutoRemediation(alert, actions) {
this.logger.warn(`Auto-remediation triggered for alert: ${alert.title}`);
actions.forEach((action) => {
switch (action.type) {
case 'restart':
this.logger.log(`Would restart ${action.config.service}`);
break;
case 'scale':
this.logger.log(`Would scale ${action.config.service} to ${action.config.replicas} replicas`);
break;
case 'cache_clear':
this.logger.log(`Would clear cache for ${action.config.keys || 'all keys'}`);
break;
default:
this.logger.warn(`Unknown auto-remediation action: ${action.type}`);
}
});
}
isRateLimited(alert) {
const key = `${alert.component}_${alert.type}`;
const now = Date.now();
const tracker = this.rateLimitTracker.get(key);
if (!tracker || now > tracker.resetTime) {
this.rateLimitTracker.set(key, {
count: 1,
resetTime: now + this.config.defaultRateLimit.timeWindow * 60 * 1000,
});
return false;
}
if (tracker.count >= this.config.defaultRateLimit.maxAlerts) {
return true;
}
tracker.count++;
return false;
}
isWithinSchedule(rule) {
if (!rule.schedule)
return true;
const now = new Date();
const currentDay = now.getDay();
const currentHour = now.getHours();
const currentMinute = now.getMinutes();
const currentTime = currentHour * 60 + currentMinute;
if (!rule.schedule.activeDays.includes(currentDay)) {
return false;
}
const startTime = this.parseTime(rule.schedule.activeHours.start);
const endTime = this.parseTime(rule.schedule.activeHours.end);
return currentTime >= startTime && currentTime <= endTime;
}
parseTime(timeStr) {
const [hours, minutes] = timeStr.split(':').map(Number);
return (hours || 0) * 60 + (minutes || 0);
}
cleanupOldHistory() {
if (this.alertHistory.length > this.config.maxHistorySize) {
const excess = this.alertHistory.length - this.config.maxHistorySize;
this.alertHistory.splice(0, excess);
this.logger.debug(`Cleaned up ${excess} old alert history entries`);
}
}
processAlertAggregation() {
const recentAlerts = this.alertHistory.filter((h) => Date.now() - h.timestamp.getTime() < this.config.aggregationWindow * 60 * 1000);
this.logger.debug(`Processing aggregation for ${recentAlerts.length} recent alerts`);
}
autoAcknowledgeOldAlerts() {
const cutoffTime = Date.now() - this.config.autoAcknowledgeTimeout * 60 * 1000;
const oldAlerts = this.alertHistory.filter((h) => h.timestamp.getTime() < cutoffTime && h.status === 'sent');
oldAlerts.forEach((alert) => {
alert.status = 'acknowledged';
});
if (oldAlerts.length > 0) {
this.logger.debug(`Auto-acknowledged ${oldAlerts.length} old alerts`);
}
}
addAlertChannel(channel) {
this.alertChannels.set(channel.id, channel);
this.logger.log(`Added alert channel: ${channel.name}`);
}
removeAlertChannel(channelId) {
const removed = this.alertChannels.delete(channelId);
if (removed) {
this.logger.log(`Removed alert channel: ${channelId}`);
}
return removed;
}
addAlertRule(rule) {
this.alertRules.set(rule.id, rule);
this.logger.log(`Added alert rule: ${rule.name}`);
}
removeAlertRule(ruleId) {
const removed = this.alertRules.delete(ruleId);
if (removed) {
this.logger.log(`Removed alert rule: ${ruleId}`);
}
return removed;
}
getAlertChannels() {
return Array.from(this.alertChannels.values());
}
getAlertRules() {
return Array.from(this.alertRules.values());
}
getAlertHistory(limit = 100) {
return this.alertHistory.slice(-limit);
}
getAlertHistoryStream() {
return this.alertHistorySubject.asObservable();
}
acknowledgeAlert(alertId) {
const historyEntry = this.alertHistory.find((h) => h.alertId === alertId);
if (historyEntry && historyEntry.status === 'sent') {
historyEntry.status = 'acknowledged';
this.logger.log(`Alert acknowledged: ${alertId}`);
return true;
}
return false;
}
getAlertMetrics() {
const recentHistory = this.alertHistory.filter((h) => Date.now() - h.timestamp.getTime() < 24 * 60 * 60 * 1000);
const totalAlerts = recentHistory.length;
const successfulAlerts = recentHistory.filter((h) => h.status === 'sent').length;
const escalatedAlerts = recentHistory.filter((h) => h.channel.includes('escalation')).length;
const acknowledgedAlerts = recentHistory.filter((h) => h.status === 'acknowledged').length;
const alertsByChannel = {};
recentHistory.forEach((h) => {
alertsByChannel[h.channel] = (alertsByChannel[h.channel] || 0) + 1;
});
const responseTimes = recentHistory.filter((h) => h.responseTime).map((h) => h.responseTime);
return {
totalAlerts,
alertsByChannel,
alertsByComponent: {},
alertsBySeverity: {},
averageResponseTime: responseTimes.length > 0
? responseTimes.reduce((a, b) => a + b, 0) / responseTimes.length
: 0,
successRate: totalAlerts > 0 ? successfulAlerts / totalAlerts : 1,
escalationRate: totalAlerts > 0 ? escalatedAlerts / totalAlerts : 0,
acknowledgedRate: totalAlerts > 0 ? acknowledgedAlerts / totalAlerts : 0,
falsePositiveRate: 0,
};
}
testAlertChannel(channelId) {
const channel = this.alertChannels.get(channelId);
if (!channel) {
throw new Error(`Channel not found: ${channelId}`);
}
const testAlert = {
id: `test_${Date.now()}`,
timestamp: new Date(),
type: 'anomaly',
severity: 'info',
title: 'Test Alert',
description: 'This is a test alert to verify channel configuration',
component: 'test',
triggeredBy: { value: 100, threshold: 90, confidence: 1 },
actions: [{ type: 'investigate', description: 'Test action', priority: 1, automated: false }],
relatedInsights: [],
};
return this.sendAlert(testAlert, channel)
.then(() => true)
.catch(() => false);
}
};
exports.AutomatedAlertingService = AutomatedAlertingService;
exports.AutomatedAlertingService = AutomatedAlertingService = AutomatedAlertingService_1 = __decorate([
(0, common_1.Injectable)(),
__metadata("design:paramtypes", [ml_analytics_service_1.MLAnalyticsService,
analytics_service_1.AnalyticsService,
memory_manager_service_1.MemoryManagerService])
], AutomatedAlertingService);
//# sourceMappingURL=automated-alerting.service.js.map