@ahmedhegazee/nestjs-telescope/dist/telescope/watchers/job/job-watcher.service.js

Advanced observability and monitoring solution for NestJS applications with ML-powered analytics, enterprise features, and production-ready scaling

"use strict";
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
    return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var __param = (this && this.__param) || function (paramIndex, decorator) {
    return function (target, key) { decorator(target, key, paramIndex); }
};
var JobWatcherService_1;
Object.defineProperty(exports, "__esModule", { value: true });
exports.JobWatcherService = exports.JobStatus = void 0;
const common_1 = require("@nestjs/common");
const telescope_service_1 = require("../../core/services/telescope.service");
const job_watcher_config_1 = require("./job-watcher.config");
const rxjs_1 = require("rxjs");
const operators_1 = require("rxjs/operators");
var JobStatus;
(function (JobStatus) {
    JobStatus["WAITING"] = "waiting";
    JobStatus["ACTIVE"] = "active";
    JobStatus["COMPLETED"] = "completed";
    JobStatus["FAILED"] = "failed";
    JobStatus["DELAYED"] = "delayed";
    JobStatus["PAUSED"] = "paused";
    JobStatus["STALLED"] = "stalled";
    JobStatus["CANCELLED"] = "cancelled";
})(JobStatus || (exports.JobStatus = JobStatus = {}));
let JobWatcherService = JobWatcherService_1 = class JobWatcherService {
    constructor(telescopeService, jobWatcherConfig) {
        this.telescopeService = telescopeService;
        this.logger = new common_1.Logger(JobWatcherService_1.name);
        this.destroy$ = new rxjs_1.Subject();
        this.metricsSubject = new rxjs_1.BehaviorSubject(this.initializeMetrics());
        this.alertsSubject = new rxjs_1.Subject();
        this.jobContextSubject = new rxjs_1.Subject();
        this.jobHistory = [];
        this.queueMetrics = new Map();
        this.workerMetrics = new Map();
        this.activeJobs = new Map();
        this.currentMetrics = this.initializeMetrics();
        this.alertHistory = [];
        this.executionTimes = [];
        this.waitTimes = [];
        this.failureHistory = [];
        this.config = { ...job_watcher_config_1.defaultJobWatcherConfig, ...jobWatcherConfig };
    }
    async onModuleInit() {
        if (!this.config.enabled) {
            return;
        }
        this.startPeriodicProcessing();
        this.logger.log('Job watcher initialized');
    }
    onModuleDestroy() {
        this.destroy$.next();
        this.destroy$.complete();
    }
    initializeMetrics() {
        return {
            totalJobs: 0,
            completedJobs: 0,
            failedJobs: 0,
            activeJobs: 0,
            waitingJobs: 0,
            delayedJobs: 0,
            stalledJobs: 0,
            averageExecutionTime: 0,
            averageWaitTime: 0,
            slowJobs: 0,
            jobsPerMinute: 0,
            jobsPerHour: 0,
            failureRate: 0,
            throughput: 0,
            queuesByStatus: {},
            queuesByType: {},
            activeWorkers: 0,
            busyWorkers: 0,
            idleWorkers: 0,
            topFailedJobs: [],
            topSlowJobs: [],
            healthScore: 100,
            healthStatus: 'healthy',
            trends: {
                lastHour: this.initializeTrendData(),
                lastDay: this.initializeTrendData(),
                lastWeek: this.initializeTrendData(),
            },
        };
    }
    initializeTrendData() {
        return {
            totalJobs: 0,
            completedJobs: 0,
            failedJobs: 0,
            averageExecutionTime: 0,
            change: 0,
            changePercent: 0,
            peak: 0,
        };
    }
    trackJob(context) {
        if (!this.config.enabled) {
            return;
        }
        try {
            if (Math.random() * 100 > this.config.sampleRate) {
                return;
            }
            if (this.shouldExcludeJob(context)) {
                return;
            }
            this.addToHistory(context);
            this.updateActiveJobs(context);
            this.updateMetrics(context);
            const entry = this.createTelescopeEntry(context);
            this.telescopeService.record(entry);
            this.checkAlerts(context);
            this.correlateJob(context);
            this.jobContextSubject.next(context);
            this.logger.debug(`Job tracked: ${context.jobName} (${context.id})`);
        }
        catch (error) {
            this.logger.error('Failed to track job:', error);
        }
    }
    shouldExcludeJob(context) {
        if (this.config.excludeJobTypes.includes(context.jobName)) {
            return true;
        }
        if (this.config.excludeQueues.includes(context.queueName)) {
            return true;
        }
        return false;
    }
    addToHistory(context) {
        this.jobHistory.push(context);
        if (this.jobHistory.length > this.config.maxHistorySize) {
            this.jobHistory.shift();
        }
        const retentionDate = new Date(Date.now() - this.config.retentionPeriod);
        this.jobHistory = this.jobHistory.filter(job => job.timestamp > retentionDate);
    }
    updateActiveJobs(context) {
        const key = `${context.queueName}:${context.jobId}`;
        if (context.status === JobStatus.ACTIVE) {
            this.activeJobs.set(key, context);
        }
        else if (context.status === JobStatus.COMPLETED || context.status === JobStatus.FAILED) {
            this.activeJobs.delete(key);
        }
    }
    updateMetrics(context) {
        this.currentMetrics.totalJobs++;
        switch (context.status) {
            case JobStatus.COMPLETED:
                this.currentMetrics.completedJobs++;
                break;
            case JobStatus.FAILED:
                this.currentMetrics.failedJobs++;
                this.trackFailure(context);
                break;
            case JobStatus.ACTIVE:
                this.currentMetrics.activeJobs++;
                break;
            case JobStatus.WAITING:
                this.currentMetrics.waitingJobs++;
                break;
            case JobStatus.DELAYED:
                this.currentMetrics.delayedJobs++;
                break;
            case JobStatus.STALLED:
                this.currentMetrics.stalledJobs++;
                break;
        }
        if (context.duration) {
            this.executionTimes.push(context.duration);
            this.currentMetrics.averageExecutionTime = this.calculateAverage(this.executionTimes);
            if (context.duration > this.config.slowJobThreshold) {
                this.currentMetrics.slowJobs++;
            }
        }
        if (context.performance?.queueWaitTime) {
            this.waitTimes.push(context.performance.queueWaitTime);
            this.currentMetrics.averageWaitTime = this.calculateAverage(this.waitTimes);
        }
        this.currentMetrics.queuesByStatus[context.status] =
            (this.currentMetrics.queuesByStatus[context.status] || 0) + 1;
        this.currentMetrics.queuesByType[context.jobName] =
            (this.currentMetrics.queuesByType[context.jobName] || 0) + 1;
        this.updateRates();
        this.updateTopJobs();
        this.updateHealthScore();
    }
    trackFailure(context) {
        this.failureHistory.push({
            timestamp: context.timestamp,
            jobName: context.jobName,
            queueName: context.queueName,
        });
        const oneHourAgo = new Date(Date.now() - 60 * 60 * 1000);
        this.failureHistory = this.failureHistory.filter(f => f.timestamp > oneHourAgo);
    }
    calculateAverage(values) {
        if (values.length === 0)
            return 0;
        return values.reduce((sum, val) => sum + val, 0) / values.length;
    }
    updateRates() {
        const now = Date.now();
        const oneMinuteAgo = now - 60 * 1000;
        const oneHourAgo = now - 60 * 60 * 1000;
        const jobsLastMinute = this.jobHistory.filter(job => job.timestamp.getTime() > oneMinuteAgo).length;
        const jobsLastHour = this.jobHistory.filter(job => job.timestamp.getTime() > oneHourAgo).length;
        this.currentMetrics.jobsPerMinute = jobsLastMinute;
        this.currentMetrics.jobsPerHour = jobsLastHour;
        const totalJobs = this.currentMetrics.totalJobs || 1;
        this.currentMetrics.failureRate = (this.currentMetrics.failedJobs / totalJobs) * 100;
        this.currentMetrics.throughput = jobsLastMinute / 60;
    }
    updateTopJobs() {
        const failureGroups = new Map();
        this.failureHistory.forEach(failure => {
            const key = `${failure.queueName}:${failure.jobName}`;
            const existing = failureGroups.get(key) || { count: 0, lastFailure: failure.timestamp, execTimes: [] };
            existing.count++;
            existing.lastFailure = failure.timestamp;
            failureGroups.set(key, existing);
        });
        this.currentMetrics.topFailedJobs = Array.from(failureGroups.entries())
            .map(([key, data]) => {
            const [queueName, jobName] = key.split(':');
            return {
                queueName,
                jobName,
                failureCount: data.count,
                lastFailure: data.lastFailure,
                avgExecutionTime: this.calculateAverage(data.execTimes),
            };
        })
            .sort((a, b) => b.failureCount - a.failureCount)
            .slice(0, 10);
        const slowJobGroups = new Map();
        this.jobHistory
            .filter(job => job.duration && job.duration > this.config.slowJobThreshold)
            .forEach(job => {
            const key = `${job.queueName}:${job.jobName}`;
            const existing = slowJobGroups.get(key) || { execTimes: [], lastExecution: job.timestamp };
            existing.execTimes.push(job.duration);
            existing.lastExecution = job.timestamp;
            slowJobGroups.set(key, existing);
        });
        this.currentMetrics.topSlowJobs = Array.from(slowJobGroups.entries())
            .map(([key, data]) => {
            const [queueName, jobName] = key.split(':');
            return {
                queueName,
                jobName,
                avgExecutionTime: this.calculateAverage(data.execTimes),
                executionCount: data.execTimes.length,
                lastExecution: data.lastExecution,
            };
        })
            .sort((a, b) => b.avgExecutionTime - a.avgExecutionTime)
            .slice(0, 10);
    }
    updateHealthScore() {
        let score = 100;
        const issues = [];
        if (this.currentMetrics.failureRate > 20) {
            score -= 30;
            issues.push('High failure rate');
        }
        else if (this.currentMetrics.failureRate > 10) {
            score -= 15;
            issues.push('Moderate failure rate');
        }
        if (this.currentMetrics.slowJobs > this.currentMetrics.totalJobs * 0.2) {
            score -= 20;
            issues.push('Many slow jobs');
        }
        if (this.currentMetrics.stalledJobs > 0) {
            score -= 25;
            issues.push('Stalled jobs detected');
        }
        if (this.currentMetrics.averageWaitTime > 30000) {
            score -= 15;
            issues.push('High queue wait times');
        }
        this.currentMetrics.healthScore = Math.max(0, score);
        if (score >= 80) {
            this.currentMetrics.healthStatus = 'healthy';
        }
        else if (score >= 60) {
            this.currentMetrics.healthStatus = 'warning';
        }
        else {
            this.currentMetrics.healthStatus = 'critical';
        }
    }
    createTelescopeEntry(context) {
        const entryId = `job_${context.id}`;
        const familyHash = `${context.queueName}:${context.jobName}`;
        return {
            id: entryId,
            type: 'job',
            familyHash,
            content: {
                job: {
                    id: context.jobId,
                    name: context.jobName,
                    queue: context.queueName,
                    status: context.status,
                    priority: context.priority,
                    attempts: context.attempts,
                    maxAttempts: context.maxAttempts,
                    delay: context.delay,
                    progress: context.progress,
                },
                execution: {
                    startTime: context.startTime,
                    endTime: context.endTime,
                    duration: context.duration,
                    result: this.config.captureJobResults ? context.result : undefined,
                    error: this.config.captureJobErrors ? context.error : undefined,
                },
                data: this.config.captureJobData ? this.sanitizeJobData(context.data) : undefined,
                performance: context.performance,
                correlation: {
                    traceId: context.traceId,
                    requestId: context.requestId,
                    userId: context.userId,
                    sessionId: context.sessionId,
                },
                worker: {
                    id: context.workerId,
                    name: context.workerName,
                },
                queueStats: context.queueStats,
            },
            tags: this.generateTags(context),
            timestamp: context.timestamp,
            sequence: context.timestamp.getTime(),
        };
    }
    sanitizeJobData(data) {
        if (!data || typeof data !== 'object') {
            return data;
        }
        const sanitized = { ...data };
        const sensitiveFields = ['password', 'token', 'secret', 'key', 'auth', 'apiKey'];
        for (const field of sensitiveFields) {
            if (sanitized[field]) {
                sanitized[field] = '[REDACTED]';
            }
        }
        const jsonString = JSON.stringify(sanitized);
        if (jsonString.length > this.config.maxDataSize) {
            return { _truncated: true, _size: jsonString.length };
        }
        return sanitized;
    }
    generateTags(context) {
        const tags = [
            'job',
            `queue:${context.queueName}`,
            `job:${context.jobName}`,
            `status:${context.status}`,
        ];
        if (context.priority) {
            tags.push(`priority:${context.priority}`);
        }
        if (context.attempts > 1) {
            tags.push('retry');
        }
        if (context.delay) {
            tags.push('delayed');
        }
        if (context.workerId) {
            tags.push(`worker:${context.workerId}`);
        }
        return tags;
    }
    checkAlerts(context) {
        if (!this.config.enableRealTimeAlerts) {
            return;
        }
        const now = Date.now();
        const timeWindow = this.config.alertThresholds.timeWindow;
        if (this.currentMetrics.failureRate > this.config.alertThresholds.failureRate) {
            this.createAlert({
                type: 'failure_rate',
                severity: 'high',
                message: `Job failure rate exceeded threshold: ${this.currentMetrics.failureRate.toFixed(1)}%`,
                data: { failureRate: this.currentMetrics.failureRate, threshold: this.config.alertThresholds.failureRate },
                queueName: context.queueName,
            });
        }
        if (context.duration && context.duration > this.config.alertThresholds.avgExecutionTime) {
            this.createAlert({
                type: 'slow_jobs',
                severity: 'medium',
                message: `Slow job detected: ${context.jobName} took ${context.duration}ms`,
                data: { duration: context.duration, threshold: this.config.alertThresholds.avgExecutionTime },
                queueName: context.queueName,
                jobName: context.jobName,
            });
        }
        if (this.currentMetrics.stalledJobs >= this.config.alertThresholds.stalledJobs) {
            this.createAlert({
                type: 'stalled_jobs',
                severity: 'critical',
                message: `Stalled jobs detected: ${this.currentMetrics.stalledJobs} jobs`,
                data: { stalledJobs: this.currentMetrics.stalledJobs, threshold: this.config.alertThresholds.stalledJobs },
            });
        }
    }
    createAlert(alert) {
        const fullAlert = {
            id: `job_alert_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`,
            timestamp: new Date(),
            acknowledged: false,
            ...alert,
        };
        this.alertHistory.push(fullAlert);
        this.alertsSubject.next(fullAlert);
        this.logger.warn(`Job alert: ${fullAlert.message}`, fullAlert.data);
    }
    correlateJob(context) {
        if (context.traceId) {
            this.logger.debug(`Correlated job ${context.jobName} with trace ${context.traceId}`);
        }
    }
    startPeriodicProcessing() {
        (0, rxjs_1.interval)(30000)
            .pipe((0, operators_1.takeUntil)(this.destroy$))
            .subscribe(() => {
            this.updateTrends();
            this.cleanupOldData();
            this.metricsSubject.next({ ...this.currentMetrics });
        });
    }
    updateTrends() {
        const now = Date.now();
        const oneHourAgo = now - 60 * 60 * 1000;
        const oneDayAgo = now - 24 * 60 * 60 * 1000;
        const oneWeekAgo = now - 7 * 24 * 60 * 60 * 1000;
        this.currentMetrics.trends.lastHour = this.calculateTrend(oneHourAgo);
        this.currentMetrics.trends.lastDay = this.calculateTrend(oneDayAgo);
        this.currentMetrics.trends.lastWeek = this.calculateTrend(oneWeekAgo);
    }
    calculateTrend(since) {
        const jobs = this.jobHistory.filter(job => job.timestamp.getTime() > since);
        const completed = jobs.filter(job => job.status === JobStatus.COMPLETED);
        const failed = jobs.filter(job => job.status === JobStatus.FAILED);
        const execTimes = jobs.filter(job => job.duration).map(job => job.duration);
        return {
            totalJobs: jobs.length,
            completedJobs: completed.length,
            failedJobs: failed.length,
            averageExecutionTime: this.calculateAverage(execTimes),
            change: 0,
            changePercent: 0,
            peak: execTimes.length > 0 ? Math.max(...execTimes) : 0,
        };
    }
    cleanupOldData() {
        const now = Date.now();
        const retentionDate = new Date(now - this.config.retentionPeriod);
        this.jobHistory = this.jobHistory.filter(job => job.timestamp > retentionDate);
        this.alertHistory = this.alertHistory.filter(alert => alert.timestamp > retentionDate);
        if (this.executionTimes.length > 1000) {
            this.executionTimes = this.executionTimes.slice(-1000);
        }
        if (this.waitTimes.length > 1000) {
            this.waitTimes = this.waitTimes.slice(-1000);
        }
    }
    getMetrics() {
        return { ...this.currentMetrics };
    }
    getMetricsStream() {
        return this.metricsSubject.asObservable().pipe((0, operators_1.shareReplay)(1));
    }
    getAlertsStream() {
        return this.alertsSubject.asObservable();
    }
    getJobStream() {
        return this.jobContextSubject.asObservable();
    }
    getRecentJobs(limit = 100) {
        return this.jobHistory.slice(-limit).reverse();
    }
    getJobsByQueue(queueName, limit = 100) {
        return this.jobHistory
            .filter(job => job.queueName === queueName)
            .slice(-limit)
            .reverse();
    }
    getJobsByStatus(status, limit = 100) {
        return this.jobHistory
            .filter(job => job.status === status)
            .slice(-limit)
            .reverse();
    }
    getQueueHealth(queueName) {
        if (queueName) {
            return this.calculateQueueHealth(queueName);
        }
        const queues = new Set(this.jobHistory.map(job => job.queueName));
        return Array.from(queues).map(queue => this.calculateQueueHealth(queue));
    }
    calculateQueueHealth(queueName) {
        const queueJobs = this.jobHistory.filter(job => job.queueName === queueName);
        const totalJobs = queueJobs.length;
        const failedJobs = queueJobs.filter(job => job.status === JobStatus.FAILED).length;
        const stalledJobs = queueJobs.filter(job => job.status === JobStatus.STALLED).length;
        const execTimes = queueJobs.filter(job => job.duration).map(job => job.duration);
        const waitTimes = queueJobs.filter(job => job.performance?.queueWaitTime).map(job => job.performance.queueWaitTime);
        const failureRate = totalJobs > 0 ? (failedJobs / totalJobs) * 100 : 0;
        const avgWaitTime = this.calculateAverage(waitTimes);
        const processingRate = totalJobs > 0 ? (totalJobs - failedJobs) / totalJobs : 0;
        let score = 100;
        const issues = [];
        const recommendations = [];
        if (failureRate > 20) {
            score -= 30;
            issues.push('High failure rate');
            recommendations.push('Investigate failing jobs and fix underlying issues');
        }
        if (stalledJobs > 0) {
            score -= 25;
            issues.push('Stalled jobs detected');
            recommendations.push('Check worker health and restart if necessary');
        }
        if (avgWaitTime > 30000) {
            score -= 15;
            issues.push('High queue wait times');
            recommendations.push('Consider adding more workers or optimizing job processing');
        }
        let status = 'healthy';
        if (score < 60) {
            status = 'critical';
        }
        else if (score < 80) {
            status = 'warning';
        }
        return {
            queueName,
            status,
            score,
            issues,
            recommendations,
            metrics: {
                totalJobs,
                processingRate,
                failureRate,
                averageWaitTime: avgWaitTime,
                stalledJobs,
            },
        };
    }
    acknowledgeAlert(alertId) {
        const alert = this.alertHistory.find(a => a.id === alertId);
        if (!alert)
            return false;
        alert.acknowledged = true;
        return true;
    }
    getConfig() {
        return { ...this.config };
    }
};
exports.JobWatcherService = JobWatcherService;
exports.JobWatcherService = JobWatcherService = JobWatcherService_1 = __decorate([
    (0, common_1.Injectable)(),
    __param(1, (0, common_1.Inject)('JOB_WATCHER_CONFIG')),
    __metadata("design:paramtypes", [telescope_service_1.TelescopeService, Object])
], JobWatcherService);
//# sourceMappingURL=job-watcher.service.js.map