datapilot-cli
Version:
Enterprise-grade streaming multi-format data analysis with comprehensive statistical insights and intelligent relationship detection - supports CSV, JSON, Excel, TSV, Parquet - memory-efficient, cross-platform
321 lines • 12.1 kB
JavaScript
"use strict";
/**
* Worker Health Monitor
* Comprehensive health monitoring and recovery for worker threads
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.WorkerHealthMonitor = void 0;
const events_1 = require("events");
const perf_hooks_1 = require("perf_hooks");
const logger_1 = require("../utils/logger");
class WorkerHealthMonitor extends events_1.EventEmitter {
workers = new Map();
healthStatus = new Map();
healthCheckTimer;
options;
pendingHealthChecks = new Map();
constructor(options = {}) {
super();
this.options = {
heartbeatInterval: options.heartbeatInterval || 5000,
heartbeatTimeout: options.heartbeatTimeout || 10000,
maxErrorCount: options.maxErrorCount || 5,
maxResponseTime: options.maxResponseTime || 30000,
memoryThreshold: options.memoryThreshold || 256 * 1024 * 1024, // 256MB
enableAutoRecovery: options.enableAutoRecovery ?? true,
healthCheckRetries: options.healthCheckRetries || 3
};
}
/**
* Register a worker for health monitoring
*/
registerWorker(workerId, worker) {
this.workers.set(workerId, worker);
// Initialize health status
this.healthStatus.set(workerId, {
workerId,
isHealthy: true,
lastHeartbeat: Date.now(),
taskCount: 0,
memoryUsage: 0,
errorCount: 0,
responseTime: 0,
status: 'idle'
});
// Set up worker event listeners
this.setupWorkerListeners(workerId, worker);
// Start health monitoring if this is the first worker
if (this.workers.size === 1) {
this.startHealthMonitoring();
}
logger_1.logger.info(`Worker ${workerId} registered for health monitoring`);
}
/**
* Unregister a worker from health monitoring
*/
unregisterWorker(workerId) {
const worker = this.workers.get(workerId);
if (worker) {
this.cleanupWorker(workerId);
this.workers.delete(workerId);
this.healthStatus.delete(workerId);
// Stop monitoring if no workers left
if (this.workers.size === 0) {
this.stopHealthMonitoring();
}
logger_1.logger.info(`Worker ${workerId} unregistered from health monitoring`);
}
}
/**
* Get health status for a specific worker
*/
getWorkerHealth(workerId) {
return this.healthStatus.get(workerId) || null;
}
/**
* Get health status for all workers
*/
getAllWorkerHealth() {
return Array.from(this.healthStatus.values());
}
/**
* Check if a worker is healthy
*/
isWorkerHealthy(workerId) {
const status = this.healthStatus.get(workerId);
return status ? status.isHealthy && status.status !== 'failed' : false;
}
/**
* Get count of healthy workers
*/
getHealthyWorkerCount() {
return Array.from(this.healthStatus.values())
.filter(status => status.isHealthy && status.status !== 'failed').length;
}
/**
* Force health check for a specific worker
*/
async checkWorkerHealth(workerId) {
const worker = this.workers.get(workerId);
const status = this.healthStatus.get(workerId);
if (!worker || !status) {
return false;
}
try {
const startTime = perf_hooks_1.performance.now();
// Send health check message
const healthCheckPromise = new Promise((resolve) => {
const timeout = setTimeout(() => {
resolve(false);
}, this.options.heartbeatTimeout);
const messageHandler = (message) => {
if (message.type === 'health-check-response') {
clearTimeout(timeout);
worker.off('message', messageHandler);
resolve(true);
}
};
worker.on('message', messageHandler);
worker.postMessage({ type: 'health-check' });
});
const isResponsive = await healthCheckPromise;
const responseTime = perf_hooks_1.performance.now() - startTime;
// Update health status
status.lastHeartbeat = Date.now();
status.responseTime = responseTime;
if (!isResponsive) {
status.errorCount++;
status.isHealthy = false;
status.status = 'unresponsive';
this.emit('worker-unresponsive', { workerId, status });
if (this.options.enableAutoRecovery) {
await this.recoverWorker(workerId);
}
return false;
}
// Reset error count on successful health check
if (status.errorCount > 0) {
status.errorCount = Math.max(0, status.errorCount - 1);
}
status.isHealthy = true;
status.status = 'active';
return true;
}
catch (error) {
logger_1.logger.error(`Health check failed for worker ${workerId}: ${error.message}`);
status.isHealthy = false;
status.status = 'failed';
status.errorCount++;
this.emit('worker-health-check-failed', { workerId, error: error.message });
return false;
}
}
/**
* Attempt to recover an unhealthy worker
*/
async recoverWorker(workerId) {
const worker = this.workers.get(workerId);
const status = this.healthStatus.get(workerId);
if (!worker || !status) {
return false;
}
logger_1.logger.warn(`Attempting to recover worker ${workerId}`);
try {
// First, try to terminate the worker gracefully
status.status = 'terminating';
const terminationPromise = new Promise((resolve) => {
const timeout = setTimeout(() => {
// Force kill if graceful termination fails
worker.terminate();
resolve();
}, 5000);
worker.on('exit', () => {
clearTimeout(timeout);
resolve();
});
});
worker.postMessage({ type: 'terminate' });
await terminationPromise;
// Emit worker recovery event
this.emit('worker-recovery-attempted', { workerId, previousStatus: status });
return true;
}
catch (error) {
logger_1.logger.error(`Worker recovery failed for ${workerId}: ${error.message}`);
this.emit('worker-recovery-failed', { workerId, error: error.message });
return false;
}
}
/**
* Set up event listeners for a worker
*/
setupWorkerListeners(workerId, worker) {
worker.on('error', (error) => {
const status = this.healthStatus.get(workerId);
if (status) {
status.errorCount++;
status.isHealthy = false;
status.status = 'failed';
this.emit('worker-error', { workerId, error: error.message, status });
if (status.errorCount >= this.options.maxErrorCount && this.options.enableAutoRecovery) {
this.recoverWorker(workerId);
}
}
});
worker.on('exit', (code) => {
const status = this.healthStatus.get(workerId);
if (status) {
status.isHealthy = false;
status.status = 'failed';
this.emit('worker-exit', { workerId, exitCode: code, status });
}
});
worker.on('message', (message) => {
if (message.type === 'task-started') {
const status = this.healthStatus.get(workerId);
if (status) {
status.taskCount++;
status.status = 'busy';
status.lastHeartbeat = Date.now();
}
}
else if (message.type === 'task-completed') {
const status = this.healthStatus.get(workerId);
if (status) {
status.status = 'idle';
status.lastHeartbeat = Date.now();
}
}
else if (message.type === 'memory-usage') {
const status = this.healthStatus.get(workerId);
if (status) {
status.memoryUsage = message.memoryUsage;
// Check memory threshold
if (message.memoryUsage > this.options.memoryThreshold) {
this.emit('worker-memory-threshold-exceeded', { workerId, memoryUsage: message.memoryUsage });
}
}
}
});
}
/**
* Start periodic health monitoring
*/
startHealthMonitoring() {
if (this.healthCheckTimer) {
return;
}
this.healthCheckTimer = setInterval(async () => {
const healthCheckPromises = Array.from(this.workers.keys()).map(workerId => this.checkWorkerHealth(workerId).catch(error => {
logger_1.logger.error(`Health check error for worker ${workerId}: ${error.message}`);
return false;
}));
await Promise.allSettled(healthCheckPromises);
// Emit overall health status
const healthyCount = this.getHealthyWorkerCount();
const totalCount = this.workers.size;
this.emit('health-check-completed', {
healthyWorkers: healthyCount,
totalWorkers: totalCount,
healthPercentage: totalCount > 0 ? (healthyCount / totalCount) * 100 : 0
});
}, this.options.heartbeatInterval);
logger_1.logger.info('Worker health monitoring started');
}
/**
* Stop health monitoring
*/
stopHealthMonitoring() {
if (this.healthCheckTimer) {
clearInterval(this.healthCheckTimer);
this.healthCheckTimer = undefined;
}
// Clear pending health checks
for (const timeout of this.pendingHealthChecks.values()) {
clearTimeout(timeout);
}
this.pendingHealthChecks.clear();
logger_1.logger.info('Worker health monitoring stopped');
}
/**
* Clean up worker resources
*/
cleanupWorker(workerId) {
const pendingTimeout = this.pendingHealthChecks.get(workerId);
if (pendingTimeout) {
clearTimeout(pendingTimeout);
this.pendingHealthChecks.delete(workerId);
}
}
/**
* Get overall system health metrics
*/
getSystemHealthMetrics() {
const statuses = Array.from(this.healthStatus.values());
return {
totalWorkers: statuses.length,
healthyWorkers: statuses.filter(s => s.isHealthy).length,
failedWorkers: statuses.filter(s => s.status === 'failed').length,
averageResponseTime: statuses.length > 0
? statuses.reduce((sum, s) => sum + s.responseTime, 0) / statuses.length
: 0,
totalErrorCount: statuses.reduce((sum, s) => sum + s.errorCount, 0),
memoryUsage: statuses.reduce((sum, s) => sum + s.memoryUsage, 0)
};
}
/**
* Shutdown health monitor
*/
shutdown() {
this.stopHealthMonitoring();
for (const [workerId, worker] of this.workers) {
this.cleanupWorker(workerId);
}
this.workers.clear();
this.healthStatus.clear();
this.emit('shutdown');
logger_1.logger.info('Worker health monitor shutdown complete');
}
}
exports.WorkerHealthMonitor = WorkerHealthMonitor;
//# sourceMappingURL=worker-health-monitor.js.map