@nodedaemon/core/dist/core/ProcessOrchestrator.js

Production-ready Node.js process manager with zero external dependencies

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.ProcessOrchestrator = void 0;
const child_process_1 = require("child_process");
const cluster_1 = __importDefault(require("cluster"));
const os_1 = require("os");
const events_1 = require("events");
const helpers_1 = require("../utils/helpers");
const env_1 = require("../utils/env");
const constants_1 = require("../utils/constants");
class ProcessOrchestrator extends events_1.EventEmitter {
    processes = new Map();
    childProcesses = new Map();
    restartTimers = new Map();
    logger;
    isShuttingDown = false;
    constructor(logger) {
        super();
        this.logger = logger;
        this.setupClusterEventHandlers();
        this.setupProcessEventHandlers();
    }
    setupClusterEventHandlers() {
        if (cluster_1.default.isPrimary) {
            cluster_1.default.on('exit', (worker, code, signal) => {
                this.handleWorkerExit(worker, code, signal);
            });
            cluster_1.default.on('disconnect', (worker) => {
                this.logger.warn(`Cluster worker ${worker.id} disconnected`, { workerId: worker.id });
            });
        }
    }
    setupProcessEventHandlers() {
        process.on('SIGTERM', () => this.gracefulShutdown());
        process.on('SIGINT', () => this.gracefulShutdown());
        process.on('SIGHUP', () => this.reloadAllProcesses());
    }
    async startProcess(config) {
        if (this.isShuttingDown) {
            throw new Error('Cannot start process during shutdown');
        }
        (0, helpers_1.validateProcessConfig)(config);
        const processId = (0, helpers_1.generateId)();
        const processName = config.name || (0, helpers_1.sanitizeProcessName)(config.script);
        const instanceCount = this.resolveInstanceCount(config.instances);
        const strategy = this.determineStrategy(config);
        // Load environment from .env files
        let envConfig = {};
        const envFilePath = (0, env_1.findEnvFile)(config.script, config.envFile);
        if (envFilePath) {
            envConfig = (0, env_1.loadEnvFile)(envFilePath);
            this.logger.info(`Loaded environment from ${envFilePath}`, { processName });
        }
        // Merge with provided env vars (provided vars take precedence)
        const finalEnv = (0, env_1.mergeEnvConfigs)(envConfig, config.env || {});
        const processInfo = {
            id: processId,
            name: processName,
            script: config.script,
            status: 'starting',
            restarts: 0,
            instances: [],
            config: { ...constants_1.DEFAULT_CONFIG, ...config, env: finalEnv },
            createdAt: Date.now(),
            updatedAt: Date.now()
        };
        this.processes.set(processId, processInfo);
        this.logger.info(`Starting process: ${processName}`, { processId, strategy, instances: instanceCount });
        try {
            if (strategy === 'cluster' && instanceCount > 1) {
                await this.startClusterProcess(processInfo, instanceCount);
            }
            else {
                await this.startSingleProcess(processInfo, strategy);
            }
            processInfo.status = 'running';
            processInfo.updatedAt = Date.now();
            this.emit('processStarted', processInfo);
            return processId;
        }
        catch (error) {
            processInfo.status = 'errored';
            processInfo.updatedAt = Date.now();
            this.logger.error(`Failed to start process ${processName}`, { error: error.message, processId });
            throw error;
        }
    }
    resolveInstanceCount(instances) {
        if (instances === 'max') {
            return (0, os_1.cpus)().length;
        }
        return (typeof instances === 'number') ? instances : constants_1.DEFAULT_CONFIG.instances;
    }
    determineStrategy(config) {
        if (config.instances && (config.instances === 'max' || config.instances > 1)) {
            return 'cluster';
        }
        if (config.script.endsWith('.js') || config.script.endsWith('.mjs')) {
            return 'fork';
        }
        return 'spawn';
    }
    async startClusterProcess(processInfo, instanceCount) {
        if (!cluster_1.default.isPrimary) {
            throw new Error('Cluster processes can only be started from primary process');
        }
        const promises = [];
        for (let i = 0; i < instanceCount; i++) {
            promises.push(this.startClusterInstance(processInfo, i));
        }
        await Promise.all(promises);
    }
    startClusterInstance(processInfo, instanceIndex) {
        return new Promise((resolve, reject) => {
            const instanceId = (0, helpers_1.generateId)();
            const instance = {
                id: instanceId,
                status: 'starting',
                restarts: 0
            };
            processInfo.instances.push(instance);
            cluster_1.default.setupPrimary({
                exec: processInfo.script,
                args: processInfo.config.args || [],
                cwd: processInfo.config.cwd
            });
            const worker = cluster_1.default.fork({ ...process.env, ...processInfo.config.env });
            this.childProcesses.set(instanceId, worker);
            worker.on('online', () => {
                instance.pid = worker.process.pid;
                instance.status = 'running';
                instance.uptime = Date.now();
                this.logger.info(`Cluster instance ${instanceIndex} started`, {
                    processId: processInfo.id,
                    instanceId,
                    pid: worker.process.pid
                });
                resolve();
            });
            worker.on('exit', (code, signal) => {
                this.handleInstanceExit(processInfo, instance, code, signal);
            });
            worker.on('error', (error) => {
                this.logger.error(`Cluster instance error`, {
                    processId: processInfo.id,
                    instanceId,
                    error: error.message
                });
                reject(error);
            });
            setTimeout(() => {
                if (instance.status === 'starting') {
                    reject(new Error(`Cluster instance ${instanceIndex} failed to start within timeout`));
                }
            }, 30000);
        });
    }
    async startSingleProcess(processInfo, strategy) {
        const instanceId = (0, helpers_1.generateId)();
        const instance = {
            id: instanceId,
            status: 'starting',
            restarts: 0
        };
        processInfo.instances.push(instance);
        let childProcess;
        if (strategy === 'fork') {
            childProcess = (0, child_process_1.fork)(processInfo.script, processInfo.config.args || [], {
                cwd: processInfo.config.cwd,
                env: { ...process.env, ...processInfo.config.env },
                silent: false
            });
        }
        else {
            const interpreter = processInfo.config.interpreter || 'node';
            childProcess = (0, child_process_1.spawn)(interpreter, [processInfo.script, ...(processInfo.config.args || [])], {
                cwd: processInfo.config.cwd,
                env: { ...process.env, ...processInfo.config.env },
                stdio: ['inherit', 'inherit', 'inherit']
            });
        }
        this.childProcesses.set(instanceId, childProcess);
        return new Promise((resolve, reject) => {
            childProcess.on('spawn', () => {
                instance.pid = childProcess.pid;
                instance.status = 'running';
                instance.uptime = Date.now();
                this.logger.info(`Process started`, {
                    processId: processInfo.id,
                    instanceId,
                    pid: childProcess.pid,
                    strategy
                });
                resolve();
            });
            childProcess.on('exit', (code, signal) => {
                this.handleInstanceExit(processInfo, instance, code, signal);
            });
            childProcess.on('error', (error) => {
                this.logger.error(`Process error`, {
                    processId: processInfo.id,
                    instanceId,
                    error: error.message
                });
                reject(error);
            });
            setTimeout(() => {
                if (instance.status === 'starting') {
                    reject(new Error('Process failed to start within timeout'));
                }
            }, 30000);
        });
    }
    handleWorkerExit(worker, code, signal) {
        const instanceId = this.findInstanceByPid(worker.process.pid);
        if (instanceId) {
            const processInfo = this.findProcessByInstanceId(instanceId);
            const instance = processInfo?.instances.find(i => i.id === instanceId);
            if (processInfo && instance) {
                this.handleInstanceExit(processInfo, instance, code, signal);
            }
        }
    }
    handleInstanceExit(processInfo, instance, code, signal) {
        instance.status = code === 0 ? 'stopped' : 'crashed';
        this.childProcesses.delete(instance.id);
        // Calculate uptime
        const uptime = instance.uptime ? Date.now() - instance.uptime : 0;
        // Reset restart counter if process ran successfully for minimum uptime
        if (uptime >= (processInfo.config.minUptime || constants_1.DEFAULT_CONFIG.minUptime)) {
            instance.restarts = 0;
            this.logger.info(`Process ran successfully for ${(0, helpers_1.formatUptime)(uptime)}, resetting restart counter`, {
                processId: processInfo.id,
                instanceId: instance.id
            });
        }
        this.logger.info(`Process instance exited`, {
            processId: processInfo.id,
            instanceId: instance.id,
            pid: instance.pid,
            code,
            signal,
            uptime: (0, helpers_1.formatUptime)(uptime),
            restarts: instance.restarts
        });
        if (processInfo.status !== 'stopping' && !this.isShuttingDown) {
            if (instance.restarts < processInfo.config.maxRestarts) {
                this.scheduleRestart(processInfo, instance);
            }
            else {
                this.logger.error(`Process instance reached max restarts, stopping permanently`, {
                    processId: processInfo.id,
                    instanceId: instance.id,
                    maxRestarts: processInfo.config.maxRestarts,
                    totalRestarts: instance.restarts
                });
                instance.status = 'errored';
                processInfo.status = 'errored';
                // Emit event for max restarts reached
                this.emit('maxRestartsReached', processInfo, instance);
            }
        }
        this.updateProcessStatus(processInfo);
        this.emit('instanceExit', processInfo, instance, code, signal);
    }
    scheduleRestart(processInfo, instance) {
        const delay = (0, helpers_1.calculateExponentialBackoff)(instance.restarts, processInfo.config.restartDelay, processInfo.config.maxRestartDelay);
        this.logger.warn(`Process crashed too quickly, scheduling restart in ${delay}ms`, {
            processId: processInfo.id,
            instanceId: instance.id,
            attempt: instance.restarts + 1,
            maxRestarts: processInfo.config.maxRestarts,
            backoffDelay: `${delay}ms`
        });
        const timer = setTimeout(() => {
            this.restartInstance(processInfo, instance);
        }, delay);
        this.restartTimers.set(instance.id, timer);
    }
    async restartInstance(processInfo, instance) {
        try {
            this.restartTimers.delete(instance.id);
            instance.restarts++;
            instance.lastRestart = Date.now();
            instance.status = 'starting';
            this.logger.info(`Restarting process instance`, {
                processId: processInfo.id,
                instanceId: instance.id,
                attempt: instance.restarts
            });
            const strategy = this.determineStrategy(processInfo.config);
            if (strategy === 'cluster') {
                await this.startClusterInstance(processInfo, 0);
            }
            else {
                await this.startSingleProcess(processInfo, strategy);
            }
            this.emit('instanceRestarted', processInfo, instance);
        }
        catch (error) {
            this.logger.error(`Failed to restart process instance`, {
                processId: processInfo.id,
                instanceId: instance.id,
                error: error.message
            });
            instance.status = 'errored';
            this.updateProcessStatus(processInfo);
        }
    }
    updateProcessStatus(processInfo) {
        const runningInstances = processInfo.instances.filter(i => i.status === 'running');
        const stoppedInstances = processInfo.instances.filter(i => i.status === 'stopped');
        const erroredInstances = processInfo.instances.filter(i => i.status === 'errored' || i.status === 'crashed');
        if (runningInstances.length > 0) {
            processInfo.status = 'running';
        }
        else if (erroredInstances.length > 0) {
            processInfo.status = 'errored';
        }
        else if (stoppedInstances.length === processInfo.instances.length) {
            processInfo.status = 'stopped';
        }
        else {
            processInfo.status = 'starting';
        }
        processInfo.updatedAt = Date.now();
        processInfo.restarts = processInfo.instances.reduce((sum, instance) => sum + instance.restarts, 0);
    }
    async stopProcess(processId, force = false) {
        const processInfo = this.processes.get(processId);
        if (!processInfo) {
            throw new Error(`Process not found: ${processId}`);
        }
        processInfo.status = 'stopping';
        processInfo.updatedAt = Date.now();
        this.logger.info(`Stopping process: ${processInfo.name}`, { processId, force });
        const stopPromises = processInfo.instances.map(instance => this.stopInstance(processInfo, instance, force));
        await Promise.all(stopPromises);
        processInfo.status = 'stopped';
        processInfo.updatedAt = Date.now();
        this.emit('processStopped', processInfo);
    }
    async stopInstance(processInfo, instance, force) {
        const childProcess = this.childProcesses.get(instance.id);
        if (!childProcess)
            return;
        return new Promise((resolve) => {
            const cleanup = () => {
                this.childProcesses.delete(instance.id);
                instance.status = 'stopped';
                resolve();
            };
            if (force) {
                childProcess.kill('SIGKILL');
                setTimeout(cleanup, 1000);
                return;
            }
            let killed = false;
            const killTimer = setTimeout(() => {
                if (!killed) {
                    killed = true;
                    childProcess.kill('SIGKILL');
                    setTimeout(cleanup, constants_1.FORCE_KILL_TIMEOUT);
                }
            }, constants_1.GRACEFUL_SHUTDOWN_TIMEOUT);
            childProcess.once('exit', () => {
                if (!killed) {
                    killed = true;
                    clearTimeout(killTimer);
                    cleanup();
                }
            });
            childProcess.kill('SIGTERM');
        });
    }
    async restartProcess(processId, graceful = false) {
        const processInfo = this.processes.get(processId);
        if (!processInfo) {
            throw new Error(`Process not found: ${processId}`);
        }
        const strategy = this.determineStrategy(processInfo.config);
        const instanceCount = this.resolveInstanceCount(processInfo.config.instances);
        // For cluster mode with multiple instances, do graceful reload
        if (graceful && strategy === 'cluster' && instanceCount > 1) {
            await this.gracefulReload(processInfo);
        }
        else {
            // Standard restart
            await this.stopProcess(processId);
            const updatedProcessInfo = this.processes.get(processId);
            if (!updatedProcessInfo) {
                throw new Error(`Process not found after stop: ${processId}`);
            }
            updatedProcessInfo.instances = [];
            updatedProcessInfo.status = 'starting';
            updatedProcessInfo.updatedAt = Date.now();
            if (strategy === 'cluster' && instanceCount > 1) {
                await this.startClusterProcess(updatedProcessInfo, instanceCount);
            }
            else {
                await this.startSingleProcess(updatedProcessInfo, strategy);
            }
            updatedProcessInfo.status = 'running';
            updatedProcessInfo.updatedAt = Date.now();
            this.emit('processRestarted', updatedProcessInfo);
        }
    }
    async gracefulReload(processInfo) {
        this.logger.info(`Starting graceful reload for ${processInfo.name}`, { processId: processInfo.id });
        const instanceCount = this.resolveInstanceCount(processInfo.config.instances);
        const oldInstances = [...processInfo.instances];
        const newInstances = [];
        processInfo.status = 'reloading';
        processInfo.updatedAt = Date.now();
        // Start new instances first
        for (let i = 0; i < instanceCount; i++) {
            const instanceId = (0, helpers_1.generateId)();
            const instance = {
                id: instanceId,
                status: 'starting',
                restarts: 0
            };
            newInstances.push(instance);
            processInfo.instances.push(instance);
            try {
                await this.startClusterInstance(processInfo, i);
                // Wait a bit for the new instance to be ready
                await new Promise(resolve => setTimeout(resolve, 2000));
            }
            catch (error) {
                this.logger.error(`Failed to start new instance during graceful reload`, {
                    processId: processInfo.id,
                    instanceId,
                    error: error.message
                });
            }
        }
        // Now stop old instances one by one
        for (const oldInstance of oldInstances) {
            await this.stopInstance(processInfo, oldInstance, false);
            // Remove old instance from the list
            const index = processInfo.instances.findIndex(i => i.id === oldInstance.id);
            if (index >= 0) {
                processInfo.instances.splice(index, 1);
            }
            // Wait a bit between stopping instances
            await new Promise(resolve => setTimeout(resolve, 1000));
        }
        processInfo.status = 'running';
        processInfo.updatedAt = Date.now();
        this.logger.info(`Graceful reload completed for ${processInfo.name}`, {
            processId: processInfo.id,
            oldInstances: oldInstances.length,
            newInstances: newInstances.length
        });
        this.emit('processReloaded', processInfo);
    }
    deleteProcess(processId) {
        const processInfo = this.processes.get(processId);
        if (!processInfo) {
            throw new Error(`Process not found: ${processId}`);
        }
        if (processInfo.status === 'running' || processInfo.status === 'starting') {
            throw new Error('Cannot delete running process. Stop it first.');
        }
        this.processes.delete(processId);
        this.logger.info(`Process deleted: ${processInfo.name}`, { processId });
        this.emit('processDeleted', processInfo);
    }
    getProcesses() {
        return Array.from(this.processes.values());
    }
    getProcess(processId) {
        return this.processes.get(processId);
    }
    getProcessByName(name) {
        return Array.from(this.processes.values()).find(p => p.name === name);
    }
    findInstanceByPid(pid) {
        for (const processInfo of this.processes.values()) {
            for (const instance of processInfo.instances) {
                if (instance.pid === pid) {
                    return instance.id;
                }
            }
        }
        return undefined;
    }
    findProcessByInstanceId(instanceId) {
        for (const processInfo of this.processes.values()) {
            if (processInfo.instances.some(i => i.id === instanceId)) {
                return processInfo;
            }
        }
        return undefined;
    }
    async reloadAllProcesses() {
        const runningProcesses = Array.from(this.processes.values())
            .filter(p => p.status === 'running');
        this.logger.info(`Reloading ${runningProcesses.length} processes`);
        const reloadPromises = runningProcesses.map(async (processInfo) => {
            try {
                await this.restartProcess(processInfo.id);
            }
            catch (error) {
                this.logger.error(`Failed to reload process ${processInfo.name}`, {
                    processId: processInfo.id,
                    error: error.message
                });
            }
        });
        await Promise.all(reloadPromises);
    }
    async gracefulShutdown() {
        if (this.isShuttingDown)
            return;
        this.isShuttingDown = true;
        this.logger.info('Starting graceful shutdown');
        const runningProcesses = Array.from(this.processes.values())
            .filter(p => p.status === 'running' || p.status === 'starting');
        if (runningProcesses.length === 0) {
            this.logger.info('No running processes to stop');
            return;
        }
        this.logger.info(`Stopping ${runningProcesses.length} processes`);
        const stopPromises = runningProcesses.map(async (processInfo) => {
            try {
                await this.stopProcess(processInfo.id);
            }
            catch (error) {
                this.logger.error(`Failed to stop process ${processInfo.name}`, {
                    processId: processInfo.id,
                    error: error.message
                });
            }
        });
        await Promise.all(stopPromises);
        this.restartTimers.forEach(timer => clearTimeout(timer));
        this.restartTimers.clear();
        this.logger.info('Graceful shutdown completed');
    }
    getHealthCheck() {
        const results = [];
        for (const processInfo of this.processes.values()) {
            for (const instance of processInfo.instances) {
                if (instance.status === 'running' && instance.pid) {
                    try {
                        const memUsage = process.memoryUsage();
                        const uptime = instance.uptime ? Date.now() - instance.uptime : 0;
                        results.push({
                            processId: processInfo.id,
                            memory: memUsage.rss,
                            cpu: 0, // TODO: Implement CPU monitoring
                            uptime,
                            healthy: instance.status === 'running'
                        });
                    }
                    catch (error) {
                        results.push({
                            processId: processInfo.id,
                            memory: 0,
                            cpu: 0,
                            uptime: 0,
                            healthy: false,
                            issues: [`Health check failed: ${error.message}`]
                        });
                    }
                }
            }
        }
        return results;
    }
}
exports.ProcessOrchestrator = ProcessOrchestrator;
//# sourceMappingURL=ProcessOrchestrator.js.map