fortify2-js
Version:
MOST POWERFUL JavaScript Security Library! Military-grade cryptography + 19 enhanced object methods + quantum-resistant algorithms + perfect TypeScript support. More powerful than Lodash with built-in security.
1,012 lines (1,006 loc) • 37.9 kB
JavaScript
'use strict';
var os = require('os');
var events = require('events');
var WorkerManager = require('./components/WorkerManager.js');
var HealthMonitor = require('./components/HealthMonitor.js');
var LoadBalancer = require('./components/LoadBalancer.js');
var IPCManager = require('./components/IPCManager.js');
var MetricsCollector = require('./components/MetricsCollector.js');
var AutoScaler = require('./components/AutoScaler.js');
var ClusterPersistenceManager = require('./components/ClusterPersistenceManager.js');
var errorHandler = require('../../../utils/errorHandler.js');
var Cluster_config = require('../server/const/Cluster.config.js');
var Logger = require('../server/utils/Logger.js');
function _interopNamespaceDefault(e) {
var n = Object.create(null);
if (e) {
Object.keys(e).forEach(function (k) {
if (k !== 'default') {
var d = Object.getOwnPropertyDescriptor(e, k);
Object.defineProperty(n, k, d.get ? d : {
enumerable: true,
get: function () { return e[k]; }
});
}
});
}
n.default = e;
return Object.freeze(n);
}
var os__namespace = /*#__PURE__*/_interopNamespaceDefault(os);
/**
* FortifyJS Cluster Manager
* cluster management for Express applications with advanced monitoring
*/
/**
* cluster manager with comprehensive monitoring and auto-scaling
*/
class ClusterManager extends events.EventEmitter {
constructor(config = {}) {
super();
this.state = "initializing";
this.startTime = new Date();
this.isShuttingDown = false;
// Apply intelligent defaults
this.config = this.applyDefaults(config);
// Initialize components with fortified security
this.errorLogger = new errorHandler.SecurityErrorLogger();
this.workerManager = new WorkerManager.WorkerManager(this.config, this.errorLogger);
this.healthMonitor = new HealthMonitor.HealthMonitor(this.config, this.errorLogger);
this.loadBalancer = new LoadBalancer.LoadBalancer(this.config);
this.ipcManager = new IPCManager.IPCManager(this.config, this.errorLogger);
this.metricsCollector = new MetricsCollector.MetricsCollector(this.config);
this.autoScaler = new AutoScaler.AutoScaler(this.config, this.errorLogger);
// Initialize persistence manager if configured
if (this.config.persistence?.enabled && this.config.persistence.type) {
this.persistenceManager = new ClusterPersistenceManager.ClusterPersistenceManager(this.config.persistence);
}
// Setup component integrations
this.setupComponentIntegrations();
// Setup event forwarding
this.setupEventForwarding();
// Setup error handling
this.setupErrorHandling();
}
/**
* Apply intelligent defaults to cluster configuration
*/
applyDefaults(config) {
const defaults = Cluster_config.DEFAULT_CLUSTER_CONFIGS;
return this.mergeDeep(defaults, config);
}
/**
* Deep merge configuration objects
*/
mergeDeep(target, source) {
const result = { ...target };
for (const key in source) {
if (source[key] &&
typeof source[key] === "object" &&
!Array.isArray(source[key])) {
result[key] = this.mergeDeep(target[key] || {}, source[key]);
}
else {
result[key] = source[key];
}
}
return result;
}
/**
* Setup component integrations for cross-component communication
*/
setupComponentIntegrations() {
// Integrate IPC Manager with Worker Manager
this.ipcManager.setWorkerManager(this.workerManager);
// Setup auto-scaler with worker manager and metrics collector integration
this.autoScaler.setWorkerManager(this.workerManager);
this.autoScaler.setMetricsCollector(this.metricsCollector);
// Setup metrics collector with worker manager integration
this.metricsCollector.setWorkerManager(this.workerManager);
// Setup health monitor with worker manager integration
this.healthMonitor.setWorkerManager(this.workerManager);
// Setup load balancer with IPC manager integration
this.loadBalancer.setIPCManager(this.ipcManager);
}
/**
* Setup event forwarding between components
*/
setupEventForwarding() {
// Forward worker manager events
this.workerManager.on("worker:started", (workerId, pid) => {
this.emit("worker:started", workerId, pid);
});
this.workerManager.on("worker:died", (workerId, code, signal) => {
this.emit("worker:died", workerId, code, signal);
});
this.workerManager.on("worker:restarted", (workerId, reason) => {
this.emit("worker:restarted", workerId, reason);
});
// Forward health monitor events
this.healthMonitor.on("worker:health:warning", (workerId, metrics) => {
this.emit("worker:health:warning", workerId, metrics);
});
this.healthMonitor.on("worker:health:critical", (workerId, metrics) => {
this.emit("worker:health:critical", workerId, metrics);
});
// Forward auto-scaler events
this.autoScaler.on("cluster:scaled", (action, newWorkerCount) => {
this.emit("cluster:scaled", action, newWorkerCount);
});
this.autoScaler.on("scaling:triggered", (reason, currentWorkers, targetWorkers) => {
this.emit("scaling:triggered", reason, currentWorkers, targetWorkers);
});
// Forward load balancer events
this.loadBalancer.on("loadbalancer:updated", (strategy, weights) => {
this.emit("loadbalancer:updated", strategy, weights);
});
// Forward IPC events
this.ipcManager.on("ipc:message", (from, to, message) => {
this.emit("ipc:message", from, to, message);
});
this.ipcManager.on("ipc:broadcast", (from, message) => {
this.emit("ipc:broadcast", from, message);
});
// Forward metrics events
this.metricsCollector.on("metrics:updated", (metrics) => {
this.updateClusterState(metrics);
});
}
/**
* Setup comprehensive error handling
*/
setupErrorHandling() {
// Handle uncaught exceptions
process.on("uncaughtException", (error) => {
const securityError = errorHandler.createSecurityError(`Uncaught exception in cluster manager: ${error.message}`, errorHandler.ErrorType.INTERNAL, errorHandler.ErrorSeverity.CRITICAL, "CLUSTER_UNCAUGHT_EXCEPTION", { operation: "cluster_manager" });
this.errorLogger.logError(securityError);
if (this.config.errorHandling?.uncaughtException === "restart") {
this.handleCriticalError("uncaught_exception");
}
});
// Handle unhandled rejections
process.on("unhandledRejection", (reason) => {
const securityError = errorHandler.createSecurityError(`Unhandled rejection in cluster manager: ${reason}`, errorHandler.ErrorType.INTERNAL, errorHandler.ErrorSeverity.CRITICAL, "CLUSTER_UNHANDLED_REJECTION", { operation: "cluster_manager" });
this.errorLogger.logError(securityError);
if (this.config.errorHandling?.unhandledRejection === "restart") {
this.handleCriticalError("unhandled_rejection");
}
});
}
/**
* Update cluster state based on metrics
*/
updateClusterState(metrics) {
// Update state based on cluster health
if (metrics.healthOverall.status === "critical") {
this.state = "degraded";
}
else if (metrics.activeWorkers === 0) {
this.state = "stopped";
}
else if (this.state === "initializing" || this.state === "starting") {
this.state = "running";
}
}
/**
* Handle critical errors with recovery strategies
*/
async handleCriticalError(errorType) {
Logger.logger.error("cluster", `Critical error detected: ${errorType}`);
try {
// Attempt graceful recovery
await this.restart();
}
catch (error) {
Logger.logger.error("cluster", "Failed to recover from critical error:", error);
process.exit(1);
}
}
// ===== CORE CLUSTER MANAGEMENT METHODS =====
/**
* Start the cluster with intelligent worker management
*/
async start() {
if (this.state === "running") {
Logger.logger.debug("cluster", "Cluster is already running");
return;
}
this.state = "starting";
try {
// logger.debug( "cluster","Starting cluster...");
// Determine if we're in master or worker process
const clusterModule = require("cluster");
if (clusterModule.isMaster) {
Logger.logger.debug("cluster", "Starting as cluster master process");
// Start monitoring components
this.healthMonitor.startMonitoring();
this.metricsCollector.startCollection();
// Start workers
await this.workerManager.startWorkers();
// Update auto-scaler with initial worker count
const workerCount = this.workerManager.getActiveWorkers().length;
this.autoScaler.updateWorkerCount(workerCount);
// Setup graceful shutdown
this.setupGracefulShutdown();
Logger.logger.debug("cluster", `Cluster master started with ${workerCount} workers`);
}
else {
Logger.logger.debug("cluster", `Worker ${process.pid} started`);
// Worker-specific initialization
this.setupWorkerProcess();
}
this.state = "running";
// logger.debug( "cluster","FortifyJS cluster started successfully");
}
catch (error) {
this.state = "error";
const securityError = errorHandler.createSecurityError(`Cluster start failed: ${error.message}`, errorHandler.ErrorType.INTERNAL, errorHandler.ErrorSeverity.CRITICAL, "CLUSTER_START_ERROR", { operation: "start" });
this.errorLogger.logError(securityError);
throw error;
}
}
/**
* Setup worker process initialization
*/
setupWorkerProcess() {
// Set worker environment
process.env.WORKER_ID =
process.env.WORKER_ID || process.pid?.toString() || "unknown";
process.env.NODE_ENV = "worker";
// Setup worker-specific error handling
process.on("uncaughtException", (error) => {
Logger.logger.error("cluster", "Worker uncaught exception:", error);
process.exit(1);
});
process.on("unhandledRejection", (reason, promise) => {
Logger.logger.error("cluster", "Worker unhandled rejection at:", promise, "reason:", reason);
process.exit(1);
});
// Send ready signal to master
if (process.send) {
process.send({
type: "worker_ready",
workerId: process.env.WORKER_ID,
pid: process.pid,
});
}
}
/**
* Setup graceful shutdown handling
*/
setupGracefulShutdown() {
const gracefulShutdown = async (signal) => {
Logger.logger.debug("cluster", `Received ${signal}, starting graceful shutdown...`);
try {
await this.stop();
process.exit(0);
}
catch (error) {
Logger.logger.error("cluster", "Error during graceful shutdown:", error);
process.exit(1);
}
};
process.on("SIGTERM", () => gracefulShutdown("SIGTERM"));
process.on("SIGINT", () => gracefulShutdown("SIGINT"));
}
/**
* Stop the cluster gracefully or forcefully
*/
async stop(graceful = true) {
if (this.state === "stopped" || this.state === "stopping") {
Logger.logger.debug("cluster", "Cluster is already stopped or stopping");
return;
}
this.state = "stopping";
try {
Logger.logger.debug("cluster", `Stopping cluster ${graceful ? "gracefully" : "forcefully"}...`);
// Stop monitoring
this.healthMonitor.stopMonitoring();
this.metricsCollector.stopCollection();
this.autoScaler.stopScaling();
// Stop all workers
await this.workerManager.stopAllWorkers(graceful);
// Close persistence manager
await this.closePersistenceManager();
this.state = "stopped";
Logger.logger.debug("cluster", "Cluster stopped successfully");
}
catch (error) {
this.state = "error";
const securityError = errorHandler.createSecurityError(`Cluster stop failed: ${error.message}`, errorHandler.ErrorType.INTERNAL, errorHandler.ErrorSeverity.HIGH, "CLUSTER_STOP_ERROR", { operation: "stop" });
this.errorLogger.logError(securityError);
throw error;
}
}
/**
* Restart the cluster or specific worker
*/
async restart(workerId) {
if (workerId) {
// Restart specific worker using WorkerManager
try {
Logger.logger.debug("cluster", `Restarting worker ${workerId}...`);
// Use WorkerManager to restart the specific worker
const newWorkerId = await this.replaceWorker(workerId);
this.emit("worker:restarted", newWorkerId, "manual_restart");
Logger.logger.debug("cluster", `Worker ${workerId} restarted as ${newWorkerId}`);
}
catch (error) {
const securityError = errorHandler.createSecurityError(`Failed to restart worker ${workerId}: ${error.message}`, errorHandler.ErrorType.INTERNAL, errorHandler.ErrorSeverity.HIGH, "WORKER_RESTART_ERROR", { operation: "restart_worker" });
this.errorLogger.logError(securityError);
throw error;
}
}
else {
// Restart entire cluster
try {
Logger.logger.debug("cluster", "Restarting entire cluster...");
await this.stop(true);
await new Promise((resolve) => setTimeout(resolve, 2000)); // Brief pause
await this.start();
Logger.logger.debug("cluster", "Cluster restarted successfully");
}
catch (error) {
const securityError = errorHandler.createSecurityError(`Failed to restart cluster: ${error.message}`, errorHandler.ErrorType.INTERNAL, errorHandler.ErrorSeverity.CRITICAL, "CLUSTER_RESTART_ERROR", { operation: "restart_cluster" });
this.errorLogger.logError(securityError);
throw error;
}
}
}
/**
* Pause cluster operations
*/
async pause() {
if (this.state !== "running") {
throw new Error("Cannot pause cluster: not running");
}
this.state = "paused";
// Pause monitoring
this.healthMonitor.stopMonitoring();
this.autoScaler.disable();
Logger.logger.debug("cluster", "Cluster paused");
}
/**
* Resume cluster operations
*/
async resume() {
if (this.state !== "paused") {
throw new Error("Cannot resume cluster: not paused");
}
this.state = "running";
// Resume monitoring
this.healthMonitor.startMonitoring();
this.autoScaler.enable();
Logger.logger.debug("cluster", "Cluster resumed");
}
// ===== WORKER MANAGEMENT METHODS =====
/**
* Add new worker to the cluster
*/
async addWorker() {
const workers = this.workerManager.getActiveWorkers();
const maxWorkers = this.config.autoScaling?.maxWorkers || 8;
if (workers.length >= maxWorkers) {
throw new Error("Cannot add worker: maximum worker limit reached");
}
try {
// Use WorkerManager to actually start a worker
const workerId = await this.workerManager.startSingleWorker();
// Update auto-scaler with new worker count
this.autoScaler.updateWorkerCount(workers.length + 1);
this.emit("worker:started", workerId, 0);
return workerId;
}
catch (error) {
const securityError = errorHandler.createSecurityError(`Failed to add worker: ${error.message}`, errorHandler.ErrorType.INTERNAL, errorHandler.ErrorSeverity.HIGH, "WORKER_ADD_ERROR", { operation: "add_worker" });
this.errorLogger.logError(securityError);
throw error;
}
}
/**
* Remove worker from the cluster
*/
async removeWorker(workerId, graceful = true) {
const workers = this.workerManager.getActiveWorkers();
const minWorkers = this.config.autoScaling?.minWorkers || 1;
if (workers.length <= minWorkers) {
throw new Error("Cannot remove worker: minimum worker limit reached");
}
try {
// Use WorkerManager to actually stop the worker
await this.workerManager.stopSingleWorker(workerId, graceful);
// Update auto-scaler with new worker count
this.autoScaler.updateWorkerCount(workers.length - 1);
this.emit("worker:died", workerId, 0, graceful ? "SIGTERM" : "SIGKILL");
}
catch (error) {
const securityError = errorHandler.createSecurityError(`Failed to remove worker ${workerId}: ${error.message}`, errorHandler.ErrorType.INTERNAL, errorHandler.ErrorSeverity.HIGH, "WORKER_REMOVE_ERROR", { operation: "remove_worker" });
this.errorLogger.logError(securityError);
throw error;
}
}
/**
* Replace worker with a new one
*/
async replaceWorker(workerId) {
Logger.logger.debug("cluster", `Replacing worker ${workerId}...`);
// Start new worker first
const newWorkerId = await this.addWorker();
// Wait for new worker to be ready
await new Promise((resolve) => setTimeout(resolve, 2000));
// Remove old worker
await this.removeWorker(workerId, true);
return newWorkerId;
}
/**
* Get worker by ID
*/
getWorker(workerId) {
return this.workerManager.getWorker(workerId);
}
/**
* Get all workers
*/
getAllWorkers() {
return this.workerManager.getActiveWorkers();
}
/**
* Get active workers
*/
getActiveWorkers() {
return this.workerManager
.getActiveWorkers()
.filter((w) => w.health.status === "healthy" ||
w.health.status === "warning");
}
/**
* Get unhealthy workers
*/
getUnhealthyWorkers() {
return this.workerManager
.getActiveWorkers()
.filter((w) => w.health.status === "critical" || w.health.status === "dead");
}
// ===== HEALTH MONITORING METHODS =====
/**
* Check health of specific worker or all workers
*/
async checkHealth(workerId) {
if (workerId) {
return await this.healthMonitor.checkWorkerHealth(workerId);
}
else {
await this.healthMonitor.performHealthChecks();
const healthStatus = this.healthMonitor.getHealthStatus();
return Object.values(healthStatus).every((healthy) => healthy);
}
}
/**
* Get health status of all workers
*/
async getHealthStatus() {
return this.healthMonitor.getHealthStatus();
}
/**
* Run comprehensive health check
*/
async runHealthCheck() {
await this.healthMonitor.performHealthChecks();
}
// ===== METRICS AND MONITORING METHODS =====
/**
* Get comprehensive cluster metrics
*/
async getMetrics() {
return this.metricsCollector.getClusterMetrics();
}
/**
* Get metrics for specific worker
*/
async getWorkerMetrics(workerId) {
return this.metricsCollector.getWorkerMetrics(workerId);
}
/**
* Get aggregated metrics summary
*/
async getAggregatedMetrics() {
return this.metricsCollector.getAggregatedMetrics();
}
/**
* Start monitoring systems
*/
startMonitoring() {
this.healthMonitor.startMonitoring();
this.metricsCollector.startCollection();
Logger.logger.debug("cluster", "Monitoring systems started");
}
/**
* Stop monitoring systems
*/
stopMonitoring() {
this.healthMonitor.stopMonitoring();
this.metricsCollector.stopCollection();
Logger.logger.debug("cluster", "Monitoring systems stopped");
}
/**
* Export metrics in specified format
*/
async exportMetrics(format = "json") {
return await this.metricsCollector.exportMetrics(format);
}
// ===== SCALING METHODS =====
/**
* Scale up by adding workers
*/
async scaleUp(count = 1) {
await this.autoScaler.scaleUp(count);
}
/**
* Scale down by removing workers
*/
async scaleDown(count = 1) {
await this.autoScaler.scaleDown(count);
}
/**
* Perform auto-scaling evaluation
*/
async autoScale() {
await this.autoScaler.autoScale();
}
/**
* Get optimal worker count
*/
async getOptimalWorkerCount() {
return await this.autoScaler.getOptimalWorkerCount();
}
// ===== LOAD BALANCING METHODS =====
/**
* Update load balancing strategy
*/
async updateLoadBalancingStrategy(strategy, options) {
await this.loadBalancer.updateStrategy(strategy, options);
}
/**
* Get load balance status
*/
async getLoadBalanceStatus() {
const status = this.loadBalancer.getLoadBalanceStatus();
return status.connections;
}
/**
* Redistribute load across workers
*/
async redistributeLoad() {
await this.loadBalancer.redistributeLoad();
}
// ===== IPC METHODS =====
/**
* Send message to specific worker
*/
async sendToWorker(workerId, message) {
await this.ipcManager.sendToWorker(workerId, message);
}
/**
* Send message to all workers
*/
async sendToAllWorkers(message) {
await this.ipcManager.sendToAllWorkers(message);
}
/**
* Broadcast message to all workers
*/
async broadcast(message) {
await this.ipcManager.broadcast(message);
}
/**
* Send message to random worker
*/
async sendToRandomWorker(message) {
await this.ipcManager.sendToRandomWorker(message);
}
/**
* Send message to least loaded worker
*/
async sendToLeastLoadedWorker(message) {
await this.ipcManager.sendToLeastLoadedWorker(message);
}
// ===== EVENT HANDLING METHODS =====
/**
* Register event handler
*/
addListener(event, handler) {
return super.on(event, handler);
}
/**
* Remove event handler
*/
removeListener(event, handler) {
return super.off(event, handler);
}
/**
* Emit cluster event
*/
emitEvent(event, ...args) {
return super.emit(event, ...args);
}
// ===== STATE MANAGEMENT METHODS =====
/**
* Save cluster state
*/
async saveState() {
try {
// Create comprehensive persistent state
const persistentState = {
state: this.state,
config: this.config,
workers: this.workerManager
.getActiveWorkers()
.map((worker) => ({
id: worker.workerId,
pid: worker.pid,
status: worker.health.status === "healthy"
? "running"
: "stopped",
startTime: worker.lastRestart || new Date(),
restarts: worker.restarts,
metrics: worker,
})),
metrics: this.metricsCollector.exportMetricsForPersistence(),
loadBalancer: {
strategy: this.config.loadBalancing?.strategy || "round-robin",
weights: {},
distribution: {},
historicalTrends: {
requestsPerMinute: [],
averageResponseTimes: [],
errorRates: [],
},
},
timestamp: new Date(),
version: "1.0.0",
};
// Save to persistent storage
await this.saveClusterStateToPersistentStorage(persistentState);
}
catch (error) {
const securityError = errorHandler.createSecurityError(`Failed to save cluster state: ${error.message}`, errorHandler.ErrorType.INTERNAL, errorHandler.ErrorSeverity.MEDIUM, "CLUSTER_STATE_SAVE_ERROR", { operation: "save_state" });
this.errorLogger.logError(securityError);
throw error;
}
}
/**
* Restore cluster state
*/
async restoreState() {
try {
// Restore from persistent storage
const savedState = await this.loadClusterStateFromPersistentStorage();
if (savedState) {
await this.applyRestoredState(savedState);
Logger.logger.debug("cluster", " Cluster state restored from persistent storage");
}
else {
Logger.logger.debug("cluster", "No saved cluster state found, starting fresh");
}
// if (savedState) {
// await this.applyRestoredState(savedState);
// }
}
catch (error) {
const securityError = errorHandler.createSecurityError(`Failed to restore cluster state: ${error.message}`, errorHandler.ErrorType.INTERNAL, errorHandler.ErrorSeverity.MEDIUM, "CLUSTER_STATE_RESTORE_ERROR", { operation: "restore_state" });
this.errorLogger.logError(securityError);
throw error;
}
}
/**
* Get current cluster state
*/
async getState() {
return this.state;
}
/**
* Export cluster configuration
*/
async exportConfiguration() {
return { ...this.config };
}
// ===== UTILITY METHODS =====
/**
* Check if cluster is healthy
*/
isHealthy() {
const activeWorkers = this.getActiveWorkers();
const totalWorkers = this.getAllWorkers();
if (totalWorkers.length === 0)
return false;
const healthyPercentage = (activeWorkers.length / totalWorkers.length) * 100;
return healthyPercentage >= 70; // 70% threshold
}
/**
* Get load balance score
*/
getLoadBalance() {
return this.loadBalancer.getLoadDistributionEfficiency();
}
/**
* Get recommended worker count
*/
getRecommendedWorkerCount() {
const cpuCount = os__namespace.cpus().length;
const currentLoad = this.getActiveWorkers().length;
// Simple recommendation based on CPU cores and current load
return Math.max(1, Math.min(cpuCount - 1, currentLoad));
}
/**
* Get cluster efficiency score
*/
getClusterEfficiency() {
const loadBalanceEfficiency = this.getLoadBalance();
const healthScore = this.isHealthy() ? 100 : 50;
const utilizationScore = this.calculateUtilizationScore();
return (loadBalanceEfficiency + healthScore + utilizationScore) / 3;
}
/**
* Calculate utilization score
*/
calculateUtilizationScore() {
const workers = this.getActiveWorkers();
if (workers.length === 0)
return 0;
const avgCpu = workers.reduce((sum, w) => sum + w.cpu.usage, 0) / workers.length;
const avgMemory = workers.reduce((sum, w) => sum + w.memory.percentage, 0) /
workers.length;
// Optimal utilization is around 60-70%
const optimalCpu = 65;
const optimalMemory = 65;
const cpuScore = 100 - Math.abs(avgCpu - optimalCpu);
const memoryScore = 100 - Math.abs(avgMemory - optimalMemory);
return (cpuScore + memoryScore) / 2;
}
// ===== ADVANCED FEATURES =====
/**
* Enable profiling for worker
*/
async enableProfiling(workerId) {
Logger.logger.debug("cluster", `Enabling profiling for worker ${workerId}`);
await this.sendToWorker(workerId, { type: "enable_profiling" });
}
/**
* Disable profiling for worker
*/
async disableProfiling(workerId) {
Logger.logger.debug("cluster", `Disabling profiling for worker ${workerId}`);
await this.sendToWorker(workerId, { type: "disable_profiling" });
}
/**
* Take heap snapshot of worker
*/
async takeHeapSnapshot(workerId) {
const snapshotPath = `/tmp/heap-snapshot-${workerId}-${Date.now()}.heapsnapshot`;
Logger.logger.debug("cluster", `Taking heap snapshot for worker ${workerId}: ${snapshotPath}`);
await this.sendToWorker(workerId, {
type: "take_heap_snapshot",
path: snapshotPath,
});
return snapshotPath;
}
/**
* Enable debug mode for worker
*/
async enableDebugMode(workerId, port) {
const debugPort = port || 9229;
Logger.logger.debug("cluster", `Enabling debug mode for worker ${workerId} on port ${debugPort}`);
await this.sendToWorker(workerId, {
type: "enable_debug",
port: debugPort,
});
}
/**
* Disable debug mode for worker
*/
async disableDebugMode(workerId) {
Logger.logger.debug("cluster", `Disabling debug mode for worker ${workerId}`);
await this.sendToWorker(workerId, { type: "disable_debug" });
}
// ===== ROLLING UPDATES =====
/**
* Perform rolling update
*/
async performRollingUpdate(updateFn) {
Logger.logger.debug("cluster", "Starting rolling update...");
const workers = this.getAllWorkers();
const maxUnavailable = this.config.advanced?.deployment?.maxUnavailable || 1;
for (let i = 0; i < workers.length; i += maxUnavailable) {
const batch = workers.slice(i, i + maxUnavailable);
// Drain workers in batch
for (const worker of batch) {
await this.drainWorker(worker.workerId);
}
// Perform update
await updateFn();
// Replace workers
for (const worker of batch) {
await this.replaceWorker(worker.workerId);
}
// Wait for new workers to be ready
await new Promise((resolve) => setTimeout(resolve, 5000));
}
Logger.logger.debug("cluster", "Rolling update completed");
}
/**
* Drain worker connections
*/
async drainWorker(workerId) {
Logger.logger.debug("cluster", `Draining worker ${workerId}...`);
await this.sendToWorker(workerId, { type: "drain_connections" });
// Wait for connections to drain
await new Promise((resolve) => setTimeout(resolve, 10000));
}
// ===== CIRCUIT BREAKER =====
/**
* Check if circuit breaker is open
*/
isCircuitOpen(workerId) {
if (workerId) {
const worker = this.getWorker(workerId);
return worker ? worker.health.consecutiveFailures >= 5 : false;
}
// Check overall cluster circuit breaker
const unhealthyWorkers = this.getUnhealthyWorkers();
const totalWorkers = this.getAllWorkers();
return (totalWorkers.length > 0 &&
unhealthyWorkers.length / totalWorkers.length > 0.5);
}
/**
* Reset circuit breaker
*/
async resetCircuitBreaker(workerId) {
if (workerId) {
Logger.logger.debug("cluster", `Resetting circuit breaker for worker ${workerId}`);
await this.sendToWorker(workerId, {
type: "reset_circuit_breaker",
});
}
else {
Logger.logger.debug("cluster", "Resetting cluster circuit breaker");
// Reset all worker circuit breakers
const workers = this.getAllWorkers();
for (const worker of workers) {
await this.sendToWorker(worker.workerId, {
type: "reset_circuit_breaker",
});
}
}
}
// ===== CLEANUP METHODS =====
/**
* Cleanup cluster resources
*/
async cleanup() {
Logger.logger.debug("cluster", "Cleaning up cluster resources...");
await this.stop(true);
// Cleanup monitoring
this.healthMonitor.stopMonitoring();
this.metricsCollector.stopCollection();
this.autoScaler.stopScaling();
Logger.logger.debug("cluster", "Cluster cleanup completed");
}
/**
* Force cleanup cluster resources
*/
async forceCleanup() {
Logger.logger.debug("cluster", "Force cleaning up cluster resources...");
await this.stop(false);
// Force cleanup
this.removeAllListeners();
Logger.logger.debug("cluster", "Force cleanup completed");
}
// ===== PERSISTENCE METHODS =====
/**
* Save cluster state to persistent storage
*/
async saveClusterStateToPersistentStorage(clusterState) {
if (!this.persistenceManager) {
// Fallback: emit event for external handling
this.emit("cluster:state:saved", clusterState);
return;
}
try {
await this.persistenceManager.saveClusterState(clusterState);
}
catch (error) {
Logger.logger.warn("cluster", `Failed to save cluster state: ${error.message}`);
// Emit event as fallback
this.emit("cluster:state:saved", clusterState);
}
}
/**
* Load cluster state from persistent storage
*/
async loadClusterStateFromPersistentStorage() {
if (!this.persistenceManager) {
// Fallback: emit event for external handling
this.emit("cluster:state:restore_requested");
return null;
}
try {
return await this.persistenceManager.loadClusterState();
}
catch (error) {
Logger.logger.warn("cluster", `Failed to load cluster state: ${error.message}`);
return null;
}
}
/**
* Apply restored cluster state
*/
async applyRestoredState(savedState) {
try {
if (savedState.workers) {
// Restore worker configurations
for (const workerState of savedState.workers) {
if (workerState.status === "running") {
// Try to reconnect to existing worker or spawn new one
await this.restoreOrSpawnWorker(workerState);
}
}
}
if (savedState.config) {
// Apply saved configuration (merge with current)
this.config = { ...this.config, ...savedState.config };
}
if (savedState.metrics) {
// Restore metrics using the MetricsCollector
if (savedState.metrics.historicalData) {
this.metricsCollector.restoreHistoricalData(savedState.metrics.historicalData);
}
if (savedState.metrics.workerHistory) {
this.metricsCollector.restoreWorkerHistoricalData(savedState.metrics.workerHistory);
}
Logger.logger.debug("cluster", "✔ Restored metrics from persistent storage");
}
}
catch (error) {
Logger.logger.warn("cluster", `Failed to apply restored state: ${error.message}`);
}
}
/**
* Restore or spawn worker from saved state
*/
async restoreOrSpawnWorker(workerState) {
try {
// Check if worker process still exists
const existingWorker = this.workerManager.getWorker(workerState.id);
if (!existingWorker) {
// Worker doesn't exist, spawn a new one
Logger.logger.debug("cluster", `Spawning new worker to replace ${workerState.id}`);
await this.addWorker();
}
else {
Logger.logger.debug("cluster", `Worker ${workerState.id} already exists and running`);
}
}
catch (error) {
Logger.logger.warn("cluster", `Failed to restore worker ${workerState.id}: ${error.message}`);
}
}
/**
* Get persistence manager statistics
*/
getPersistenceStats() {
if (!this.persistenceManager) {
return { enabled: false, type: "none" };
}
return this.persistenceManager.getStats();
}
/**
* Close persistence manager
*/
async closePersistenceManager() {
if (this.persistenceManager) {
await this.persistenceManager.close();
}
}
}
exports.ClusterManager = ClusterManager;
//# sourceMappingURL=cluster-manager.js.map