hook-engine
Version:
Production-grade webhook engine with comprehensive adapter support, security, reliability, structured logging, and CLI tools.
498 lines (497 loc) • 17.3 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.ReliabilityManager = void 0;
const events_1 = require("events");
const perf_hooks_1 = require("perf_hooks");
/**
* Comprehensive reliability manager for production webhook processing
*/
class ReliabilityManager extends events_1.EventEmitter {
constructor(config) {
super();
this.isShuttingDown = false;
this.deduplicationStore = new Map();
this.requestMetrics = {
total: 0,
errors: 0,
responseTimes: []
};
this.config = config;
this.startTime = Date.now();
this.initialize();
}
/**
* Initialize all reliability features
*/
initialize() {
// Start health checks
if (this.config.healthChecks.enabled) {
this.startHealthChecks();
}
// Start memory monitoring
if (this.config.memoryMonitoring.enabled) {
this.startMemoryMonitoring();
}
// Setup graceful shutdown
if (this.config.gracefulShutdown.enabled) {
this.setupGracefulShutdown();
}
// Start deduplication cleanup
if (this.config.deduplication.enabled) {
this.startDeduplicationCleanup();
}
this.emit('initialized');
}
/**
* Start health check monitoring
*/
startHealthChecks() {
const config = this.config.healthChecks;
// Wait for grace period before starting
setTimeout(() => {
this.healthCheckInterval = setInterval(async () => {
try {
await this.performHealthCheck();
}
catch (error) {
this.emit('healthCheckError', error);
}
}, config.interval);
// Perform initial health check
this.performHealthCheck();
}, config.gracePeriod);
}
/**
* Perform comprehensive health check
*/
async performHealthCheck() {
const startTime = perf_hooks_1.performance.now();
const checks = [];
const dependencies = [];
// Check endpoints
for (const endpoint of this.config.healthChecks.endpoints) {
const checkResult = await this.checkEndpoint(endpoint);
checks.push(checkResult);
}
// Check dependencies
for (const dependency of this.config.healthChecks.dependencies) {
const depStatus = await this.checkDependency(dependency);
dependencies.push(depStatus);
}
// Gather metrics
const metrics = await this.gatherHealthMetrics();
// Determine overall status
const hasFailedCriticalChecks = checks.some(c => c.status === 'fail') ||
dependencies.some(d => d.status === 'unavailable' &&
this.config.healthChecks.dependencies.find(dep => dep.name === d.name)?.critical);
const hasWarnings = checks.some(c => c.status === 'warn') ||
dependencies.some(d => d.status === 'degraded');
const status = {
status: hasFailedCriticalChecks ? 'unhealthy' : hasWarnings ? 'degraded' : 'healthy',
timestamp: new Date(),
uptime: Date.now() - this.startTime,
checks,
dependencies,
metrics
};
this.emit('healthCheck', status);
return status;
}
/**
* Check individual endpoint
*/
async checkEndpoint(endpoint) {
const startTime = perf_hooks_1.performance.now();
try {
// Simulate endpoint check (in real implementation, use HTTP client)
await new Promise(resolve => setTimeout(resolve, Math.random() * 100));
const duration = perf_hooks_1.performance.now() - startTime;
return {
name: endpoint.name,
status: 'pass',
duration,
details: { path: endpoint.path, method: endpoint.method }
};
}
catch (error) {
return {
name: endpoint.name,
status: 'fail',
duration: perf_hooks_1.performance.now() - startTime,
error: error.message
};
}
}
/**
* Check dependency status
*/
async checkDependency(dependency) {
const startTime = perf_hooks_1.performance.now();
try {
const healthStatus = await dependency.check();
const responseTime = perf_hooks_1.performance.now() - startTime;
return {
name: dependency.name,
status: healthStatus.status === 'healthy' ? 'available' :
healthStatus.status === 'degraded' ? 'degraded' : 'unavailable',
responseTime,
lastChecked: new Date()
};
}
catch (error) {
return {
name: dependency.name,
status: 'unavailable',
responseTime: perf_hooks_1.performance.now() - startTime,
error: error.message,
lastChecked: new Date()
};
}
}
/**
* Gather health metrics
*/
async gatherHealthMetrics() {
const memoryUsage = this.getMemoryUsage();
const cpuUsage = await this.getCPUUsage();
// Calculate request metrics
const totalRequests = this.requestMetrics.total;
const errorRate = totalRequests > 0 ? (this.requestMetrics.errors / totalRequests) * 100 : 0;
const avgResponseTime = this.requestMetrics.responseTimes.length > 0 ?
this.requestMetrics.responseTimes.reduce((a, b) => a + b, 0) / this.requestMetrics.responseTimes.length : 0;
return {
memoryUsage,
cpuUsage,
activeConnections: 0, // Would be populated from actual connection pools
requestsPerSecond: this.calculateRequestsPerSecond(),
errorRate,
averageResponseTime: avgResponseTime
};
}
/**
* Get memory usage information
*/
getMemoryUsage() {
const memUsage = process.memoryUsage();
const totalMemory = require('os').totalmem();
const freeMemory = require('os').freemem();
const usedMemory = totalMemory - freeMemory;
return {
used: usedMemory,
total: totalMemory,
percentage: (usedMemory / totalMemory) * 100,
heap: {
used: memUsage.heapUsed,
total: memUsage.heapTotal,
percentage: (memUsage.heapUsed / memUsage.heapTotal) * 100
}
};
}
/**
* Get CPU usage (simplified implementation)
*/
async getCPUUsage() {
// Simplified CPU usage calculation
const startUsage = process.cpuUsage();
await new Promise(resolve => setTimeout(resolve, 100));
const endUsage = process.cpuUsage(startUsage);
const totalUsage = endUsage.user + endUsage.system;
return (totalUsage / 1000000) * 10; // Convert to percentage
}
/**
* Calculate requests per second
*/
calculateRequestsPerSecond() {
const uptime = (Date.now() - this.startTime) / 1000;
return uptime > 0 ? this.requestMetrics.total / uptime : 0;
}
/**
* Start memory monitoring
*/
startMemoryMonitoring() {
const config = this.config.memoryMonitoring;
this.memoryMonitorInterval = setInterval(() => {
this.checkMemoryUsage();
}, config.interval);
}
/**
* Check memory usage and trigger actions if needed
*/
checkMemoryUsage() {
const memoryUsage = this.getMemoryUsage();
const config = this.config.memoryMonitoring;
// Check system memory thresholds
if (memoryUsage.percentage >= config.thresholds.critical) {
this.triggerMemoryActions('critical', memoryUsage);
}
else if (memoryUsage.percentage >= config.thresholds.warning) {
this.triggerMemoryActions('warning', memoryUsage);
}
// Check heap memory thresholds
if (memoryUsage.heap.percentage >= config.thresholds.heap.critical) {
this.triggerMemoryActions('critical', memoryUsage);
}
else if (memoryUsage.heap.percentage >= config.thresholds.heap.warning) {
this.triggerMemoryActions('warning', memoryUsage);
}
this.emit('memoryCheck', memoryUsage);
}
/**
* Trigger memory actions based on threshold
*/
async triggerMemoryActions(threshold, memoryUsage) {
const actions = this.config.memoryMonitoring.actions.filter(a => a.threshold === threshold);
for (const action of actions) {
try {
switch (action.action) {
case 'log':
this.emit('memoryWarning', { threshold, memoryUsage, action: 'log' });
break;
case 'gc':
if (global.gc) {
global.gc();
this.emit('memoryAction', { threshold, action: 'gc', memoryUsage });
}
break;
case 'alert':
this.emit('memoryAlert', { threshold, memoryUsage, severity: threshold });
break;
case 'throttle':
this.emit('memoryAction', { threshold, action: 'throttle', memoryUsage });
break;
case 'reject':
this.emit('memoryAction', { threshold, action: 'reject', memoryUsage });
break;
case 'custom':
if (action.execute) {
await action.execute();
}
break;
}
}
catch (error) {
this.emit('memoryActionError', { action: action.action, error });
}
}
}
/**
* Setup graceful shutdown handling
*/
setupGracefulShutdown() {
const config = this.config.gracefulShutdown;
for (const signal of config.signals) {
process.on(signal, () => {
this.initiateGracefulShutdown(signal);
});
}
}
/**
* Initiate graceful shutdown process
*/
initiateGracefulShutdown(signal) {
if (this.isShuttingDown) {
return;
}
this.isShuttingDown = true;
this.emit('shutdownInitiated', { signal });
this.shutdownPromise = this.performGracefulShutdown();
// Force exit if graceful shutdown takes too long
setTimeout(() => {
if (this.isShuttingDown) {
this.emit('forceExit', { reason: 'timeout' });
process.exit(1);
}
}, this.config.gracefulShutdown.forceExitTimeout);
}
/**
* Perform graceful shutdown
*/
async performGracefulShutdown() {
const config = this.config.gracefulShutdown;
const sortedHooks = [...config.hooks].sort((a, b) => a.priority - b.priority);
for (const hook of sortedHooks) {
try {
this.emit('shutdownHookStarted', { hook: hook.name });
const timeoutPromise = new Promise((_, reject) => {
setTimeout(() => reject(new Error(`Hook ${hook.name} timed out`)), hook.timeout);
});
await Promise.race([hook.execute(), timeoutPromise]);
this.emit('shutdownHookCompleted', { hook: hook.name });
}
catch (error) {
this.emit('shutdownHookError', { hook: hook.name, error });
}
}
// Cleanup intervals
this.cleanup();
this.emit('shutdownCompleted');
process.exit(0);
}
/**
* Check for duplicate requests
*/
async checkDeduplication(req) {
if (!this.config.deduplication.enabled) {
return {
isDuplicate: false,
count: 1,
action: 'processed',
key: ''
};
}
for (const level of this.config.deduplication.levels) {
if (!level.enabled)
continue;
const key = level.keyGenerator(req);
const existing = this.deduplicationStore.get(key);
const now = new Date();
if (existing) {
// Check if still within TTL
const age = now.getTime() - existing.timestamp.getTime();
if (age <= level.ttl) {
// Update count
existing.count++;
this.deduplicationStore.set(key, existing);
return {
isDuplicate: true,
originalTimestamp: existing.timestamp,
count: existing.count,
action: level.action,
key
};
}
else {
// Expired, remove and continue
this.deduplicationStore.delete(key);
}
}
// Not a duplicate, store for future checks
this.deduplicationStore.set(key, {
timestamp: now,
count: 1
});
}
return {
isDuplicate: false,
count: 1,
action: 'processed',
key: ''
};
}
/**
* Start deduplication cleanup
*/
startDeduplicationCleanup() {
const config = this.config.deduplication.cleanup;
if (!config.enabled)
return;
this.deduplicationCleanupInterval = setInterval(() => {
this.cleanupDeduplicationStore();
}, config.interval);
}
/**
* Cleanup expired deduplication entries
*/
cleanupDeduplicationStore() {
const config = this.config.deduplication.cleanup;
const now = Date.now();
let cleaned = 0;
for (const [key, entry] of this.deduplicationStore.entries()) {
const age = now - entry.timestamp.getTime();
if (age > config.maxAge) {
this.deduplicationStore.delete(key);
cleaned++;
if (cleaned >= config.batchSize) {
break;
}
}
}
if (cleaned > 0) {
this.emit('deduplicationCleanup', { cleaned, remaining: this.deduplicationStore.size });
}
}
/**
* Record request metrics
*/
recordRequest(responseTime, isError = false) {
this.requestMetrics.total++;
if (isError) {
this.requestMetrics.errors++;
}
this.requestMetrics.responseTimes.push(responseTime);
// Keep only last 1000 response times
if (this.requestMetrics.responseTimes.length > 1000) {
this.requestMetrics.responseTimes = this.requestMetrics.responseTimes.slice(-1000);
}
}
/**
* Get current health status
*/
async getCurrentHealthStatus() {
return await this.performHealthCheck();
}
/**
* Get reliability metrics
*/
async getMetrics() {
const healthStatus = await this.performHealthCheck();
const memoryUsage = this.getMemoryUsage();
const deduplicationMetrics = {
totalRequests: this.requestMetrics.total,
duplicateRequests: 0, // Would be tracked separately
deduplicationRate: 0,
storageSize: this.deduplicationStore.size,
cleanupRuns: 0, // Would be tracked
lastCleanup: new Date()
};
const connectionPools = {
// Would be populated from actual connection pools
};
return {
uptime: Date.now() - this.startTime,
healthStatus,
memoryUsage,
connectionPools,
deduplication: deduplicationMetrics,
lastUpdated: new Date()
};
}
/**
* Check if system is ready to accept requests
*/
async isReady() {
if (this.isShuttingDown) {
return false;
}
const healthStatus = await this.performHealthCheck();
return healthStatus.status !== 'unhealthy';
}
/**
* Check if system is alive (basic liveness check)
*/
isAlive() {
return !this.isShuttingDown;
}
/**
* Cleanup resources
*/
cleanup() {
if (this.healthCheckInterval) {
clearInterval(this.healthCheckInterval);
}
if (this.memoryMonitorInterval) {
clearInterval(this.memoryMonitorInterval);
}
if (this.deduplicationCleanupInterval) {
clearInterval(this.deduplicationCleanupInterval);
}
this.removeAllListeners();
}
/**
* Destroy the reliability manager
*/
destroy() {
this.cleanup();
}
}
exports.ReliabilityManager = ReliabilityManager;