datapilot-cli
Version:
Enterprise-grade streaming multi-format data analysis with comprehensive statistical insights and intelligent relationship detection - supports CSV, JSON, Excel, TSV, Parquet - memory-efficient, cross-platform
480 lines • 19.2 kB
JavaScript
"use strict";
/**
* Enhanced Error Recovery System
* Comprehensive error handling with automatic recovery and context preservation
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.EnhancedErrorHandler = void 0;
exports.getGlobalEnhancedErrorHandler = getGlobalEnhancedErrorHandler;
exports.shutdownGlobalEnhancedErrorHandler = shutdownGlobalEnhancedErrorHandler;
exports.withErrorHandling = withErrorHandling;
const events_1 = require("events");
const perf_hooks_1 = require("perf_hooks");
const logger_1 = require("./logger");
const types_1 = require("../core/types");
const circuit_breaker_1 = require("../performance/circuit-breaker");
const resource_leak_detector_1 = require("../performance/resource-leak-detector");
class EnhancedErrorHandler extends events_1.EventEmitter {
recoveryStrategies = [];
errorMetrics;
options;
errorHistory = new Map();
recentErrors = [];
maxErrorHistory = 1000;
constructor(options = {}) {
super();
this.options = {
enableRecovery: options.enableRecovery ?? true,
maxRetries: options.maxRetries || 3,
retryDelays: options.retryDelays || [1000, 2000, 4000], // Exponential backoff
trackMetrics: options.trackMetrics ?? true,
enableCircuitBreaker: options.enableCircuitBreaker ?? true,
enableResourceTracking: options.enableResourceTracking ?? true,
contextEnrichment: options.contextEnrichment ?? true,
};
this.errorMetrics = {
totalErrors: 0,
errorsByCategory: {},
errorsBySeverity: {},
recoverySuccessRate: 0,
averageRecoveryTime: 0,
frequentErrors: [],
recentErrors: [],
};
this.initializeDefaultStrategies();
}
/**
* Handle an error with automatic recovery attempts
*/
async handleError(error, context, originalOperation) {
const enrichedContext = this.enrichContext(error, context);
const startTime = perf_hooks_1.performance.now();
// Update metrics
this.updateMetrics(error, enrichedContext);
try {
// Check if operation is circuit broken
if (this.options.enableCircuitBreaker) {
const circuitBreaker = (0, circuit_breaker_1.getGlobalCircuitBreakerManager)();
const operationName = `${enrichedContext.component}.${enrichedContext.operation}`;
if (!circuitBreaker.getCircuitBreaker(operationName, async () => { }).isAvailable()) {
this.emit('circuit-breaker-blocked', { context: enrichedContext, error });
return { success: false, finalError: error };
}
}
// Attempt recovery if enabled
if (this.options.enableRecovery) {
const recoveryResult = await this.attemptRecovery(error, enrichedContext, originalOperation);
if (recoveryResult.success) {
const recoveryTime = perf_hooks_1.performance.now() - startTime;
this.emit('recovery-success', {
context: enrichedContext,
recoveryTime,
strategy: recoveryResult.strategy,
});
return { success: true, result: recoveryResult.result };
}
}
// Recovery failed or disabled
this.emit('error-unrecoverable', { context: enrichedContext, error });
return { success: false, finalError: error };
}
catch (handlingError) {
logger_1.logger.error(`Error in error handler: ${handlingError.message}`, enrichedContext);
return { success: false, finalError: error };
}
}
/**
* Wrap a function with automatic error handling
*/
wrapFunction(fn, context) {
return (async (...args) => {
const fullContext = {
operation: fn.name || 'anonymous',
component: 'wrapped-function',
startTime: perf_hooks_1.performance.now(),
...context,
};
try {
return await fn(...args);
}
catch (error) {
const result = await this.handleError(error, fullContext, () => fn(...args));
if (result.success) {
return result.result;
}
else {
throw result.finalError;
}
}
});
}
/**
* Add a custom recovery strategy
*/
addRecoveryStrategy(strategy) {
// Insert strategy in priority order
const insertIndex = this.recoveryStrategies.findIndex((s) => s.priority > strategy.priority);
if (insertIndex === -1) {
this.recoveryStrategies.push(strategy);
}
else {
this.recoveryStrategies.splice(insertIndex, 0, strategy);
}
this.emit('strategy-added', { name: strategy.name, priority: strategy.priority });
}
/**
* Attempt recovery using available strategies
*/
async attemptRecovery(error, context, originalOperation) {
for (const strategy of this.recoveryStrategies) {
if (strategy.canRecover(error, context)) {
const maxRetries = strategy.maxRetries || this.options.maxRetries;
const currentRetries = context.retryCount || 0;
if (currentRetries >= maxRetries) {
continue; // Skip this strategy if max retries exceeded
}
try {
logger_1.logger.info(`Attempting recovery with strategy: ${strategy.name}`, context);
const recoveryResult = await strategy.recover(error, {
...context,
retryCount: currentRetries + 1,
});
// If recovery succeeded and we have the original operation, try it again
if (originalOperation && recoveryResult !== false) {
try {
const result = await originalOperation();
return { success: true, result, strategy: strategy.name };
}
catch (retryError) {
// If retry failed, continue to next strategy
continue;
}
}
return {
success: true,
result: recoveryResult,
strategy: strategy.name,
};
}
catch (recoveryError) {
logger_1.logger.warn(`Recovery strategy ${strategy.name} failed: ${recoveryError.message}`, context);
continue;
}
}
}
return { success: false };
}
/**
* Enrich error context with additional information
*/
enrichContext(error, context) {
if (!this.options.contextEnrichment) {
return context;
}
const enriched = {
...context,
memoryUsage: process.memoryUsage().heapUsed,
stackTrace: error.stack,
startTime: context.startTime || perf_hooks_1.performance.now(),
};
// Add system resource information
if (this.options.enableResourceTracking) {
const leakDetector = (0, resource_leak_detector_1.getGlobalResourceLeakDetector)();
const resourceStats = leakDetector.getResourceStats();
enriched.metadata = {
...enriched.metadata,
resourceStats,
};
}
return enriched;
}
/**
* Update error metrics
*/
updateMetrics(error, context) {
if (!this.options.trackMetrics) {
return;
}
this.errorMetrics.totalErrors++;
// Update category counts
const category = this.categorizeError(error);
this.errorMetrics.errorsByCategory[category] =
(this.errorMetrics.errorsByCategory[category] || 0) + 1;
// Update severity counts
const severity = this.getSeverity(error);
this.errorMetrics.errorsBySeverity[severity] =
(this.errorMetrics.errorsBySeverity[severity] || 0) + 1;
// Track frequent errors
const errorKey = `${error.name}: ${error.message}`;
this.errorHistory.set(errorKey, (this.errorHistory.get(errorKey) || 0) + 1);
// Add to recent errors
this.recentErrors.push({
timestamp: Date.now(),
error: errorKey,
context,
recovered: false, // Will be updated if recovery succeeds
});
// Maintain recent errors limit
if (this.recentErrors.length > 100) {
this.recentErrors.shift();
}
// Update frequent errors list
this.updateFrequentErrors();
}
/**
* Update frequent errors list
*/
updateFrequentErrors() {
this.errorMetrics.frequentErrors = Array.from(this.errorHistory.entries())
.sort(([, a], [, b]) => b - a)
.slice(0, 10)
.map(([message, count]) => ({ message, count }));
}
/**
* Categorize error for metrics
*/
categorizeError(error) {
if (error instanceof types_1.DataPilotError) {
return error.category;
}
// Categorize based on error type and message
const message = error.message.toLowerCase();
if (message.includes('memory') || message.includes('heap')) {
return types_1.ErrorCategory.MEMORY;
}
else if (message.includes('timeout') || message.includes('network')) {
return types_1.ErrorCategory.NETWORK;
}
else if (message.includes('file') || message.includes('permission')) {
return types_1.ErrorCategory.IO;
}
else if (message.includes('parse') || message.includes('invalid')) {
return types_1.ErrorCategory.PARSING;
}
return types_1.ErrorCategory.ANALYSIS;
}
/**
* Get error severity
*/
getSeverity(error) {
if (error instanceof types_1.DataPilotError) {
return error.severity;
}
// Determine severity based on error characteristics
const message = error.message.toLowerCase();
if (message.includes('critical') || message.includes('fatal')) {
return types_1.ErrorSeverity.CRITICAL;
}
else if (message.includes('memory') || message.includes('worker')) {
return types_1.ErrorSeverity.HIGH;
}
else if (message.includes('timeout') || message.includes('retry')) {
return types_1.ErrorSeverity.MEDIUM;
}
return types_1.ErrorSeverity.LOW;
}
/**
* Initialize default recovery strategies
*/
initializeDefaultStrategies() {
// Memory pressure recovery
this.addRecoveryStrategy({
name: 'memory-pressure-recovery',
description: 'Recover from memory pressure by triggering GC and reducing chunk size',
priority: 1,
canRecover: (error, context) => {
return (error.message.toLowerCase().includes('memory') ||
error.message.toLowerCase().includes('heap'));
},
recover: async (error, context) => {
// Force garbage collection
if (global.gc) {
global.gc();
}
// Reduce memory usage in context metadata
if (context.metadata) {
context.metadata.memoryRecoveryApplied = true;
}
await new Promise((resolve) => setTimeout(resolve, 1000)); // Brief pause
return true;
},
});
// Worker failure recovery
this.addRecoveryStrategy({
name: 'worker-failure-recovery',
description: 'Recover from worker failures by creating new workers',
priority: 2,
canRecover: (error, context) => {
return (error.message.toLowerCase().includes('worker') ||
context.component?.toLowerCase().includes('worker'));
},
recover: async (error, context) => {
logger_1.logger.warn('Attempting worker recovery', context);
// Signal to create new worker (implementation would depend on specific worker pool)
if (context.metadata) {
context.metadata.workerRecoveryApplied = true;
}
return true;
},
});
// File system recovery
this.addRecoveryStrategy({
name: 'file-system-recovery',
description: 'Recover from file system errors with retries and fallbacks',
priority: 3,
maxRetries: 5,
canRecover: (error, context) => {
const message = error.message.toLowerCase();
return (message.includes('enoent') ||
message.includes('eacces') ||
message.includes('emfile') ||
message.includes('file'));
},
recover: async (error, context) => {
const retryCount = context.retryCount || 0;
const delay = Math.min(1000 * Math.pow(2, retryCount), 10000); // Exponential backoff, max 10s
logger_1.logger.info(`File system recovery attempt ${retryCount + 1}, waiting ${delay}ms`, context);
await new Promise((resolve) => setTimeout(resolve, delay));
return true;
},
});
// Network/timeout recovery
this.addRecoveryStrategy({
name: 'network-timeout-recovery',
description: 'Recover from network timeouts and connection issues',
priority: 4,
canRecover: (error, context) => {
const message = error.message.toLowerCase();
return (message.includes('timeout') ||
message.includes('network') ||
message.includes('connection'));
},
recover: async (error, context) => {
const retryCount = context.retryCount || 0;
const delay = Math.min(2000 * Math.pow(2, retryCount), 30000); // Exponential backoff, max 30s
logger_1.logger.info(`Network recovery attempt ${retryCount + 1}, waiting ${delay}ms`, context);
await new Promise((resolve) => setTimeout(resolve, delay));
return true;
},
});
// Generic retry strategy (lowest priority)
this.addRecoveryStrategy({
name: 'generic-retry',
description: 'Generic retry strategy for transient errors',
priority: 100,
maxRetries: 2,
canRecover: (error, context) => {
// Don't retry validation errors or programming errors
const message = error.message.toLowerCase();
return (!message.includes('validation') &&
!message.includes('type') &&
!message.includes('syntax'));
},
recover: async (error, context) => {
const retryCount = context.retryCount || 0;
const delay = this.options.retryDelays[Math.min(retryCount, this.options.retryDelays.length - 1)];
logger_1.logger.info(`Generic retry attempt ${retryCount + 1}, waiting ${delay}ms`, context);
await new Promise((resolve) => setTimeout(resolve, delay));
return true;
},
});
}
/**
* Get current error metrics
*/
getMetrics() {
return {
...this.errorMetrics,
recentErrors: [...this.recentErrors],
};
}
/**
* Reset error metrics
*/
resetMetrics() {
this.errorMetrics = {
totalErrors: 0,
errorsByCategory: {},
errorsBySeverity: {},
recoverySuccessRate: 0,
averageRecoveryTime: 0,
frequentErrors: [],
recentErrors: [],
};
this.errorHistory.clear();
this.recentErrors = [];
}
/**
* Get health status
*/
getHealthStatus() {
const recentErrorCount = this.recentErrors.filter((e) => Date.now() - e.timestamp < 300000).length;
const errorRate = recentErrorCount; // Errors per 5 minutes
const recoveredCount = this.recentErrors.filter((e) => e.recovered).length;
const recoveryRate = this.recentErrors.length > 0 ? recoveredCount / this.recentErrors.length : 1;
let status = 'healthy';
const recommendations = [];
if (errorRate > 10) {
status = 'unhealthy';
recommendations.push('High error rate detected - investigate root causes');
}
else if (errorRate > 3) {
status = 'degraded';
recommendations.push('Elevated error rate - monitor system closely');
}
if (recoveryRate < 0.5) {
status = status === 'healthy' ? 'degraded' : 'unhealthy';
recommendations.push('Low recovery rate - review recovery strategies');
}
const topErrors = this.errorMetrics.frequentErrors.slice(0, 3);
if (topErrors.length > 0 && topErrors[0].count > 5) {
recommendations.push(`Frequent error pattern: ${topErrors[0].message} (${topErrors[0].count} occurrences)`);
}
return {
status,
errorRate,
recoveryRate,
recommendations,
};
}
/**
* Shutdown error handler
*/
shutdown() {
this.removeAllListeners();
this.emit('shutdown');
}
}
exports.EnhancedErrorHandler = EnhancedErrorHandler;
/**
* Global enhanced error handler
*/
let globalEnhancedErrorHandler = null;
function getGlobalEnhancedErrorHandler(options) {
if (!globalEnhancedErrorHandler) {
globalEnhancedErrorHandler = new EnhancedErrorHandler(options);
}
return globalEnhancedErrorHandler;
}
function shutdownGlobalEnhancedErrorHandler() {
if (globalEnhancedErrorHandler) {
globalEnhancedErrorHandler.shutdown();
globalEnhancedErrorHandler = null;
}
}
/**
* Decorator for automatic error handling
*/
function withErrorHandling(context) {
return function (target, propertyName, descriptor) {
const method = descriptor.value;
const errorHandler = getGlobalEnhancedErrorHandler();
descriptor.value = errorHandler.wrapFunction(method, {
operation: propertyName,
component: target.constructor.name,
...context,
});
return descriptor;
};
}
//# sourceMappingURL=enhanced-error-handler.js.map