datapilot-cli
Version:
Enterprise-grade streaming multi-format data analysis with comprehensive statistical insights and intelligent relationship detection - supports CSV, JSON, Excel, TSV, Parquet - memory-efficient, cross-platform
348 lines • 19.4 kB
JavaScript
;
/**
* Performance Optimization Module
* Exports all performance enhancement components
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.InputValidator = exports.withErrorHandling = exports.shutdownGlobalEnhancedErrorHandler = exports.getGlobalEnhancedErrorHandler = exports.EnhancedErrorHandler = exports.trackResource = exports.shutdownGlobalResourceLeakDetector = exports.getGlobalResourceLeakDetector = exports.ResourceLeakDetector = exports.CircuitState = exports.shutdownGlobalCircuitBreakerManager = exports.getGlobalCircuitBreakerManager = exports.CircuitBreakerManager = exports.CircuitBreaker = exports.createObjectPool = exports.createBufferPool = exports.shutdownGlobalResourcePoolManager = exports.getGlobalResourcePoolManager = exports.GcOptimizer = exports.ResourcePoolManager = exports.ResourcePool = exports.shutdownGlobalIntelligentChunker = exports.getGlobalIntelligentChunker = exports.IntelligentChunker = exports.shutdownGlobalAdaptiveStreamer = exports.getGlobalAdaptiveStreamer = exports.AdaptiveStreamer = exports.withMemoryOptimization = exports.shutdownGlobalMemoryOptimizer = exports.getGlobalMemoryOptimizer = exports.MemoryOptimizer = exports.shutdownGlobalParallelAnalyzer = exports.getGlobalParallelAnalyzer = exports.ParallelAnalyzer = exports.shutdownGlobalWorkerPool = exports.getGlobalWorkerPool = exports.WorkerPool = void 0;
exports.initializePerformanceOptimizations = initializePerformanceOptimizations;
exports.shutdownPerformanceOptimizations = shutdownPerformanceOptimizations;
exports.shutdownPerformanceOptimizationsEnhanced = shutdownPerformanceOptimizationsEnhanced;
exports.getPerformanceStats = getPerformanceStats;
exports.getSystemHealthStatus = getSystemHealthStatus;
exports.emergencySystemRecovery = emergencySystemRecovery;
// Worker pool and parallel processing
var worker_pool_1 = require("./worker-pool");
Object.defineProperty(exports, "WorkerPool", { enumerable: true, get: function () { return worker_pool_1.WorkerPool; } });
Object.defineProperty(exports, "getGlobalWorkerPool", { enumerable: true, get: function () { return worker_pool_1.getGlobalWorkerPool; } });
Object.defineProperty(exports, "shutdownGlobalWorkerPool", { enumerable: true, get: function () { return worker_pool_1.shutdownGlobalWorkerPool; } });
var parallel_analyzer_1 = require("./parallel-analyzer");
Object.defineProperty(exports, "ParallelAnalyzer", { enumerable: true, get: function () { return parallel_analyzer_1.ParallelAnalyzer; } });
Object.defineProperty(exports, "getGlobalParallelAnalyzer", { enumerable: true, get: function () { return parallel_analyzer_1.getGlobalParallelAnalyzer; } });
Object.defineProperty(exports, "shutdownGlobalParallelAnalyzer", { enumerable: true, get: function () { return parallel_analyzer_1.shutdownGlobalParallelAnalyzer; } });
// Memory optimization
var memory_optimizer_1 = require("./memory-optimizer");
Object.defineProperty(exports, "MemoryOptimizer", { enumerable: true, get: function () { return memory_optimizer_1.MemoryOptimizer; } });
Object.defineProperty(exports, "getGlobalMemoryOptimizer", { enumerable: true, get: function () { return memory_optimizer_1.getGlobalMemoryOptimizer; } });
Object.defineProperty(exports, "shutdownGlobalMemoryOptimizer", { enumerable: true, get: function () { return memory_optimizer_1.shutdownGlobalMemoryOptimizer; } });
Object.defineProperty(exports, "withMemoryOptimization", { enumerable: true, get: function () { return memory_optimizer_1.withMemoryOptimization; } });
// Adaptive streaming
var adaptive_streamer_1 = require("./adaptive-streamer");
Object.defineProperty(exports, "AdaptiveStreamer", { enumerable: true, get: function () { return adaptive_streamer_1.AdaptiveStreamer; } });
Object.defineProperty(exports, "getGlobalAdaptiveStreamer", { enumerable: true, get: function () { return adaptive_streamer_1.getGlobalAdaptiveStreamer; } });
Object.defineProperty(exports, "shutdownGlobalAdaptiveStreamer", { enumerable: true, get: function () { return adaptive_streamer_1.shutdownGlobalAdaptiveStreamer; } });
// Enhanced streaming with parallel processing - Disabled due to missing exports
// export {
// ParallelStreamingAnalyzer,
// createParallelStreamingAnalyzer,
// type ParallelStreamingOptions,
// type ParallelAnalysisResult
// } from '../analyzers/streaming/parallel-streaming-analyzer';
// Intelligent chunking
var intelligent_chunker_1 = require("./intelligent-chunker");
Object.defineProperty(exports, "IntelligentChunker", { enumerable: true, get: function () { return intelligent_chunker_1.IntelligentChunker; } });
Object.defineProperty(exports, "getGlobalIntelligentChunker", { enumerable: true, get: function () { return intelligent_chunker_1.getGlobalIntelligentChunker; } });
Object.defineProperty(exports, "shutdownGlobalIntelligentChunker", { enumerable: true, get: function () { return intelligent_chunker_1.shutdownGlobalIntelligentChunker; } });
// Resource pooling and GC optimization
var resource_pool_1 = require("./resource-pool");
Object.defineProperty(exports, "ResourcePool", { enumerable: true, get: function () { return resource_pool_1.ResourcePool; } });
Object.defineProperty(exports, "ResourcePoolManager", { enumerable: true, get: function () { return resource_pool_1.ResourcePoolManager; } });
Object.defineProperty(exports, "GcOptimizer", { enumerable: true, get: function () { return resource_pool_1.GcOptimizer; } });
Object.defineProperty(exports, "getGlobalResourcePoolManager", { enumerable: true, get: function () { return resource_pool_1.getGlobalResourcePoolManager; } });
Object.defineProperty(exports, "shutdownGlobalResourcePoolManager", { enumerable: true, get: function () { return resource_pool_1.shutdownGlobalResourcePoolManager; } });
Object.defineProperty(exports, "createBufferPool", { enumerable: true, get: function () { return resource_pool_1.createBufferPool; } });
Object.defineProperty(exports, "createObjectPool", { enumerable: true, get: function () { return resource_pool_1.createObjectPool; } });
// Error reduction and reliability components - Disabled due to missing exports
// export {
// WorkerHealthMonitor,
// type WorkerHealthStatus
// } from './worker-health-monitor';
var circuit_breaker_1 = require("./circuit-breaker");
Object.defineProperty(exports, "CircuitBreaker", { enumerable: true, get: function () { return circuit_breaker_1.CircuitBreaker; } });
Object.defineProperty(exports, "CircuitBreakerManager", { enumerable: true, get: function () { return circuit_breaker_1.CircuitBreakerManager; } });
Object.defineProperty(exports, "getGlobalCircuitBreakerManager", { enumerable: true, get: function () { return circuit_breaker_1.getGlobalCircuitBreakerManager; } });
Object.defineProperty(exports, "shutdownGlobalCircuitBreakerManager", { enumerable: true, get: function () { return circuit_breaker_1.shutdownGlobalCircuitBreakerManager; } });
Object.defineProperty(exports, "CircuitState", { enumerable: true, get: function () { return circuit_breaker_1.CircuitState; } });
var resource_leak_detector_1 = require("./resource-leak-detector");
Object.defineProperty(exports, "ResourceLeakDetector", { enumerable: true, get: function () { return resource_leak_detector_1.ResourceLeakDetector; } });
Object.defineProperty(exports, "getGlobalResourceLeakDetector", { enumerable: true, get: function () { return resource_leak_detector_1.getGlobalResourceLeakDetector; } });
Object.defineProperty(exports, "shutdownGlobalResourceLeakDetector", { enumerable: true, get: function () { return resource_leak_detector_1.shutdownGlobalResourceLeakDetector; } });
Object.defineProperty(exports, "trackResource", { enumerable: true, get: function () { return resource_leak_detector_1.trackResource; } });
// Memory optimizer already exported above
// Enhanced error handling
var enhanced_error_handler_1 = require("../utils/enhanced-error-handler");
Object.defineProperty(exports, "EnhancedErrorHandler", { enumerable: true, get: function () { return enhanced_error_handler_1.EnhancedErrorHandler; } });
Object.defineProperty(exports, "getGlobalEnhancedErrorHandler", { enumerable: true, get: function () { return enhanced_error_handler_1.getGlobalEnhancedErrorHandler; } });
Object.defineProperty(exports, "shutdownGlobalEnhancedErrorHandler", { enumerable: true, get: function () { return enhanced_error_handler_1.shutdownGlobalEnhancedErrorHandler; } });
Object.defineProperty(exports, "withErrorHandling", { enumerable: true, get: function () { return enhanced_error_handler_1.withErrorHandling; } });
// Input validation
var input_validator_1 = require("../utils/input-validator");
Object.defineProperty(exports, "InputValidator", { enumerable: true, get: function () { return input_validator_1.InputValidator; } });
// Format-specific optimizers - Disabled due to TypeScript compilation issues
// export {
// ParquetOptimizer,
// ExcelOptimizer,
// JsonOptimizer,
// getGlobalParquetOptimizer,
// getGlobalExcelOptimizer,
// getGlobalJsonOptimizer,
// initializeFormatOptimizers,
// shutdownAllFormatOptimizers,
// getAllFormatMetrics,
// type ParquetOptimizationOptions,
// type ExcelOptimizationOptions,
// type JsonOptimizationOptions
// } from './format-optimizers';
// Performance monitoring and adaptive configuration - Disabled due to TypeScript compilation issues
// export {
// PerformanceDashboard,
// AdaptiveConfigurationManager,
// PerformanceBenchmarkSuite,
// getGlobalPerformanceDashboard,
// getGlobalAdaptiveConfigManager,
// getGlobalBenchmarkSuite,
// runQuickBenchmark,
// initializeMonitoring,
// shutdownAllMonitoring,
// getMonitoringStatus,
// type DashboardConfig,
// type AdaptiveConfigOptions,
// type BenchmarkConfig,
// type BenchmarkReport
// } from './monitoring';
// Internal imports for functions used within this module
const memory_optimizer_2 = require("./memory-optimizer");
const enhanced_error_handler_2 = require("../utils/enhanced-error-handler");
const resource_leak_detector_2 = require("./resource-leak-detector");
const circuit_breaker_2 = require("./circuit-breaker");
const intelligent_chunker_2 = require("./intelligent-chunker");
const resource_pool_2 = require("./resource-pool");
const parallel_analyzer_2 = require("./parallel-analyzer");
const adaptive_streamer_2 = require("./adaptive-streamer");
/**
* Initialize error reduction components
*/
function initializeErrorReduction(level) {
// Always initialize enhanced error handler
(0, enhanced_error_handler_2.getGlobalEnhancedErrorHandler)({
enableRecovery: true,
maxRetries: level === 'basic' ? 2 : level === 'standard' ? 3 : 5,
trackMetrics: true,
enableCircuitBreaker: level !== 'basic',
enableResourceTracking: level === 'comprehensive'
});
// Initialize circuit breaker for standard and comprehensive levels
if (level !== 'basic') {
(0, circuit_breaker_2.getGlobalCircuitBreakerManager)();
}
// Initialize resource leak detection for comprehensive level
if (level === 'comprehensive') {
(0, resource_leak_detector_2.getGlobalResourceLeakDetector)({
trackingEnabled: true,
maxAge: 300000, // 5 minutes
checkInterval: 30000, // 30 seconds
enableStackTrace: true
});
}
}
/**
* Initialize all performance optimizations with default settings
*/
function initializePerformanceOptimizations(options = {}) {
const { enableParallelProcessing = true, enableMemoryOptimization = true, enableAdaptiveStreaming = true, enableIntelligentChunking = true, enableResourcePooling = true, enableErrorReduction = true, maxWorkers = Math.max(2, require('os').cpus().length - 1), memoryLimitMB = 512, errorReductionLevel = 'standard' } = options;
// Initialize error reduction components first
if (enableErrorReduction) {
initializeErrorReduction(errorReductionLevel);
}
if (enableMemoryOptimization) {
(0, memory_optimizer_2.getGlobalMemoryOptimizer)({
maxMemoryMB: memoryLimitMB,
enableMemoryPooling: enableResourcePooling,
adaptiveChunkSizing: enableIntelligentChunking
});
}
if (enableParallelProcessing) {
(0, parallel_analyzer_2.getGlobalParallelAnalyzer)({
maxWorkers,
memoryLimitMB: memoryLimitMB / maxWorkers,
enableMemoryMonitoring: enableMemoryOptimization
});
}
if (enableAdaptiveStreaming) {
(0, adaptive_streamer_2.getGlobalAdaptiveStreamer)({
maxConcurrentChunks: Math.min(3, maxWorkers),
memoryPressureThreshold: 0.8,
enableProgressiveLoading: true
});
}
if (enableIntelligentChunking) {
(0, intelligent_chunker_2.getGlobalIntelligentChunker)({
enableLearning: true,
adaptationSensitivity: 0.2
});
}
if (enableResourcePooling) {
(0, resource_pool_2.getGlobalResourcePoolManager)({
enableGcOptimization: true,
maxPoolSize: 50,
resourceTypes: [
{
name: 'buffer',
factory: () => Buffer.alloc(64 * 1024),
cleanup: (buffer) => buffer.fill(0),
validator: (buffer) => Buffer.isBuffer(buffer),
maxPoolSize: 100
},
{
name: 'array',
factory: () => new Array(1000),
cleanup: (array) => array.length = 0,
validator: (array) => Array.isArray(array),
maxPoolSize: 50
}
]
});
}
}
/**
* Shutdown all performance optimizations
*/
function shutdownPerformanceOptimizations() {
// shutdownGlobalParallelAnalyzer();
// shutdownGlobalMemoryOptimizer();
// shutdownGlobalAdaptiveStreamer();
// shutdownGlobalIntelligentChunker();
// shutdownGlobalResourcePoolManager();
// shutdownGlobalWorkerPool();
}
/**
* Enhanced shutdown with proper error handling
*/
async function shutdownPerformanceOptimizationsEnhanced() {
return Promise.all([
// Shutdown error reduction components first
(0, enhanced_error_handler_2.shutdownGlobalEnhancedErrorHandler)(),
(0, circuit_breaker_2.shutdownGlobalCircuitBreakerManager)(),
(0, resource_leak_detector_2.shutdownGlobalResourceLeakDetector)(),
// Then shutdown performance components
// shutdownGlobalParallelAnalyzer(),
// shutdownGlobalMemoryOptimizer(),
// shutdownGlobalAdaptiveStreamer(),
// shutdownGlobalIntelligentChunker(),
// shutdownGlobalResourcePoolManager(),
// shutdownGlobalWorkerPool()
]).then(() => {
console.log('All performance optimizations shutdown successfully');
}).catch(error => {
console.error('Error during performance optimization shutdown:', error);
throw error;
});
}
/**
* Get comprehensive performance statistics
*/
function getPerformanceStats() {
return {
parallel: (0, parallel_analyzer_2.getGlobalParallelAnalyzer)().getPerformanceStats(),
memory: (0, memory_optimizer_2.getGlobalMemoryOptimizer)().getDetailedStats(),
streaming: (0, adaptive_streamer_2.getGlobalAdaptiveStreamer)().getOverallStats(),
chunking: (0, intelligent_chunker_2.getGlobalIntelligentChunker)().getLearningStats(),
resources: (0, resource_pool_2.getGlobalResourcePoolManager)().getAllStats()
};
}
/**
* Get overall system health status
*/
function getSystemHealthStatus() {
const issues = [];
const recommendations = [];
let totalScore = 100;
try {
// Check error handler health
const errorHealth = (0, enhanced_error_handler_2.getGlobalEnhancedErrorHandler)().getHealthStatus();
if (errorHealth.status === 'unhealthy') {
totalScore -= 30;
issues.push('High error rate detected');
recommendations.push(...errorHealth.recommendations);
}
else if (errorHealth.status === 'degraded') {
totalScore -= 15;
issues.push('Elevated error rate');
}
// Check circuit breaker health
const circuitHealth = (0, circuit_breaker_2.getGlobalCircuitBreakerManager)().getSystemHealth();
if (circuitHealth.overallHealth < 70) {
totalScore -= 20;
issues.push(`${circuitHealth.openBreakers} circuit breakers open`);
recommendations.push('Investigate failing operations');
}
// Check resource leak status
const resourceStats = (0, resource_leak_detector_2.getGlobalResourceLeakDetector)().getResourceStats();
if (resourceStats.potentialLeaks > 10) {
totalScore -= 25;
issues.push(`${resourceStats.potentialLeaks} potential resource leaks`);
recommendations.push('Review resource cleanup patterns');
}
// Check memory pressure
const memoryStats = (0, memory_optimizer_2.getGlobalMemoryOptimizer)().getDetailedStats();
if (memoryStats.pressure.level > 0.8) {
totalScore -= 20;
issues.push('High memory pressure');
recommendations.push('Consider reducing memory usage or increasing limits');
}
}
catch (error) {
totalScore -= 50;
issues.push('Error collecting system health metrics');
recommendations.push('Check system component initialization');
}
let status = 'healthy';
if (totalScore < 50) {
status = 'unhealthy';
}
else if (totalScore < 80) {
status = 'degraded';
}
return {
status,
score: Math.max(0, totalScore),
issues,
recommendations
};
}
/**
* Emergency system recovery function
*/
async function emergencySystemRecovery() {
const actions = [];
const errors = [];
try {
// Force all circuit breakers to closed state
const circuitBreaker = (0, circuit_breaker_2.getGlobalCircuitBreakerManager)();
circuitBreaker.forceAllClosed();
actions.push('Reset all circuit breakers to closed state');
// Force garbage collection
if (global.gc) {
global.gc();
actions.push('Forced garbage collection');
}
// Clean up resource leaks
const leakDetector = (0, resource_leak_detector_2.getGlobalResourceLeakDetector)();
const cleanedCount = leakDetector.forceCleanupAll();
actions.push(`Cleaned up ${cleanedCount} tracked resources`);
// Reset error metrics
const errorHandler = (0, enhanced_error_handler_2.getGlobalEnhancedErrorHandler)();
errorHandler.resetMetrics();
actions.push('Reset error handler metrics');
// Clear memory optimizer buffers
const memoryOptimizer = (0, memory_optimizer_2.getGlobalMemoryOptimizer)();
memoryOptimizer.forceGarbageCollection();
actions.push('Cleared memory optimizer buffers');
return { success: true, actions, errors };
}
catch (error) {
errors.push(`Recovery action failed: ${error.message}`);
return { success: false, actions, errors };
}
}
//# sourceMappingURL=index.js.map