datapilot-cli
Version:
Enterprise-grade streaming multi-format data analysis with comprehensive statistical insights and intelligent relationship detection - supports CSV, JSON, Excel, TSV, Parquet - memory-efficient, cross-platform
513 lines • 20.3 kB
JavaScript
"use strict";
/**
* Performance Monitoring and Adaptive Thresholds System
* Provides real-time performance tracking and dynamic configuration adjustment
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.ResourceDetector = exports.PerformanceMonitor = void 0;
exports.getPerformanceMonitor = getPerformanceMonitor;
exports.monitorPerformance = monitorPerformance;
const config_1 = require("./config");
const logger_1 = require("../utils/logger");
/**
* Performance Monitor with Adaptive Configuration Management
*/
class PerformanceMonitor {
static instance;
metrics = [];
alerts = [];
adaptiveThresholds = new Map();
monitoringInterval;
isMonitoring = false;
// Performance tracking
startTime = 0;
operationCount = 0;
rowsProcessed = 0;
errorsEncountered = 0;
// Adaptive configuration
autoAdaptEnabled = true;
adaptationHistory = [];
constructor() {
this.initializeAdaptiveThresholds();
}
static getInstance() {
if (!PerformanceMonitor.instance) {
PerformanceMonitor.instance = new PerformanceMonitor();
}
return PerformanceMonitor.instance;
}
/**
* Initialize adaptive thresholds with default values
*/
initializeAdaptiveThresholds() {
const config = (0, config_1.getConfig)().getConfig();
const thresholds = [
{ name: 'maxRows', value: config.performance.maxRows },
{ name: 'chunkSize', value: config.performance.chunkSize },
{ name: 'memoryThresholdMB', value: config.streaming.memoryThresholdMB },
{ name: 'maxCorrelationPairs', value: config.analysis.maxCorrelationPairs },
{ name: 'significanceLevel', value: config.statistical.significanceLevel },
{ name: 'samplingThreshold', value: config.analysis.samplingThreshold },
];
thresholds.forEach(({ name, value }) => {
this.adaptiveThresholds.set(name, {
name,
currentValue: value,
defaultValue: value,
adaptedValue: value,
lastAdjustment: Date.now(),
adjustmentReason: 'Initial value',
});
});
}
/**
* Start performance monitoring
*/
startMonitoring(intervalMs = 5000) {
if (this.isMonitoring) {
return;
}
this.isMonitoring = true;
this.startTime = Date.now();
this.monitoringInterval = setInterval(() => {
this.collectMetrics();
this.analyzePerformance();
this.adaptThresholds();
}, intervalMs);
logger_1.logger.info('Performance monitoring started');
}
/**
* Stop performance monitoring
*/
stopMonitoring() {
if (this.monitoringInterval) {
clearInterval(this.monitoringInterval);
this.monitoringInterval = undefined;
}
this.isMonitoring = false;
logger_1.logger.info('Performance monitoring stopped');
}
/**
* Collect current performance metrics
*/
collectMetrics() {
const memUsage = process.memoryUsage();
const cpuUsage = process.cpuUsage();
const metrics = {
timestamp: Date.now(),
memoryUsageMB: Math.round(memUsage.heapUsed / (1024 * 1024)),
heapUsedMB: Math.round(memUsage.heapUsed / (1024 * 1024)),
heapTotalMB: Math.round(memUsage.heapTotal / (1024 * 1024)),
externalMB: Math.round(memUsage.external / (1024 * 1024)),
rss: Math.round(memUsage.rss / (1024 * 1024)),
cpuTime: (cpuUsage.user + cpuUsage.system) / 1000000, // Convert to seconds
};
// Calculate processing rates
const elapsedSeconds = (Date.now() - this.startTime) / 1000;
if (elapsedSeconds > 0) {
metrics.processingRate = this.rowsProcessed / elapsedSeconds;
metrics.throughput = this.operationCount / elapsedSeconds;
}
this.metrics.push(metrics);
// Keep only last 100 metrics
if (this.metrics.length > 100) {
this.metrics = this.metrics.slice(-100);
}
}
/**
* Analyze performance and generate alerts
*/
analyzePerformance() {
if (this.metrics.length === 0)
return;
const currentMetrics = this.metrics[this.metrics.length - 1];
const config = (0, config_1.getConfig)().getConfig();
// Memory usage alerts
if (currentMetrics.memoryUsageMB > config.streaming.memoryThresholdMB * 0.9) {
this.createAlert({
type: 'memory',
severity: currentMetrics.memoryUsageMB > config.streaming.memoryThresholdMB ? 'critical' : 'high',
message: `High memory usage: ${currentMetrics.memoryUsageMB}MB`,
metrics: currentMetrics,
recommendation: 'Consider reducing chunk size or enabling more aggressive memory cleanup',
timestamp: Date.now(),
});
}
// Throughput degradation alerts
if (this.metrics.length >= 5) {
const recentMetrics = this.metrics.slice(-5);
const avgThroughput = recentMetrics.reduce((sum, m) => sum + (m.throughput || 0), 0) / recentMetrics.length;
const firstThroughput = recentMetrics[0].throughput || 0;
if (firstThroughput > 0 && avgThroughput < firstThroughput * 0.5) {
this.createAlert({
type: 'throughput',
severity: 'medium',
message: `Throughput degraded by ${Math.round((1 - avgThroughput / firstThroughput) * 100)}%`,
metrics: currentMetrics,
recommendation: 'Performance may be degrading due to memory pressure or CPU load',
timestamp: Date.now(),
});
}
}
// Error rate monitoring
const errorRate = this.operationCount > 0 ? this.errorsEncountered / this.operationCount : 0;
if (errorRate > 0.05) {
// 5% error rate threshold
this.createAlert({
type: 'error_rate',
severity: errorRate > 0.2 ? 'critical' : 'high',
message: `High error rate: ${(errorRate * 100).toFixed(1)}%`,
metrics: currentMetrics,
recommendation: 'Check data quality or reduce processing complexity',
timestamp: Date.now(),
});
}
}
/**
* Adaptively adjust thresholds based on performance
*/
adaptThresholds() {
if (!this.autoAdaptEnabled || this.metrics.length < 3) {
return;
}
const currentMetrics = this.metrics[this.metrics.length - 1];
const config = (0, config_1.getConfig)();
// Memory-based adaptations
if (currentMetrics.memoryUsageMB > config.getStreamingConfig().memoryThresholdMB * 0.8) {
// Reduce chunk size for memory efficiency
this.adaptThreshold('chunkSize', (current) => Math.max(1000, Math.floor(current * 0.8)), 'High memory usage detected');
// Reduce correlation pairs
this.adaptThreshold('maxCorrelationPairs', (current) => Math.max(20, Math.floor(current * 0.8)), 'Memory optimization');
}
else if (currentMetrics.memoryUsageMB < config.getStreamingConfig().memoryThresholdMB * 0.3) {
// Increase processing capacity when memory is available
this.adaptThreshold('chunkSize', (current) => Math.min(32768, Math.floor(current * 1.2)), 'Low memory usage - increasing efficiency');
}
// Throughput-based adaptations
if (currentMetrics.processingRate && currentMetrics.processingRate < 1000) {
// Less than 1K rows/sec
this.adaptThreshold('significanceLevel', (current) => Math.min(0.1, current * 1.5), 'Low throughput - relaxing statistical rigor');
this.adaptThreshold('samplingThreshold', (current) => Math.max(1000, Math.floor(current * 0.7)), 'Low throughput - increasing sampling');
}
// Apply adaptations to configuration
this.applyAdaptiveConfiguration();
}
/**
* Adapt a specific threshold
*/
adaptThreshold(name, adaptFunction, reason) {
const threshold = this.adaptiveThresholds.get(name);
if (!threshold)
return;
const newValue = adaptFunction(threshold.currentValue);
if (newValue !== threshold.currentValue) {
this.adaptationHistory.push({
timestamp: Date.now(),
threshold: name,
oldValue: threshold.currentValue,
newValue,
reason,
});
threshold.adaptedValue = newValue;
threshold.currentValue = newValue;
threshold.lastAdjustment = Date.now();
threshold.adjustmentReason = reason;
logger_1.logger.info(`Adapted ${name}: ${threshold.adaptedValue} (${reason})`);
}
}
/**
* Apply adaptive configuration to the global config
*/
applyAdaptiveConfiguration() {
const configManager = (0, config_1.getConfig)();
const adaptiveUpdates = {};
// Apply performance adaptations
const chunkSize = this.adaptiveThresholds.get('chunkSize');
const memoryThreshold = this.adaptiveThresholds.get('memoryThresholdMB');
if (chunkSize || memoryThreshold) {
const baseConfig = configManager.getConfig();
adaptiveUpdates.performance = { ...baseConfig.performance };
if (chunkSize) {
adaptiveUpdates.performance.chunkSize = chunkSize.currentValue;
}
}
// Apply streaming adaptations
if (memoryThreshold) {
const baseConfig = configManager.getConfig();
adaptiveUpdates.streaming = {
...baseConfig.streaming,
memoryThresholdMB: memoryThreshold.currentValue,
};
}
// Apply analysis adaptations
const maxCorrelationPairs = this.adaptiveThresholds.get('maxCorrelationPairs');
const samplingThreshold = this.adaptiveThresholds.get('samplingThreshold');
if (maxCorrelationPairs || samplingThreshold) {
const baseConfig = configManager.getConfig();
adaptiveUpdates.analysis = { ...baseConfig.analysis };
if (maxCorrelationPairs) {
adaptiveUpdates.analysis.maxCorrelationPairs = maxCorrelationPairs.currentValue;
}
if (samplingThreshold) {
adaptiveUpdates.analysis.samplingThreshold = samplingThreshold.currentValue;
}
}
// Apply statistical adaptations
const significanceLevel = this.adaptiveThresholds.get('significanceLevel');
if (significanceLevel) {
const baseConfig = configManager.getConfig();
adaptiveUpdates.statistical = {
...baseConfig.statistical,
significanceLevel: significanceLevel.currentValue,
};
}
if (Object.keys(adaptiveUpdates).length > 0) {
configManager.updateConfig(adaptiveUpdates);
}
}
/**
* Create a performance alert
*/
createAlert(alert) {
this.alerts.push(alert);
// Keep only last 50 alerts
if (this.alerts.length > 50) {
this.alerts = this.alerts.slice(-50);
}
// Log critical alerts
if (alert.severity === 'critical' || alert.severity === 'high') {
logger_1.logger.warn(`Performance Alert [${alert.severity.toUpperCase()}]: ${alert.message}`);
}
}
/**
* Record operation metrics
*/
recordOperation(type, count = 1) {
switch (type) {
case 'row':
this.rowsProcessed += count;
break;
case 'operation':
this.operationCount += count;
break;
case 'error':
this.errorsEncountered += count;
break;
}
}
/**
* Get current performance summary
*/
getPerformanceSummary() {
const currentMetrics = this.metrics.length > 0 ? this.metrics[this.metrics.length - 1] : null;
const recentAlerts = this.alerts.slice(-10);
const adaptiveThresholds = Array.from(this.adaptiveThresholds.values());
return {
currentMetrics,
recentAlerts,
adaptiveThresholds,
adaptationHistory: [...this.adaptationHistory],
operationalMetrics: {
rowsProcessed: this.rowsProcessed,
operationCount: this.operationCount,
errorsEncountered: this.errorsEncountered,
errorRate: this.operationCount > 0 ? this.errorsEncountered / this.operationCount : 0,
elapsedTimeMs: Date.now() - this.startTime,
},
};
}
/**
* Reset monitoring state
*/
reset() {
this.metrics = [];
this.alerts = [];
this.adaptationHistory = [];
this.startTime = Date.now();
this.operationCount = 0;
this.rowsProcessed = 0;
this.errorsEncountered = 0;
this.initializeAdaptiveThresholds();
}
/**
* Enable/disable auto-adaptation
*/
setAutoAdaptation(enabled) {
this.autoAdaptEnabled = enabled;
logger_1.logger.info(`Auto-adaptation ${enabled ? 'enabled' : 'disabled'}`);
}
/**
* Get adaptive threshold value
*/
getAdaptiveThreshold(name) {
return this.adaptiveThresholds.get(name)?.currentValue;
}
/**
* Manually adjust threshold
*/
setThreshold(name, value, reason = 'Manual adjustment') {
const threshold = this.adaptiveThresholds.get(name);
if (threshold) {
this.adaptThreshold(name, () => value, reason);
this.applyAdaptiveConfiguration();
}
}
}
exports.PerformanceMonitor = PerformanceMonitor;
/**
* Convenience function to get performance monitor instance
*/
function getPerformanceMonitor() {
return PerformanceMonitor.getInstance();
}
/**
* Performance monitoring decorator for methods
*/
function monitorPerformance(target, propertyName, descriptor) {
const method = descriptor.value;
descriptor.value = function (...args) {
const monitor = getPerformanceMonitor();
try {
monitor.recordOperation('operation');
const result = method.apply(this, args);
// Handle async methods
if (result && typeof result === 'object' && 'then' in result && typeof result.then === 'function') {
return result.catch((error) => {
monitor.recordOperation('error');
throw error;
});
}
return result;
}
catch (error) {
monitor.recordOperation('error');
throw error;
}
};
}
/**
* System resource detection for adaptive configuration
*/
class ResourceDetector {
/**
* Detect available system resources
*/
static detectSystemResources() {
const memUsage = process.memoryUsage();
const totalMemoryMB = Math.round(memUsage.rss / (1024 * 1024));
const availableMemoryMB = Math.round((process.memoryUsage().heapTotal - process.memoryUsage().heapUsed) / (1024 * 1024));
// Estimate total system memory (rough approximation)
const estimatedTotalMemoryMB = totalMemoryMB * 4; // Assuming process uses ~25% of system memory
let recommendedConfig = {};
// Low memory system (< 1GB available)
if (availableMemoryMB < 1024) {
recommendedConfig = {
performance: {
maxRows: 50000,
maxFieldSize: 512 * 1024,
memoryThresholdBytes: 256 * 1024 * 1024, // 256MB
chunkSize: 8 * 1024,
sampleSize: 512 * 1024,
adaptiveChunkSizing: true,
maxCollectedRowsMultivariate: 500,
batchSize: 100,
performanceMonitoringInterval: 10,
memoryCleanupInterval: 20,
emergencyMemoryThresholdMultiplier: 1.5,
},
streaming: {
memoryThresholdMB: 50,
maxRowsAnalyzed: 50000,
adaptiveChunkSizing: {
enabled: true,
minChunkSize: 50,
maxChunkSize: 1000,
reductionFactor: 0.5,
expansionFactor: 1.1,
targetMemoryUtilization: 0.7,
},
memoryManagement: {
cleanupInterval: 10,
emergencyThresholdMultiplier: 1.2,
forceGarbageCollection: true,
gcFrequency: 500,
memoryLeakDetection: true,
autoGarbageCollect: true,
},
},
analysis: {
maxCategoricalLevels: 20,
maxCorrelationPairs: 20,
samplingThreshold: 5000,
outlierMethods: ['iqr'],
normalityTests: ['shapiro'],
enableMultivariate: false,
enabledAnalyses: ['univariate'],
highCardinalityThreshold: 80,
missingValueQualityThreshold: 20,
multivariateThreshold: 500,
maxDimensionsForPCA: 3,
clusteringMethods: ['kmeans'],
},
};
}
// Medium memory system (1-4GB available)
else if (availableMemoryMB < 4096) {
const configManager = (0, config_1.getConfig)();
const baseConfig = configManager.getConfig();
recommendedConfig = {
performance: {
...baseConfig.performance,
maxRows: 200000,
chunkSize: 32 * 1024,
batchSize: 500,
maxCollectedRowsMultivariate: 1000,
memoryThresholdBytes: 512 * 1024 * 1024, // 512MB
},
streaming: {
...baseConfig.streaming,
memoryThresholdMB: 100,
maxRowsAnalyzed: 200000,
},
analysis: {
...baseConfig.analysis,
maxCorrelationPairs: 50,
samplingThreshold: 10000,
},
};
}
// High memory system (4GB+ available)
else {
const configManager = (0, config_1.getConfig)();
const baseConfig = configManager.getConfig();
recommendedConfig = {
performance: {
...baseConfig.performance,
maxRows: 1000000,
chunkSize: 64 * 1024,
batchSize: 1000,
maxCollectedRowsMultivariate: 2000,
memoryThresholdBytes: 1024 * 1024 * 1024, // 1GB
},
streaming: {
...baseConfig.streaming,
memoryThresholdMB: 200,
maxRowsAnalyzed: 1000000,
},
analysis: {
...baseConfig.analysis,
maxCorrelationPairs: 100,
samplingThreshold: 20000,
},
};
}
return {
availableMemoryMB,
totalMemoryMB: estimatedTotalMemoryMB,
cpuCores: 1, // Node.js doesn't easily expose this
recommendedConfig,
};
}
}
exports.ResourceDetector = ResourceDetector;
//# sourceMappingURL=performance-monitor.js.map