datapilot-cli
Version:
Enterprise-grade streaming multi-format data analysis with comprehensive statistical insights and intelligent relationship detection - supports CSV, JSON, Excel, TSV, Parquet - memory-efficient, cross-platform
386 lines • 13.6 kB
JavaScript
"use strict";
/**
* Memory Optimizer
* Advanced memory management for streaming data analysis
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.MemoryOptimizer = void 0;
exports.getGlobalMemoryOptimizer = getGlobalMemoryOptimizer;
exports.shutdownGlobalMemoryOptimizer = shutdownGlobalMemoryOptimizer;
exports.withMemoryOptimization = withMemoryOptimization;
const events_1 = require("events");
const logger_1 = require("../utils/logger");
/**
* Buffer pool for memory reuse
*/
class BufferPool {
pools = new Map();
maxPoolSize;
totalBuffersCreated = 0;
totalBuffersReused = 0;
constructor(maxPoolSize = 50) {
this.maxPoolSize = maxPoolSize;
}
/**
* Get buffer from pool or create new one
*/
getBuffer(size) {
const roundedSize = this.roundToStandardSize(size);
const pool = this.pools.get(roundedSize);
if (pool && pool.length > 0) {
this.totalBuffersReused++;
return pool.pop();
}
this.totalBuffersCreated++;
return Buffer.alloc(roundedSize);
}
/**
* Return buffer to pool
*/
returnBuffer(buffer) {
const size = buffer.length;
const roundedSize = this.roundToStandardSize(size);
if (!this.pools.has(roundedSize)) {
this.pools.set(roundedSize, []);
}
const pool = this.pools.get(roundedSize);
if (pool.length < this.maxPoolSize) {
// Clear buffer before returning to pool
buffer.fill(0);
pool.push(buffer);
}
}
/**
* Round buffer size to standard sizes for better pooling
*/
roundToStandardSize(size) {
const standardSizes = [
1024, // 1KB
4096, // 4KB
16384, // 16KB
65536, // 64KB
262144, // 256KB
1048576, // 1MB
4194304, // 4MB
16777216, // 16MB
67108864, // 64MB
];
return standardSizes.find((s) => s >= size) || size;
}
/**
* Clear all pools and release memory
*/
clear() {
this.pools.clear();
}
/**
* Get pool statistics
*/
getStats() {
const totalBuffersInPool = Array.from(this.pools.values()).reduce((sum, pool) => sum + pool.length, 0);
return {
totalBuffersCreated: this.totalBuffersCreated,
totalBuffersReused: this.totalBuffersReused,
reuseRate: this.totalBuffersCreated > 0
? ((this.totalBuffersReused / this.totalBuffersCreated) * 100).toFixed(2) + '%'
: '0%',
buffersInPool: totalBuffersInPool,
poolSizes: Object.fromEntries(Array.from(this.pools.entries()).map(([size, pool]) => [size, pool.length])),
};
}
}
/**
* Advanced memory optimizer with adaptive streaming
*/
class MemoryOptimizer extends events_1.EventEmitter {
options;
bufferPool;
memoryCheckTimer;
memoryHistory = [];
isMonitoring = false;
lastGcTime = 0;
baselineMemory;
constructor(options = {}) {
super();
this.options = {
maxMemoryMB: options.maxMemoryMB || 512,
gcThresholdMB: options.gcThresholdMB || 256,
memoryCheckInterval: options.memoryCheckInterval || 1000,
enableMemoryPooling: options.enableMemoryPooling ?? true,
bufferPoolSize: options.bufferPoolSize || 100,
adaptiveChunkSizing: options.adaptiveChunkSizing ?? true,
memoryPressureThreshold: options.memoryPressureThreshold || 0.8,
};
this.bufferPool = new BufferPool(this.options.bufferPoolSize);
this.baselineMemory = this.getCurrentMemoryStats();
logger_1.logger.info(`Memory optimizer initialized with ${this.options.maxMemoryMB}MB limit`);
}
/**
* Start memory monitoring
*/
startMonitoring() {
if (this.isMonitoring)
return;
this.isMonitoring = true;
this.memoryCheckTimer = setInterval(() => {
this.checkMemoryUsage();
}, this.options.memoryCheckInterval);
logger_1.logger.info('Memory monitoring started');
}
/**
* Stop memory monitoring
*/
stopMonitoring() {
if (!this.isMonitoring)
return;
this.isMonitoring = false;
if (this.memoryCheckTimer) {
clearInterval(this.memoryCheckTimer);
this.memoryCheckTimer = undefined;
}
logger_1.logger.info('Memory monitoring stopped');
}
/**
* Get current memory statistics
*/
getCurrentMemoryStats() {
const usage = process.memoryUsage();
return {
heapUsed: usage.heapUsed / 1024 / 1024, // MB
heapTotal: usage.heapTotal / 1024 / 1024, // MB
external: usage.external / 1024 / 1024, // MB
rss: usage.rss / 1024 / 1024, // MB
arrayBuffers: usage.arrayBuffers / 1024 / 1024, // MB
};
}
/**
* Calculate memory pressure (0-1 scale)
*/
getMemoryPressure() {
const current = this.getCurrentMemoryStats();
const pressure = current.heapUsed / this.options.maxMemoryMB;
return Math.min(1, Math.max(0, pressure));
}
/**
* Get adaptive chunk size recommendation
*/
getAdaptiveChunkSize(baseChunkSize, dataComplexity = 1) {
if (!this.options.adaptiveChunkSizing) {
return {
recommendedSize: baseChunkSize,
reason: 'Adaptive sizing disabled',
memoryPressure: this.getMemoryPressure(),
adaptationFactor: 1,
};
}
const memoryPressure = this.getMemoryPressure();
const memoryTrend = this.calculateMemoryTrend();
// Base adaptation factor on memory pressure and trend
let adaptationFactor = 1;
let reason = 'Normal memory conditions';
if (memoryPressure > this.options.memoryPressureThreshold) {
// High memory pressure - reduce chunk size
adaptationFactor = Math.max(0.25, 1 - (memoryPressure - this.options.memoryPressureThreshold) * 2);
reason = `High memory pressure (${(memoryPressure * 100).toFixed(1)}%)`;
}
else if (memoryTrend > 0.1) {
// Memory growing rapidly - be conservative
adaptationFactor = Math.max(0.5, 1 - memoryTrend);
reason = `Memory trending upward (${(memoryTrend * 100).toFixed(1)}%/sec)`;
}
else if (memoryPressure < 0.3 && memoryTrend < 0.05) {
// Low memory pressure and stable - can increase chunk size
adaptationFactor = Math.min(2, 1 + (0.3 - memoryPressure));
reason = `Low memory pressure, increasing efficiency`;
}
// Apply data complexity factor
adaptationFactor /= Math.max(1, dataComplexity);
const recommendedSize = Math.max(1024, // Minimum 1KB
Math.min(64 * 1024 * 1024, // Maximum 64MB
Math.round(baseChunkSize * adaptationFactor)));
return {
recommendedSize,
reason,
memoryPressure,
adaptationFactor,
};
}
/**
* Get or create buffer with memory pooling
*/
getBuffer(size) {
if (this.options.enableMemoryPooling) {
return this.bufferPool.getBuffer(size);
}
return Buffer.alloc(size);
}
/**
* Return buffer to pool
*/
returnBuffer(buffer) {
if (this.options.enableMemoryPooling) {
this.bufferPool.returnBuffer(buffer);
}
}
/**
* Force garbage collection if possible and beneficial
*/
forceGarbageCollection() {
const now = Date.now();
const timeSinceLastGc = now - this.lastGcTime;
// Only run GC if enough time has passed and memory pressure is high
if (timeSinceLastGc < 5000 || this.getMemoryPressure() < 0.6) {
return false;
}
try {
if (global.gc) {
const beforeGc = this.getCurrentMemoryStats();
global.gc();
const afterGc = this.getCurrentMemoryStats();
this.lastGcTime = now;
const memoryFreed = beforeGc.heapUsed - afterGc.heapUsed;
logger_1.logger.info(`Garbage collection freed ${memoryFreed.toFixed(2)}MB`);
this.emit('gc-completed', { memoryFreed, beforeGc, afterGc });
return true;
}
}
catch (error) {
logger_1.logger.warn(`Garbage collection failed: ${error.message}`);
}
return false;
}
/**
* Check memory usage and take action if needed
*/
checkMemoryUsage() {
const current = this.getCurrentMemoryStats();
this.memoryHistory.push(current);
// Keep only last 60 readings (1 minute at 1 second intervals)
if (this.memoryHistory.length > 60) {
this.memoryHistory.shift();
}
const memoryPressure = this.getMemoryPressure();
// Emit memory pressure events
if (memoryPressure > 0.9) {
this.emit('memory-critical', { pressure: memoryPressure, stats: current });
}
else if (memoryPressure > this.options.memoryPressureThreshold) {
this.emit('memory-pressure', { pressure: memoryPressure, stats: current });
}
// Auto-trigger GC if memory usage is high
if (current.heapUsed > this.options.gcThresholdMB) {
this.forceGarbageCollection();
}
// Clear buffer pools if memory pressure is very high
if (memoryPressure > 0.95) {
this.bufferPool.clear();
logger_1.logger.warn('Cleared buffer pools due to extreme memory pressure');
}
}
/**
* Calculate memory usage trend (MB per second)
*/
calculateMemoryTrend() {
if (this.memoryHistory.length < 10)
return 0;
const recent = this.memoryHistory.slice(-10);
const timeSpan = (recent.length - 1) * (this.options.memoryCheckInterval / 1000);
const memoryChange = recent[recent.length - 1].heapUsed - recent[0].heapUsed;
return memoryChange / timeSpan; // MB per second
}
/**
* Get comprehensive memory statistics
*/
getDetailedStats() {
const current = this.getCurrentMemoryStats();
const memoryPressure = this.getMemoryPressure();
const memoryTrend = this.calculateMemoryTrend();
return {
current,
baseline: this.baselineMemory,
memoryGrowth: {
heapUsed: current.heapUsed - this.baselineMemory.heapUsed,
heapTotal: current.heapTotal - this.baselineMemory.heapTotal,
rss: current.rss - this.baselineMemory.rss,
},
pressure: {
level: memoryPressure,
threshold: this.options.memoryPressureThreshold,
status: memoryPressure > 0.9
? 'critical'
: memoryPressure > this.options.memoryPressureThreshold
? 'high'
: 'normal',
},
trend: {
mbPerSecond: memoryTrend,
direction: memoryTrend > 0.1 ? 'increasing' : memoryTrend < -0.1 ? 'decreasing' : 'stable',
},
bufferPool: this.options.enableMemoryPooling ? this.bufferPool.getStats() : null,
monitoring: {
isActive: this.isMonitoring,
interval: this.options.memoryCheckInterval,
historySize: this.memoryHistory.length,
},
};
}
/**
* Clean up resources
*/
cleanup() {
this.stopMonitoring();
this.bufferPool.clear();
this.memoryHistory = [];
this.emit('cleanup-completed');
logger_1.logger.info('Memory optimizer cleanup completed');
}
}
exports.MemoryOptimizer = MemoryOptimizer;
/**
* Global memory optimizer instance
*/
let globalMemoryOptimizer = null;
/**
* Get or create global memory optimizer
*/
function getGlobalMemoryOptimizer(options) {
if (!globalMemoryOptimizer) {
globalMemoryOptimizer = new MemoryOptimizer(options);
globalMemoryOptimizer.startMonitoring();
}
return globalMemoryOptimizer;
}
/**
* Shutdown global memory optimizer
*/
function shutdownGlobalMemoryOptimizer() {
if (globalMemoryOptimizer) {
globalMemoryOptimizer.cleanup();
globalMemoryOptimizer = null;
}
}
/**
* Memory optimization decorator for async functions
*/
function withMemoryOptimization(fn, options) {
return (async (...args) => {
const optimizer = getGlobalMemoryOptimizer();
const initialPressure = optimizer.getMemoryPressure();
try {
const result = await fn(...args);
// Optionally trigger GC after operation
if (options?.enableGc && optimizer.getMemoryPressure() > (options.memoryThreshold || 0.7)) {
optimizer.forceGarbageCollection();
}
return result;
}
catch (error) {
// Force cleanup on error
if (optimizer.getMemoryPressure() > initialPressure + 0.2) {
optimizer.forceGarbageCollection();
}
throw error;
}
});
}
//# sourceMappingURL=memory-optimizer.js.map