datapilot-cli
Version:
Enterprise-grade streaming multi-format data analysis with comprehensive statistical insights and intelligent relationship detection - supports CSV, JSON, Excel, TSV, Parquet - memory-efficient, cross-platform
498 lines • 16.8 kB
JavaScript
"use strict";
/**
* Memory management and cleanup utilities for DataPilot
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.globalCleanupHandler = exports.ProcessCleanupHandler = exports.globalResourceManager = exports.ResourceManager = exports.globalMemoryManager = exports.MemoryManager = void 0;
const events_1 = require("events");
const types_1 = require("../core/types");
const logger_1 = require("./logger");
class MemoryManager extends events_1.EventEmitter {
config;
monitoringTimer;
cleanupCallbacks = [];
isMonitoring = false;
lastMemoryStats;
memoryHistory = [];
maxHistorySize = 100;
constructor(config = {}) {
super();
this.config = {
thresholds: {
warningMB: 256,
criticalMB: 512,
maxMB: 1024,
},
monitoringInterval: 5000, // 5 seconds
enableAutomaticCleanup: true,
enableGarbageCollection: true,
logMemoryUsage: false,
...config,
};
// Set up event listeners
this.on('warning', this.handleMemoryWarning.bind(this));
this.on('critical', this.handleMemoryCritical.bind(this));
this.on('max', this.handleMemoryMax.bind(this));
}
/**
* Start memory monitoring
*/
startMonitoring(context) {
if (this.isMonitoring) {
return;
}
this.isMonitoring = true;
logger_1.logger.debug('Starting memory monitoring', context);
this.monitoringTimer = setInterval(() => {
try {
this.checkMemoryUsage(context);
}
catch (error) {
logger_1.logger.error('Error during memory monitoring', context, error);
}
}, this.config.monitoringInterval);
// Initial check
this.checkMemoryUsage(context);
}
/**
* Stop memory monitoring
*/
stopMonitoring(context) {
if (!this.isMonitoring) {
return;
}
this.isMonitoring = false;
logger_1.logger.debug('Stopping memory monitoring', context);
if (this.monitoringTimer) {
clearInterval(this.monitoringTimer);
this.monitoringTimer = undefined;
}
}
/**
* Check current memory usage and emit events if thresholds are exceeded
*/
checkMemoryUsage(context) {
const memoryUsage = process.memoryUsage();
const stats = {
heapUsed: memoryUsage.heapUsed,
heapTotal: memoryUsage.heapTotal,
external: memoryUsage.external,
rss: memoryUsage.rss,
arrayBuffers: memoryUsage.arrayBuffers || 0,
};
// Add to history
this.memoryHistory.push(stats);
if (this.memoryHistory.length > this.maxHistorySize) {
this.memoryHistory = this.memoryHistory.slice(-this.maxHistorySize);
}
const heapMB = stats.heapUsed / (1024 * 1024);
const rssMB = stats.rss / (1024 * 1024);
const totalMB = Math.max(heapMB, rssMB);
// Log memory usage if enabled
if (this.config.logMemoryUsage) {
logger_1.logger.trace(`Memory usage: ${heapMB.toFixed(1)}MB heap, ${rssMB.toFixed(1)}MB RSS`, {
...context,
memoryUsage: stats.heapUsed,
});
}
// Check thresholds and emit events
if (totalMB >= this.config.thresholds.maxMB) {
this.emit('max', stats, context);
}
else if (totalMB >= this.config.thresholds.criticalMB) {
this.emit('critical', stats, context);
}
else if (totalMB >= this.config.thresholds.warningMB) {
this.emit('warning', stats, context);
}
this.lastMemoryStats = stats;
return stats;
}
/**
* Register cleanup callback
*/
registerCleanupCallback(callback) {
this.cleanupCallbacks.push(callback);
}
/**
* Force garbage collection if available
*/
forceGarbageCollection(context) {
if (!this.config.enableGarbageCollection) {
return;
}
if (global.gc) {
logger_1.logger.debug('Forcing garbage collection', context);
const before = process.memoryUsage().heapUsed;
global.gc();
const after = process.memoryUsage().heapUsed;
const freedMB = (before - after) / (1024 * 1024);
if (freedMB > 1) {
logger_1.logger.debug(`Garbage collection freed ${freedMB.toFixed(1)}MB`, context);
}
}
else {
logger_1.logger.debug('Garbage collection not available (run with --expose-gc)', context);
}
}
/**
* Run cleanup callbacks
*/
runCleanup(context) {
if (!this.config.enableAutomaticCleanup) {
return;
}
logger_1.logger.debug(`Running ${this.cleanupCallbacks.length} cleanup callbacks`, context);
for (const callback of this.cleanupCallbacks) {
try {
callback();
}
catch (error) {
logger_1.logger.error('Error in cleanup callback', context, error);
}
}
}
/**
* Get memory statistics
*/
getMemoryStats() {
if (this.memoryHistory.length === 0) {
return {
current: this.lastMemoryStats,
peak: undefined,
average: undefined,
history: [],
};
}
const peak = this.memoryHistory.reduce((max, stats) => stats.heapUsed > max.heapUsed ? stats : max);
const average = {
heapUsed: this.memoryHistory.reduce((sum, stats) => sum + stats.heapUsed, 0) /
this.memoryHistory.length,
heapTotal: this.memoryHistory.reduce((sum, stats) => sum + stats.heapTotal, 0) /
this.memoryHistory.length,
external: this.memoryHistory.reduce((sum, stats) => sum + stats.external, 0) /
this.memoryHistory.length,
rss: this.memoryHistory.reduce((sum, stats) => sum + stats.rss, 0) / this.memoryHistory.length,
arrayBuffers: this.memoryHistory.reduce((sum, stats) => sum + stats.arrayBuffers, 0) /
this.memoryHistory.length,
};
return {
current: this.lastMemoryStats,
peak,
average,
history: [...this.memoryHistory],
};
}
/**
* Get memory growth rate
*/
getMemoryGrowthRate() {
if (this.memoryHistory.length < 2) {
return undefined;
}
const recent = this.memoryHistory.slice(-10); // Last 10 measurements
if (recent.length < 2) {
return undefined;
}
const first = recent[0];
const last = recent[recent.length - 1];
const timeDiff = recent.length * this.config.monitoringInterval; // Approximate time diff
const memoryDiff = last.heapUsed - first.heapUsed;
return memoryDiff / timeDiff; // Bytes per millisecond
}
/**
* Predict memory exhaustion time
*/
predictMemoryExhaustion() {
const growthRate = this.getMemoryGrowthRate();
if (!growthRate || growthRate <= 0 || !this.lastMemoryStats) {
return undefined;
}
const currentMB = this.lastMemoryStats.heapUsed / (1024 * 1024);
const remainingMB = this.config.thresholds.maxMB - currentMB;
const remainingBytes = remainingMB * 1024 * 1024;
const timeToExhaustionMs = remainingBytes / growthRate;
if (timeToExhaustionMs > 0 && timeToExhaustionMs < 300000) {
// Within 5 minutes
return new Date(Date.now() + timeToExhaustionMs);
}
return undefined;
}
/**
* Create memory error based on current usage
*/
createMemoryError(context) {
const stats = this.checkMemoryUsage(context);
const heapMB = stats.heapUsed / (1024 * 1024);
return types_1.DataPilotError.memory(`Memory limit exceeded: ${heapMB.toFixed(1)}MB used (limit: ${this.config.thresholds.maxMB}MB)`, 'MEMORY_LIMIT_EXCEEDED', {
...context,
memoryUsage: stats.heapUsed,
}, [
{
action: 'Reduce data scope',
description: 'Process data in smaller chunks or reduce maxRows',
severity: types_1.ErrorSeverity.HIGH,
command: '--maxRows 10000 or --chunkSize 1000',
},
{
action: 'Increase memory limit',
description: 'Increase available memory for the process',
severity: types_1.ErrorSeverity.MEDIUM,
command: '--max-old-space-size=2048',
},
{
action: 'Use streaming mode',
description: 'Enable streaming analysis for large datasets',
severity: types_1.ErrorSeverity.HIGH,
automated: true,
},
]);
}
/**
* Cleanup and dispose
*/
dispose() {
this.stopMonitoring();
this.removeAllListeners();
this.cleanupCallbacks = [];
this.memoryHistory = [];
}
handleMemoryWarning(stats, context) {
const heapMB = stats.heapUsed / (1024 * 1024);
logger_1.logger.warn(`Memory usage warning: ${heapMB.toFixed(1)}MB`, {
...context,
memoryUsage: stats.heapUsed,
});
if (this.config.enableAutomaticCleanup) {
this.forceGarbageCollection(context);
}
}
handleMemoryCritical(stats, context) {
const heapMB = stats.heapUsed / (1024 * 1024);
logger_1.logger.error(`Critical memory usage: ${heapMB.toFixed(1)}MB`, {
...context,
memoryUsage: stats.heapUsed,
});
if (this.config.enableAutomaticCleanup) {
this.runCleanup(context);
this.forceGarbageCollection(context);
}
// Predict exhaustion
const exhaustionTime = this.predictMemoryExhaustion();
if (exhaustionTime) {
const timeLeft = exhaustionTime.getTime() - Date.now();
logger_1.logger.error(`Memory exhaustion predicted in ${(timeLeft / 1000).toFixed(1)} seconds`, context);
}
}
handleMemoryMax(stats, context) {
const heapMB = stats.heapUsed / (1024 * 1024);
logger_1.logger.error(`Maximum memory limit reached: ${heapMB.toFixed(1)}MB`, {
...context,
memoryUsage: stats.heapUsed,
});
if (this.config.enableAutomaticCleanup) {
this.runCleanup(context);
this.forceGarbageCollection(context);
}
// Throw error to stop processing
throw this.createMemoryError(context);
}
}
exports.MemoryManager = MemoryManager;
// Global memory manager instance
exports.globalMemoryManager = new MemoryManager();
/**
* Resource cleanup utilities
*/
class ResourceManager {
resources = new Map();
disposed = false;
/**
* Register a resource for cleanup
*/
register(id, cleanup, type = 'generic') {
if (this.disposed) {
logger_1.logger.warn(`Cannot register resource ${id}: ResourceManager is disposed`);
return;
}
this.resources.set(id, { cleanup, type });
}
/**
* Unregister a resource
*/
unregister(id) {
return this.resources.delete(id);
}
/**
* Clean up a specific resource
*/
cleanup(id, context) {
const resource = this.resources.get(id);
if (!resource) {
return false;
}
try {
resource.cleanup();
this.resources.delete(id);
logger_1.logger.debug(`Cleaned up resource: ${id} (${resource.type})`, context);
return true;
}
catch (error) {
logger_1.logger.error(`Failed to clean up resource ${id}`, context, error);
return false;
}
}
/**
* Clean up all resources of a specific type
*/
cleanupByType(type, context) {
let cleaned = 0;
for (const [id, resource] of this.resources.entries()) {
if (resource.type === type) {
if (this.cleanup(id, context)) {
cleaned++;
}
}
}
return cleaned;
}
/**
* Clean up all resources
*/
cleanupAll(context) {
let cleaned = 0;
for (const id of this.resources.keys()) {
if (this.cleanup(id, context)) {
cleaned++;
}
}
return cleaned;
}
/**
* Get resource count by type
*/
getResourceCount(type) {
if (!type) {
return this.resources.size;
}
return Array.from(this.resources.values()).filter((r) => r.type === type).length;
}
/**
* List all resources
*/
listResources() {
return Array.from(this.resources.entries()).map(([id, resource]) => ({
id,
type: resource.type,
}));
}
/**
* Dispose of the resource manager
*/
dispose(context) {
if (this.disposed) {
return;
}
logger_1.logger.debug(`Disposing ResourceManager with ${this.resources.size} resources`, context);
this.cleanupAll(context);
this.disposed = true;
}
/**
* Check if disposed
*/
isDisposed() {
return this.disposed;
}
}
exports.ResourceManager = ResourceManager;
// Global resource manager instance
exports.globalResourceManager = new ResourceManager();
/**
* Process cleanup handler
*/
class ProcessCleanupHandler {
static instance;
handlers = [];
isShuttingDown = false;
constructor() {
// Register process exit handlers
process.on('exit', this.handleExit.bind(this));
process.on('SIGINT', (signal) => {
void this.handleSignal(signal);
});
process.on('SIGTERM', (signal) => {
void this.handleSignal(signal);
});
process.on('uncaughtException', (error) => {
void this.handleUncaughtException(error);
});
process.on('unhandledRejection', (reason) => {
void this.handleUnhandledRejection(reason);
});
}
static getInstance() {
if (!ProcessCleanupHandler.instance) {
ProcessCleanupHandler.instance = new ProcessCleanupHandler();
}
return ProcessCleanupHandler.instance;
}
/**
* Register cleanup handler
*/
register(handler) {
this.handlers.push(handler);
}
/**
* Run all cleanup handlers
*/
async runCleanup() {
if (this.isShuttingDown) {
return;
}
this.isShuttingDown = true;
logger_1.logger.info('Running process cleanup handlers');
for (const handler of this.handlers) {
try {
await handler();
}
catch (error) {
logger_1.logger.error('Error in cleanup handler', undefined, error);
}
}
// Clean up global managers
try {
exports.globalMemoryManager.dispose();
exports.globalResourceManager.dispose();
}
catch (error) {
logger_1.logger.error('Error disposing global managers', undefined, error);
}
}
handleExit() {
if (!this.isShuttingDown) {
// Synchronous cleanup only for exit event
logger_1.logger.info('Process exiting, running synchronous cleanup');
exports.globalMemoryManager.dispose();
exports.globalResourceManager.dispose();
}
}
async handleSignal(signal) {
logger_1.logger.info(`Received ${signal}, initiating graceful shutdown`);
await this.runCleanup();
process.exit(0);
}
async handleUncaughtException(error) {
logger_1.logger.error('Uncaught exception, running cleanup before exit', undefined, error);
await this.runCleanup();
process.exit(1);
}
async handleUnhandledRejection(reason) {
logger_1.logger.error('Unhandled promise rejection, running cleanup before exit', undefined, reason);
await this.runCleanup();
process.exit(1);
}
}
exports.ProcessCleanupHandler = ProcessCleanupHandler;
// Initialize global cleanup handler
exports.globalCleanupHandler = ProcessCleanupHandler.getInstance();
//# sourceMappingURL=memory-manager.js.map