supa-seed
Version:
A constraint-aware, framework-agnostic database seeding framework with deep PostgreSQL business logic discovery and MakerKit integration support
383 lines • 14.5 kB
JavaScript
"use strict";
/**
* Memory Management and Cleanup System
* Phase 6, Checkpoint F1 - Efficient memory usage and automatic cleanup
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.MemoryManager = void 0;
exports.monitorMemory = monitorMemory;
const logger_1 = require("./logger");
const performance_monitor_1 = require("./performance-monitor");
class MemoryManager {
/**
* Initialize memory management system
*/
static initialize(config) {
if (config) {
this.config = { ...this.config, ...config };
}
// Register default cleanup tasks
this.registerDefaultCleanupTasks();
// Start monitoring
this.startMemoryMonitoring();
// Handle process signals for cleanup
process.on('SIGINT', this.handleShutdown.bind(this));
process.on('SIGTERM', this.handleShutdown.bind(this));
process.on('exit', this.handleShutdown.bind(this));
logger_1.Logger.info('🧠 Memory management initialized:', {
maxHeapMB: this.config.maxHeapUsageMB,
warningThresholdMB: this.config.warningThresholdMB,
cleanupIntervalMs: this.config.cleanupIntervalMs
});
}
/**
* Register a cleanup task
*/
static registerCleanupTask(task) {
this.cleanupTasks.push(task);
// Sort by priority (critical first)
this.cleanupTasks.sort((a, b) => {
const priorityOrder = { critical: 0, high: 1, medium: 2, low: 3 };
return priorityOrder[a.priority] - priorityOrder[b.priority];
});
logger_1.Logger.debug(`🧹 Registered cleanup task: ${task.name} (${task.priority} priority)`);
}
/**
* Get current memory statistics
*/
static getMemoryStats() {
const memory = process.memoryUsage();
const heapUsedMB = memory.heapUsed / 1024 / 1024;
const heapTotalMB = memory.heapTotal / 1024 / 1024;
const externalMB = memory.external / 1024 / 1024;
const percentageUsed = (heapUsedMB / this.config.maxHeapUsageMB) * 100;
const recommendations = [];
if (heapUsedMB > this.config.warningThresholdMB) {
recommendations.push(`Memory usage high (${heapUsedMB.toFixed(1)}MB) - consider cleanup`);
}
if (heapUsedMB > this.config.forceGCThresholdMB) {
recommendations.push('Memory usage critical - forcing garbage collection');
}
if (percentageUsed > 80) {
recommendations.push('Consider increasing maxHeapUsageMB or optimizing memory usage');
}
const growthRate = this.calculateMemoryGrowthRate();
if (growthRate > 10) {
recommendations.push(`High memory growth rate detected (${growthRate.toFixed(1)}MB/min)`);
}
return {
heapUsed: memory.heapUsed,
heapTotal: memory.heapTotal,
external: memory.external,
arrayBuffers: memory.arrayBuffers,
rss: memory.rss,
usage: {
heapUsedMB,
heapTotalMB,
externalMB
},
percentageUsed,
recommendations
};
}
/**
* Force memory cleanup
*/
static async forceCleanup(reason = 'manual') {
if (this.isCleaningUp) {
logger_1.Logger.warn('🧹 Cleanup already in progress, skipping');
return { beforeMB: 0, afterMB: 0, freedMB: 0, tasksExecuted: [] };
}
this.isCleaningUp = true;
const beforeMemory = process.memoryUsage().heapUsed / 1024 / 1024;
const tasksExecuted = [];
logger_1.Logger.info(`🧹 Starting memory cleanup (reason: ${reason})`);
try {
// Execute cleanup tasks
for (const task of this.cleanupTasks) {
try {
// Check condition if provided
if (task.condition && !task.condition()) {
continue;
}
logger_1.Logger.debug(`🧹 Executing cleanup task: ${task.name}`);
await task.cleanup();
tasksExecuted.push(task.name);
// Record performance metric
performance_monitor_1.PerformanceMonitor.recordMetric({
name: 'memory_cleanup_task_executed',
value: 1,
unit: 'count',
timestamp: new Date(),
tags: {
task: task.name,
priority: task.priority,
reason
}
});
}
catch (error) {
logger_1.Logger.warn(`⚠️ Cleanup task failed: ${task.name}`, error);
}
}
// Force garbage collection if available
if (global.gc) {
logger_1.Logger.debug('🗑️ Forcing garbage collection');
global.gc();
}
else {
logger_1.Logger.debug('⚠️ Garbage collection not available (run with --expose-gc)');
}
const afterMemory = process.memoryUsage().heapUsed / 1024 / 1024;
const freedMB = beforeMemory - afterMemory;
logger_1.Logger.info('✅ Memory cleanup completed:', {
beforeMB: beforeMemory.toFixed(1),
afterMB: afterMemory.toFixed(1),
freedMB: freedMB.toFixed(1),
tasksExecuted: tasksExecuted.length
});
return {
beforeMB: beforeMemory,
afterMB: afterMemory,
freedMB,
tasksExecuted
};
}
finally {
this.isCleaningUp = false;
}
}
/**
* Check if memory cleanup is needed
*/
static shouldCleanup() {
const stats = this.getMemoryStats();
return stats.usage.heapUsedMB > this.config.warningThresholdMB ||
stats.percentageUsed > 70 ||
this.calculateMemoryGrowthRate() > 5;
}
/**
* Set memory limits
*/
static setMemoryLimits(limits) {
this.config = { ...this.config, ...limits };
logger_1.Logger.info('🎯 Memory limits updated:', limits);
}
/**
* Get memory usage history
*/
static getMemoryHistory(minutes = 10) {
const cutoff = new Date(Date.now() - minutes * 60 * 1000);
return this.memoryHistory
.filter(entry => entry.timestamp >= cutoff)
.map(entry => ({
timestamp: entry.timestamp,
heapUsedMB: entry.memory.heapUsed / 1024 / 1024,
heapTotalMB: entry.memory.heapTotal / 1024 / 1024
}));
}
/**
* Start memory monitoring
*/
static startMemoryMonitoring() {
if (this.cleanupInterval) {
clearInterval(this.cleanupInterval);
}
this.cleanupInterval = setInterval(async () => {
// Record memory usage
this.recordMemoryUsage();
// Check if cleanup is needed
if (this.shouldCleanup()) {
await this.forceCleanup('automatic');
}
// Check for critical memory usage
const stats = this.getMemoryStats();
if (stats.usage.heapUsedMB > this.config.forceGCThresholdMB) {
logger_1.Logger.warn('🚨 Critical memory usage detected, forcing cleanup');
await this.forceCleanup('critical');
}
}, this.config.cleanupIntervalMs);
logger_1.Logger.debug('📊 Memory monitoring started');
}
/**
* Record memory usage for history
*/
static recordMemoryUsage() {
const memory = process.memoryUsage();
this.memoryHistory.push({
timestamp: new Date(),
memory
});
// Keep only last 1000 entries (about 8 hours at 30s intervals)
if (this.memoryHistory.length > 1000) {
this.memoryHistory.shift();
}
// Record performance metric
performance_monitor_1.PerformanceMonitor.recordMetric({
name: 'memory_heap_used',
value: memory.heapUsed / 1024 / 1024,
unit: 'mb',
timestamp: new Date(),
tags: {
type: 'heap_used'
}
});
}
/**
* Calculate memory growth rate in MB per minute
*/
static calculateMemoryGrowthRate() {
if (this.memoryHistory.length < 2) {
return 0;
}
const recent = this.memoryHistory.slice(-10); // Last 10 entries
if (recent.length < 2) {
return 0;
}
const first = recent[0];
const last = recent[recent.length - 1];
const timeDiffMinutes = (last.timestamp.getTime() - first.timestamp.getTime()) / (1000 * 60);
const memoryDiffMB = (last.memory.heapUsed - first.memory.heapUsed) / 1024 / 1024;
return timeDiffMinutes > 0 ? memoryDiffMB / timeDiffMinutes : 0;
}
/**
* Register default cleanup tasks
*/
static registerDefaultCleanupTasks() {
// Clear temporary caches
this.registerCleanupTask({
name: 'clear_temporary_caches',
priority: 'high',
estimatedMemoryFreedMB: 10,
cleanup: () => {
// This would clear various temporary caches
logger_1.Logger.debug('🧹 Clearing temporary caches');
}
});
// Clear old performance metrics
this.registerCleanupTask({
name: 'clear_old_metrics',
priority: 'medium',
estimatedMemoryFreedMB: 5,
cleanup: () => {
// Clear old metrics to free memory
const oldMetricsCleared = Math.max(0, performance_monitor_1.PerformanceMonitor.getPerformanceStats().totalOperations - 5000);
if (oldMetricsCleared > 0) {
logger_1.Logger.debug(`🧹 Cleared ${oldMetricsCleared} old performance metrics`);
}
}
});
// Clear old memory history
this.registerCleanupTask({
name: 'clear_old_memory_history',
priority: 'low',
estimatedMemoryFreedMB: 2,
cleanup: () => {
if (this.memoryHistory.length > 500) {
const removed = this.memoryHistory.length - 500;
this.memoryHistory = this.memoryHistory.slice(-500);
logger_1.Logger.debug(`🧹 Cleared ${removed} old memory history entries`);
}
}
});
// Force garbage collection
this.registerCleanupTask({
name: 'force_garbage_collection',
priority: 'critical',
estimatedMemoryFreedMB: 20,
cleanup: () => {
if (global.gc) {
global.gc();
logger_1.Logger.debug('🗑️ Forced garbage collection');
}
},
condition: () => global.gc !== undefined
});
}
/**
* Handle process shutdown
*/
static async handleShutdown() {
logger_1.Logger.info('🛑 Process shutdown detected, performing final cleanup');
if (this.cleanupInterval) {
clearInterval(this.cleanupInterval);
}
await this.forceCleanup('shutdown');
}
/**
* Get cleanup recommendations
*/
static getCleanupRecommendations() {
const stats = this.getMemoryStats();
const growthRate = this.calculateMemoryGrowthRate();
const recommendations = {
urgent: [],
suggested: [],
preventive: []
};
if (stats.usage.heapUsedMB > this.config.forceGCThresholdMB) {
recommendations.urgent.push('Force garbage collection immediately');
recommendations.urgent.push('Clear all caches and temporary data');
}
if (stats.usage.heapUsedMB > this.config.warningThresholdMB) {
recommendations.suggested.push('Run cleanup tasks');
recommendations.suggested.push('Review active operations for memory leaks');
}
if (growthRate > 5) {
recommendations.suggested.push('Investigate memory growth pattern');
recommendations.suggested.push('Consider reducing batch sizes');
}
if (stats.percentageUsed > 60) {
recommendations.preventive.push('Monitor memory usage more frequently');
recommendations.preventive.push('Consider increasing memory limits');
}
return recommendations;
}
/**
* Clear memory history
*/
static clearHistory() {
this.memoryHistory.length = 0;
logger_1.Logger.info('🧹 Memory history cleared');
}
}
exports.MemoryManager = MemoryManager;
MemoryManager.config = {
maxHeapUsageMB: 512, // 512MB default limit
cleanupIntervalMs: 30000, // 30 seconds
warningThresholdMB: 256, // 256MB warning
forceGCThresholdMB: 400, // 400MB force GC
maxCacheSize: 1000
};
MemoryManager.cleanupTasks = [];
MemoryManager.cleanupInterval = null;
MemoryManager.memoryHistory = [];
MemoryManager.isCleaningUp = false;
/**
* Memory monitoring decorator
*/
function monitorMemory(thresholdMB) {
return function (target, propertyName, descriptor) {
const method = descriptor.value;
descriptor.value = async function (...args) {
const beforeMemory = process.memoryUsage().heapUsed / 1024 / 1024;
try {
const result = await method.apply(this, args);
const afterMemory = process.memoryUsage().heapUsed / 1024 / 1024;
const memoryDelta = afterMemory - beforeMemory;
if (thresholdMB && memoryDelta > thresholdMB) {
logger_1.Logger.warn(`⚠️ High memory usage in ${propertyName}:`, {
delta: `${memoryDelta.toFixed(1)}MB`,
threshold: `${thresholdMB}MB`
});
}
return result;
}
catch (error) {
throw error;
}
};
};
}
exports.default = MemoryManager;
//# sourceMappingURL=memory-manager.js.map