datapilot-cli
Version:
Enterprise-grade streaming multi-format data analysis with comprehensive statistical insights and intelligent relationship detection - supports CSV, JSON, Excel, TSV, Parquet - memory-efficient, cross-platform
592 lines • 20.2 kB
JavaScript
"use strict";
/**
* Advanced Resource Pool Management
* Memory pooling, garbage collection optimization, and resource lifecycle management
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.ResourcePoolManager = exports.GcOptimizer = exports.ResourcePool = void 0;
exports.getGlobalResourcePoolManager = getGlobalResourcePoolManager;
exports.shutdownGlobalResourcePoolManager = shutdownGlobalResourcePoolManager;
exports.createBufferPool = createBufferPool;
exports.createObjectPool = createObjectPool;
const events_1 = require("events");
const perf_hooks_1 = require("perf_hooks");
const logger_1 = require("../utils/logger");
/**
* Generic resource pool with advanced lifecycle management
*/
class ResourcePool extends events_1.EventEmitter {
pool = [];
activeResources = new Set();
options;
stats;
cleanupTimer;
factory;
cleanup;
validator;
constructor(factory, cleanup, validator = () => true, options = {}) {
super();
this.factory = factory;
this.cleanup = cleanup;
this.validator = validator;
this.options = {
maxPoolSize: options.maxPoolSize || 50,
minPoolSize: options.minPoolSize || 5,
maxResourceAge: options.maxResourceAge || 300000, // 5 minutes
cleanupInterval: options.cleanupInterval || 30000, // 30 seconds
enableGcOptimization: options.enableGcOptimization ?? true,
gcHeuristics: {
memoryPressureThreshold: 0.8,
timeBetweenGcMin: 5000, // 5 seconds
timeBetweenGcMax: 60000, // 1 minute
gcEfficiencyThreshold: 0.1, // 10% memory freed minimum
adaptiveGcTiming: true,
...options.gcHeuristics
},
resourceTypes: options.resourceTypes || []
};
this.stats = {
totalCreated: 0,
totalDestroyed: 0,
totalReused: 0,
currentPoolSize: 0,
activeResources: 0,
poolHitRate: 0,
averageResourceAge: 0,
memoryUsage: 0
};
this.startCleanupTimer();
this.prewarmPool();
logger_1.logger.info(`Resource pool initialized with ${this.options.maxPoolSize} max resources`);
}
/**
* Acquire a resource from the pool
*/
acquire() {
const startTime = perf_hooks_1.performance.now();
// Try to get from pool first
let pooledResource = this.getFromPool();
if (pooledResource) {
this.stats.totalReused++;
this.emit('resource-reused', { resource: pooledResource.resource, useCount: pooledResource.useCount });
}
else {
// Create new resource
const resource = this.factory();
pooledResource = {
resource,
createdAt: Date.now(),
lastUsed: Date.now(),
useCount: 0,
isHealthy: true
};
this.stats.totalCreated++;
this.emit('resource-created', { resource });
}
pooledResource.lastUsed = Date.now();
pooledResource.useCount++;
this.activeResources.add(pooledResource);
this.updateStats();
this.emit('resource-acquired', {
resource: pooledResource.resource,
acquisitionTime: perf_hooks_1.performance.now() - startTime
});
return pooledResource.resource;
}
/**
* Release a resource back to the pool
*/
release(resource) {
const pooledResource = this.findActiveResource(resource);
if (!pooledResource) {
logger_1.logger.warn('Attempted to release unknown resource');
return;
}
this.activeResources.delete(pooledResource);
// Validate resource health
if (this.validator(resource) && pooledResource.isHealthy) {
// Resource is healthy, return to pool if there's space
if (this.pool.length < this.options.maxPoolSize) {
pooledResource.lastUsed = Date.now();
this.pool.push(pooledResource);
this.emit('resource-returned', { resource });
}
else {
// Pool is full, destroy resource
this.destroyResource(pooledResource);
}
}
else {
// Resource is unhealthy, destroy it
pooledResource.isHealthy = false;
this.destroyResource(pooledResource);
}
this.updateStats();
}
/**
* Get resource from pool with health check
*/
getFromPool() {
while (this.pool.length > 0) {
const pooledResource = this.pool.shift();
// Check if resource is still valid
if (this.isResourceValid(pooledResource)) {
return pooledResource;
}
else {
// Resource is expired or unhealthy, destroy it
this.destroyResource(pooledResource);
}
}
return null;
}
/**
* Check if a pooled resource is still valid
*/
isResourceValid(pooledResource) {
const now = Date.now();
const age = now - pooledResource.createdAt;
return age < this.options.maxResourceAge &&
pooledResource.isHealthy &&
this.validator(pooledResource.resource);
}
/**
* Find active resource by reference
*/
findActiveResource(resource) {
for (const pooledResource of this.activeResources) {
if (pooledResource.resource === resource) {
return pooledResource;
}
}
return undefined;
}
/**
* Destroy a resource and update stats
*/
destroyResource(pooledResource) {
try {
this.cleanup(pooledResource.resource);
this.stats.totalDestroyed++;
this.emit('resource-destroyed', {
resource: pooledResource.resource,
age: Date.now() - pooledResource.createdAt,
useCount: pooledResource.useCount
});
}
catch (error) {
logger_1.logger.error(`Error destroying resource: ${error.message}`);
}
}
/**
* Prewarm the pool with minimum resources
*/
prewarmPool() {
const targetSize = Math.min(this.options.minPoolSize, this.options.maxPoolSize);
for (let i = 0; i < targetSize; i++) {
try {
const resource = this.factory();
const pooledResource = {
resource,
createdAt: Date.now(),
lastUsed: Date.now(),
useCount: 0,
isHealthy: true
};
this.pool.push(pooledResource);
this.stats.totalCreated++;
}
catch (error) {
logger_1.logger.error(`Error prewarming pool: ${error.message}`);
break;
}
}
this.updateStats();
logger_1.logger.info(`Pool prewarmed with ${this.pool.length} resources`);
}
/**
* Cleanup expired resources
*/
cleanupExpiredResources() {
const now = Date.now();
let cleanupCount = 0;
// Clean pool
this.pool = this.pool.filter(pooledResource => {
if (!this.isResourceValid(pooledResource)) {
this.destroyResource(pooledResource);
cleanupCount++;
return false;
}
return true;
});
// Clean active resources (mark as unhealthy)
for (const pooledResource of this.activeResources) {
if (!this.isResourceValid(pooledResource)) {
pooledResource.isHealthy = false;
cleanupCount++;
}
}
if (cleanupCount > 0) {
logger_1.logger.info(`Cleaned up ${cleanupCount} expired resources`);
this.emit('cleanup-completed', { cleanupCount });
}
this.updateStats();
}
/**
* Start cleanup timer
*/
startCleanupTimer() {
this.cleanupTimer = setInterval(() => {
this.cleanupExpiredResources();
}, this.options.cleanupInterval);
}
/**
* Update pool statistics
*/
updateStats() {
const totalRequests = this.stats.totalCreated + this.stats.totalReused;
this.stats.poolHitRate = totalRequests > 0 ? this.stats.totalReused / totalRequests : 0;
this.stats.currentPoolSize = this.pool.length;
this.stats.activeResources = this.activeResources.size;
if (this.pool.length > 0) {
const now = Date.now();
const totalAge = this.pool.reduce((sum, pr) => sum + (now - pr.createdAt), 0);
this.stats.averageResourceAge = totalAge / this.pool.length;
}
// Estimate memory usage (rough calculation)
const memoryUsage = process.memoryUsage();
this.stats.memoryUsage = memoryUsage.heapUsed / 1024 / 1024; // MB
}
/**
* Get current pool statistics
*/
getStats() {
this.updateStats();
return { ...this.stats };
}
/**
* Shutdown pool and cleanup all resources
*/
shutdown() {
if (this.cleanupTimer) {
clearInterval(this.cleanupTimer);
this.cleanupTimer = undefined;
}
// Destroy all pooled resources
for (const pooledResource of this.pool) {
this.destroyResource(pooledResource);
}
this.pool = [];
// Mark active resources as unhealthy
for (const pooledResource of this.activeResources) {
pooledResource.isHealthy = false;
}
this.updateStats();
logger_1.logger.info('Resource pool shutdown complete');
this.emit('shutdown');
}
}
exports.ResourcePool = ResourcePool;
/**
* Advanced garbage collection optimizer
*/
class GcOptimizer extends events_1.EventEmitter {
options;
stats;
lastGcTime = 0;
lastMemoryUsage = 0;
gcTimer;
isOptimizing = false;
constructor(options = {}) {
super();
this.options = {
memoryPressureThreshold: options.memoryPressureThreshold || 0.8,
timeBetweenGcMin: options.timeBetweenGcMin || 5000,
timeBetweenGcMax: options.timeBetweenGcMax || 60000,
gcEfficiencyThreshold: options.gcEfficiencyThreshold || 0.1,
adaptiveGcTiming: options.adaptiveGcTiming ?? true
};
this.stats = {
totalGcRuns: 0,
totalGcTime: 0,
averageGcTime: 0,
memoryFreed: 0,
lastGcTime: 0,
gcEfficiency: 0,
adaptiveTimingEnabled: this.options.adaptiveGcTiming
};
this.lastMemoryUsage = process.memoryUsage().heapUsed;
if (this.options.adaptiveGcTiming) {
this.startAdaptiveGc();
}
}
/**
* Force garbage collection with optimization
*/
forceGc() {
if (!global.gc) {
logger_1.logger.warn('Garbage collection not available (run with --expose-gc)');
return false;
}
const startTime = perf_hooks_1.performance.now();
const beforeMemory = process.memoryUsage().heapUsed;
try {
global.gc();
const afterMemory = process.memoryUsage().heapUsed;
const gcTime = perf_hooks_1.performance.now() - startTime;
const memoryFreed = beforeMemory - afterMemory;
const efficiency = memoryFreed / beforeMemory;
// Update statistics
this.stats.totalGcRuns++;
this.stats.totalGcTime += gcTime;
this.stats.averageGcTime = this.stats.totalGcTime / this.stats.totalGcRuns;
this.stats.memoryFreed += memoryFreed;
this.stats.lastGcTime = Date.now();
this.stats.gcEfficiency = efficiency;
this.lastGcTime = Date.now();
this.lastMemoryUsage = afterMemory;
logger_1.logger.info(`GC completed: ${(memoryFreed / 1024 / 1024).toFixed(2)}MB freed in ${gcTime.toFixed(2)}ms (${(efficiency * 100).toFixed(1)}% efficiency)`);
this.emit('gc-completed', {
memoryFreed,
gcTime,
efficiency,
beforeMemory,
afterMemory
});
return true;
}
catch (error) {
logger_1.logger.error(`Garbage collection failed: ${error.message}`);
return false;
}
}
/**
* Check if GC should be triggered
*/
shouldTriggerGc() {
const now = Date.now();
const timeSinceLastGc = now - this.lastGcTime;
const currentMemory = process.memoryUsage().heapUsed;
const memoryGrowth = currentMemory - this.lastMemoryUsage;
const memoryGrowthRatio = this.lastMemoryUsage > 0 ? memoryGrowth / this.lastMemoryUsage : 0;
// Don't run GC too frequently
if (timeSinceLastGc < this.options.timeBetweenGcMin) {
return false;
}
// Force GC if memory pressure is high
const memoryPressure = this.getCurrentMemoryPressure();
if (memoryPressure > this.options.memoryPressureThreshold) {
return true;
}
// Trigger GC if memory growth is significant
if (memoryGrowthRatio > 0.2 && timeSinceLastGc > this.options.timeBetweenGcMin * 2) {
return true;
}
// Force GC if too much time has passed
if (timeSinceLastGc > this.options.timeBetweenGcMax) {
return true;
}
return false;
}
/**
* Get current memory pressure (0-1)
*/
getCurrentMemoryPressure() {
const usage = process.memoryUsage();
const totalMemory = usage.heapTotal + usage.external;
const usedMemory = usage.heapUsed + usage.external;
return totalMemory > 0 ? usedMemory / totalMemory : 0;
}
/**
* Start adaptive GC timing
*/
startAdaptiveGc() {
if (this.isOptimizing)
return;
this.isOptimizing = true;
const checkInterval = Math.min(this.options.timeBetweenGcMin / 2, 2000); // Check every 2 seconds max
this.gcTimer = setInterval(() => {
if (this.shouldTriggerGc()) {
this.forceGc();
}
}, checkInterval);
logger_1.logger.info('Adaptive garbage collection optimization started');
}
/**
* Stop adaptive GC timing
*/
stopAdaptiveGc() {
if (this.gcTimer) {
clearInterval(this.gcTimer);
this.gcTimer = undefined;
}
this.isOptimizing = false;
logger_1.logger.info('Adaptive garbage collection optimization stopped');
}
/**
* Get GC statistics
*/
getStats() {
return { ...this.stats };
}
/**
* Reset GC statistics
*/
resetStats() {
this.stats = {
totalGcRuns: 0,
totalGcTime: 0,
averageGcTime: 0,
memoryFreed: 0,
lastGcTime: 0,
gcEfficiency: 0,
adaptiveTimingEnabled: this.options.adaptiveGcTiming
};
}
/**
* Shutdown GC optimizer
*/
shutdown() {
this.stopAdaptiveGc();
this.emit('shutdown');
}
}
exports.GcOptimizer = GcOptimizer;
/**
* Multi-type resource pool manager
*/
class ResourcePoolManager extends events_1.EventEmitter {
pools = new Map();
gcOptimizer;
options;
constructor(options = {}) {
super();
this.options = {
maxPoolSize: options.maxPoolSize || 50,
minPoolSize: options.minPoolSize || 5,
maxResourceAge: options.maxResourceAge || 300000,
cleanupInterval: options.cleanupInterval || 30000,
enableGcOptimization: options.enableGcOptimization ?? true,
gcHeuristics: {
memoryPressureThreshold: 0.8,
timeBetweenGcMin: 5000,
timeBetweenGcMax: 60000,
gcEfficiencyThreshold: 0.1,
adaptiveGcTiming: true,
...options.gcHeuristics
},
resourceTypes: options.resourceTypes || []
};
this.gcOptimizer = new GcOptimizer(this.options.gcHeuristics);
// Initialize predefined resource types
for (const resourceType of this.options.resourceTypes) {
this.addResourceType(resourceType);
}
logger_1.logger.info('Resource pool manager initialized');
}
/**
* Add a new resource type
*/
addResourceType(config) {
const pool = new ResourcePool(config.factory, config.cleanup, config.validator, {
maxPoolSize: config.maxPoolSize || this.options.maxPoolSize,
maxResourceAge: config.maxAge || this.options.maxResourceAge,
cleanupInterval: this.options.cleanupInterval
});
this.pools.set(config.name, pool);
// Forward events
pool.on('resource-created', (data) => this.emit('resource-created', { type: config.name, ...data }));
pool.on('resource-destroyed', (data) => this.emit('resource-destroyed', { type: config.name, ...data }));
pool.on('cleanup-completed', (data) => this.emit('cleanup-completed', { type: config.name, ...data }));
logger_1.logger.info(`Added resource pool for type: ${config.name}`);
}
/**
* Get resource pool by type
*/
getPool(type) {
return this.pools.get(type);
}
/**
* Acquire resource of specific type
*/
acquire(type) {
const pool = this.getPool(type);
return pool ? pool.acquire() : undefined;
}
/**
* Release resource of specific type
*/
release(type, resource) {
const pool = this.getPool(type);
if (pool) {
pool.release(resource);
}
}
/**
* Get aggregated statistics for all pools
*/
getAllStats() {
const stats = {};
for (const [type, pool] of this.pools) {
stats[type] = pool.getStats();
}
return Object.assign(stats, { gc: this.gcOptimizer.getStats() });
}
/**
* Force garbage collection
*/
forceGc() {
return this.gcOptimizer.forceGc();
}
/**
* Shutdown all pools and GC optimizer
*/
shutdown() {
for (const [type, pool] of this.pools) {
pool.shutdown();
}
this.pools.clear();
this.gcOptimizer.shutdown();
this.emit('shutdown');
logger_1.logger.info('Resource pool manager shutdown complete');
}
}
exports.ResourcePoolManager = ResourcePoolManager;
/**
* Global resource pool manager
*/
let globalResourcePoolManager = null;
/**
* Get or create global resource pool manager
*/
function getGlobalResourcePoolManager(options) {
if (!globalResourcePoolManager) {
globalResourcePoolManager = new ResourcePoolManager(options);
}
return globalResourcePoolManager;
}
/**
* Shutdown global resource pool manager
*/
function shutdownGlobalResourcePoolManager() {
if (globalResourcePoolManager) {
globalResourcePoolManager.shutdown();
globalResourcePoolManager = null;
}
}
/**
* Convenient buffer pool factory
*/
function createBufferPool(options) {
const bufferSizes = options.bufferSizes || [1024, 4096, 16384, 65536, 262144]; // 1KB to 256KB
let currentSizeIndex = 0;
return new ResourcePool(() => Buffer.alloc(bufferSizes[currentSizeIndex++ % bufferSizes.length]), (buffer) => buffer.fill(0), (buffer) => Buffer.isBuffer(buffer), { maxPoolSize: options.maxSize || 100 });
}
/**
* Convenient object pool factory
*/
function createObjectPool(factory, reset, options) {
return new ResourcePool(factory, reset, () => true, // Objects are generally always valid unless explicitly marked
options);
}
//# sourceMappingURL=resource-pool.js.map