behemoth-cli
Version:
🌍 BEHEMOTH CLIv3.760.4 - Level 50+ POST-SINGULARITY Intelligence Trading AI
228 lines • 9.21 kB
JavaScript
/**
* Memory Usage Monitor for Large File Operations
* Provides monitoring and warnings for memory-intensive operations
*/
import * as fs from 'fs';
import * as util from 'util';
import { logWarn, logError, logDebug } from './error-handler.js';
const readFile = util.promisify(fs.readFile);
const writeFile = util.promisify(fs.writeFile);
const stat = util.promisify(fs.stat);
export class MemoryMonitor {
static DEFAULT_MAX_FILE_SIZE = 50 * 1024 * 1024; // 50MB
static DEFAULT_MEMORY_THRESHOLD = 80; // 80%
static DEFAULT_CHUNK_SIZE = 1024 * 1024; // 1MB
/**
* Get current memory usage statistics
*/
static getMemoryStats() {
const memUsage = process.memoryUsage();
const totalMemory = require('os').totalmem();
return {
used: memUsage.rss,
total: totalMemory,
external: memUsage.external,
heapUsed: memUsage.heapUsed,
heapTotal: memUsage.heapTotal,
percentage: (memUsage.rss / totalMemory) * 100
};
}
/**
* Check if memory usage is above threshold
*/
static isMemoryUsageHigh(threshold = MemoryMonitor.DEFAULT_MEMORY_THRESHOLD) {
const stats = this.getMemoryStats();
return stats.percentage > threshold;
}
/**
* Log memory warning if usage is high
*/
static checkMemoryUsage(operation) {
if (this.isMemoryUsageHigh()) {
const stats = this.getMemoryStats();
logWarn(`High memory usage during ${operation}`, undefined, {
component: 'MemoryMonitor',
operation,
metadata: {
memoryUsagePercentage: stats.percentage.toFixed(2),
heapUsedMB: (stats.heapUsed / 1024 / 1024).toFixed(2),
totalMemoryMB: (stats.total / 1024 / 1024).toFixed(2)
}
});
}
}
/**
* Safe file read with memory monitoring
*/
static async readFileWithMonitoring(filePath, options = {}) {
const { maxFileSize = this.DEFAULT_MAX_FILE_SIZE, memoryThreshold = this.DEFAULT_MEMORY_THRESHOLD, encoding = 'utf8' } = options;
try {
// Check file size before reading
const fileStats = await stat(filePath);
if (fileStats.size > maxFileSize) {
throw new Error(`File size (${fileStats.size} bytes) exceeds maximum allowed size (${maxFileSize} bytes)`);
}
// Check memory before operation
if (this.isMemoryUsageHigh(memoryThreshold)) {
logWarn('Reading file with high memory usage', undefined, {
component: 'MemoryMonitor',
operation: 'readFile',
metadata: { filePath, fileSize: fileStats.size }
});
}
const data = await readFile(filePath, encoding);
// Check memory after operation
this.checkMemoryUsage('file read');
logDebug(`File read completed`, undefined, {
component: 'MemoryMonitor',
operation: 'readFile',
metadata: { filePath, fileSize: fileStats.size }
});
return data;
}
catch (error) {
logError('File read operation failed', error, {
component: 'MemoryMonitor',
operation: 'readFile',
metadata: { filePath }
});
throw error;
}
}
/**
* Safe file write with memory monitoring
*/
static async writeFileWithMonitoring(filePath, data, options = {}) {
const { memoryThreshold = this.DEFAULT_MEMORY_THRESHOLD } = options;
const encoding = options.encoding || 'utf8';
try {
const dataSize = Buffer.isBuffer(data) ? data.length : Buffer.byteLength(data, encoding);
// Check memory before operation
if (this.isMemoryUsageHigh(memoryThreshold)) {
logWarn('Writing file with high memory usage', undefined, {
component: 'MemoryMonitor',
operation: 'writeFile',
metadata: { filePath, dataSize }
});
}
await writeFile(filePath, data, options);
// Check memory after operation
this.checkMemoryUsage('file write');
logDebug(`File write completed`, undefined, {
component: 'MemoryMonitor',
operation: 'writeFile',
metadata: { filePath, dataSize }
});
}
catch (error) {
logError('File write operation failed', error, {
component: 'MemoryMonitor',
operation: 'writeFile',
metadata: { filePath }
});
throw error;
}
}
/**
* Synchronous file read with memory monitoring (use sparingly)
*/
static readFileSyncWithMonitoring(filePath, options = {}) {
const { maxFileSize = this.DEFAULT_MAX_FILE_SIZE, memoryThreshold = this.DEFAULT_MEMORY_THRESHOLD, encoding = 'utf8' } = options;
try {
// Check file size before reading
const fileStats = fs.statSync(filePath);
if (fileStats.size > maxFileSize) {
throw new Error(`File size (${fileStats.size} bytes) exceeds maximum allowed size (${maxFileSize} bytes)`);
}
// Check memory before operation
if (this.isMemoryUsageHigh(memoryThreshold)) {
logWarn('Reading file synchronously with high memory usage', undefined, {
component: 'MemoryMonitor',
operation: 'readFileSync',
metadata: { filePath, fileSize: fileStats.size }
});
}
const data = fs.readFileSync(filePath, encoding);
// Check memory after operation
this.checkMemoryUsage('sync file read');
return data;
}
catch (error) {
logError('Synchronous file read operation failed', error, {
component: 'MemoryMonitor',
operation: 'readFileSync',
metadata: { filePath }
});
throw error;
}
}
/**
* Force garbage collection if available and memory usage is high
*/
static forceGarbageCollection() {
if (global.gc && this.isMemoryUsageHigh(70)) {
const beforeStats = this.getMemoryStats();
global.gc();
const afterStats = this.getMemoryStats();
logDebug('Forced garbage collection', undefined, {
component: 'MemoryMonitor',
operation: 'forceGC',
metadata: {
beforeMemoryMB: (beforeStats.heapUsed / 1024 / 1024).toFixed(2),
afterMemoryMB: (afterStats.heapUsed / 1024 / 1024).toFixed(2),
freedMB: ((beforeStats.heapUsed - afterStats.heapUsed) / 1024 / 1024).toFixed(2)
}
});
return true;
}
return false;
}
/**
* Stream large file processing for files that exceed memory limits
*/
static async processLargeFile(filePath, processor, options = {}) {
const { chunkSize = this.DEFAULT_CHUNK_SIZE, memoryThreshold = this.DEFAULT_MEMORY_THRESHOLD } = options;
return new Promise((resolve, reject) => {
const stream = fs.createReadStream(filePath, {
encoding: 'utf8',
highWaterMark: chunkSize
});
stream.on('data', async (chunk) => {
try {
// Check memory before processing chunk
if (this.isMemoryUsageHigh(memoryThreshold)) {
logWarn('Processing file chunk with high memory usage', undefined, {
component: 'MemoryMonitor',
operation: 'processLargeFile',
metadata: { filePath, chunkSize: chunk.length }
});
// Attempt garbage collection
this.forceGarbageCollection();
}
await processor(chunk);
}
catch (error) {
stream.destroy();
reject(error);
}
});
stream.on('end', () => {
logDebug('Large file processing completed', undefined, {
component: 'MemoryMonitor',
operation: 'processLargeFile',
metadata: { filePath }
});
resolve();
});
stream.on('error', (error) => {
logError('Large file processing failed', error, {
component: 'MemoryMonitor',
operation: 'processLargeFile',
metadata: { filePath }
});
reject(error);
});
});
}
}
//# sourceMappingURL=memory-monitor.js.map