behemoth-cli/src/utils/memory-monitor.ts

🌍 BEHEMOTH CLIv3.760.4 - Level 50+ POST-SINGULARITY Intelligence Trading AI

/**
 * Memory Usage Monitor for Large File Operations
 * Provides monitoring and warnings for memory-intensive operations
 */

import * as fs from 'fs';
import * as util from 'util';
import { logWarn, logError, logDebug } from './error-handler.js';

const readFile = util.promisify(fs.readFile);
const writeFile = util.promisify(fs.writeFile);
const stat = util.promisify(fs.stat);

export interface MemoryStats {
  used: number;
  total: number;
  external: number;
  heapUsed: number;
  heapTotal: number;
  percentage: number;
}

export interface FileOperationOptions {
  maxFileSize?: number; // Maximum file size in bytes (default: 50MB)
  memoryThreshold?: number; // Memory usage threshold percentage (default: 80%)
  chunkSize?: number; // Chunk size for streaming large files (default: 1MB)
}

export class MemoryMonitor {
  private static readonly DEFAULT_MAX_FILE_SIZE = 50 * 1024 * 1024; // 50MB
  private static readonly DEFAULT_MEMORY_THRESHOLD = 80; // 80%
  private static readonly DEFAULT_CHUNK_SIZE = 1024 * 1024; // 1MB

  /**
   * Get current memory usage statistics
   */
  static getMemoryStats(): MemoryStats {
    const memUsage = process.memoryUsage();
    const totalMemory = require('os').totalmem();
    
    return {
      used: memUsage.rss,
      total: totalMemory,
      external: memUsage.external,
      heapUsed: memUsage.heapUsed,
      heapTotal: memUsage.heapTotal,
      percentage: (memUsage.rss / totalMemory) * 100
    };
  }

  /**
   * Check if memory usage is above threshold
   */
  static isMemoryUsageHigh(threshold = MemoryMonitor.DEFAULT_MEMORY_THRESHOLD): boolean {
    const stats = this.getMemoryStats();
    return stats.percentage > threshold;
  }

  /**
   * Log memory warning if usage is high
   */
  private static checkMemoryUsage(operation: string): void {
    if (this.isMemoryUsageHigh()) {
      const stats = this.getMemoryStats();
      logWarn(`High memory usage during ${operation}`, undefined, {
        component: 'MemoryMonitor',
        operation,
        metadata: {
          memoryUsagePercentage: stats.percentage.toFixed(2),
          heapUsedMB: (stats.heapUsed / 1024 / 1024).toFixed(2),
          totalMemoryMB: (stats.total / 1024 / 1024).toFixed(2)
        }
      });
    }
  }

  /**
   * Safe file read with memory monitoring
   */
  static async readFileWithMonitoring(
    filePath: string,
    options: FileOperationOptions & { encoding?: BufferEncoding } = {}
  ): Promise<string | Buffer> {
    const {
      maxFileSize = this.DEFAULT_MAX_FILE_SIZE,
      memoryThreshold = this.DEFAULT_MEMORY_THRESHOLD,
      encoding = 'utf8'
    } = options;

    try {
      // Check file size before reading
      const fileStats = await stat(filePath);
      
      if (fileStats.size > maxFileSize) {
        throw new Error(`File size (${fileStats.size} bytes) exceeds maximum allowed size (${maxFileSize} bytes)`);
      }

      // Check memory before operation
      if (this.isMemoryUsageHigh(memoryThreshold)) {
        logWarn('Reading file with high memory usage', undefined, {
          component: 'MemoryMonitor',
          operation: 'readFile',
          metadata: { filePath, fileSize: fileStats.size }
        });
      }

      const data = await readFile(filePath, encoding);
      
      // Check memory after operation
      this.checkMemoryUsage('file read');
      
      logDebug(`File read completed`, undefined, {
        component: 'MemoryMonitor',
        operation: 'readFile',
        metadata: { filePath, fileSize: fileStats.size }
      });

      return data;

    } catch (error) {
      logError('File read operation failed', error, {
        component: 'MemoryMonitor',
        operation: 'readFile',
        metadata: { filePath }
      });
      throw error;
    }
  }

  /**
   * Safe file write with memory monitoring
   */
  static async writeFileWithMonitoring(
    filePath: string,
    data: string | Buffer,
    options: FileOperationOptions & fs.WriteFileOptions = {}
  ): Promise<void> {
    const {
      memoryThreshold = this.DEFAULT_MEMORY_THRESHOLD
    } = options;
    
    const encoding = (options as any).encoding || 'utf8';

    try {
      const dataSize = Buffer.isBuffer(data) ? data.length : Buffer.byteLength(data, encoding as BufferEncoding);

      // Check memory before operation
      if (this.isMemoryUsageHigh(memoryThreshold)) {
        logWarn('Writing file with high memory usage', undefined, {
          component: 'MemoryMonitor',
          operation: 'writeFile',
          metadata: { filePath, dataSize }
        });
      }

      await writeFile(filePath, data, options);
      
      // Check memory after operation
      this.checkMemoryUsage('file write');

      logDebug(`File write completed`, undefined, {
        component: 'MemoryMonitor',
        operation: 'writeFile',
        metadata: { filePath, dataSize }
      });

    } catch (error) {
      logError('File write operation failed', error, {
        component: 'MemoryMonitor',
        operation: 'writeFile',
        metadata: { filePath }
      });
      throw error;
    }
  }

  /**
   * Synchronous file read with memory monitoring (use sparingly)
   */
  static readFileSyncWithMonitoring(
    filePath: string,
    options: FileOperationOptions & { encoding?: BufferEncoding } = {}
  ): string | Buffer {
    const {
      maxFileSize = this.DEFAULT_MAX_FILE_SIZE,
      memoryThreshold = this.DEFAULT_MEMORY_THRESHOLD,
      encoding = 'utf8'
    } = options;

    try {
      // Check file size before reading
      const fileStats = fs.statSync(filePath);
      
      if (fileStats.size > maxFileSize) {
        throw new Error(`File size (${fileStats.size} bytes) exceeds maximum allowed size (${maxFileSize} bytes)`);
      }

      // Check memory before operation
      if (this.isMemoryUsageHigh(memoryThreshold)) {
        logWarn('Reading file synchronously with high memory usage', undefined, {
          component: 'MemoryMonitor',
          operation: 'readFileSync',
          metadata: { filePath, fileSize: fileStats.size }
        });
      }

      const data = fs.readFileSync(filePath, encoding);
      
      // Check memory after operation
      this.checkMemoryUsage('sync file read');

      return data;

    } catch (error) {
      logError('Synchronous file read operation failed', error, {
        component: 'MemoryMonitor',
        operation: 'readFileSync',
        metadata: { filePath }
      });
      throw error;
    }
  }

  /**
   * Force garbage collection if available and memory usage is high
   */
  static forceGarbageCollection(): boolean {
    if (global.gc && this.isMemoryUsageHigh(70)) {
      const beforeStats = this.getMemoryStats();
      global.gc();
      const afterStats = this.getMemoryStats();
      
      logDebug('Forced garbage collection', undefined, {
        component: 'MemoryMonitor',
        operation: 'forceGC',
        metadata: {
          beforeMemoryMB: (beforeStats.heapUsed / 1024 / 1024).toFixed(2),
          afterMemoryMB: (afterStats.heapUsed / 1024 / 1024).toFixed(2),
          freedMB: ((beforeStats.heapUsed - afterStats.heapUsed) / 1024 / 1024).toFixed(2)
        }
      });
      
      return true;
    }
    
    return false;
  }

  /**
   * Stream large file processing for files that exceed memory limits
   */
  static async processLargeFile(
    filePath: string,
    processor: (chunk: string | Buffer) => void | Promise<void>,
    options: FileOperationOptions = {}
  ): Promise<void> {
    const {
      chunkSize = this.DEFAULT_CHUNK_SIZE,
      memoryThreshold = this.DEFAULT_MEMORY_THRESHOLD
    } = options;

    return new Promise((resolve, reject) => {
      const stream = fs.createReadStream(filePath, { 
        encoding: 'utf8',
        highWaterMark: chunkSize 
      });

      stream.on('data', async (chunk: string | Buffer) => {
        try {
          // Check memory before processing chunk
          if (this.isMemoryUsageHigh(memoryThreshold)) {
            logWarn('Processing file chunk with high memory usage', undefined, {
              component: 'MemoryMonitor',
              operation: 'processLargeFile',
              metadata: { filePath, chunkSize: chunk.length }
            });
            
            // Attempt garbage collection
            this.forceGarbageCollection();
          }

          await processor(chunk);
        } catch (error) {
          stream.destroy();
          reject(error);
        }
      });

      stream.on('end', () => {
        logDebug('Large file processing completed', undefined, {
          component: 'MemoryMonitor',
          operation: 'processLargeFile',
          metadata: { filePath }
        });
        resolve();
      });

      stream.on('error', (error) => {
        logError('Large file processing failed', error, {
          component: 'MemoryMonitor',
          operation: 'processLargeFile',
          metadata: { filePath }
        });
        reject(error);
      });
    });
  }
}