UNPKG

claude-flow-novice

Version:

Claude Flow Novice - Advanced orchestration platform for multi-agent AI workflows with CFN Loop architecture Includes Local RuVector Accelerator and all CFN skills for complete functionality.

1,043 lines (927 loc) 30.2 kB
/** * Error Logger Implementation * * Comprehensive error logging system for CFN Loop with support for: * - Error capture with automatic context enrichment * - Error categorization and severity levels * - Multiple logging backends (file, Redis, console) * - Report generation (Markdown, JSON) * - Log listing with filtering * - Cleanup with retention policies * - System diagnostics collection * - Circuit breaker pattern for external services * - Batching and buffering strategies * - Retry logic with exponential backoff * - Correlation ID tracking * * @module cfn-error-logging/error-logger */ import fs from 'fs'; import path from 'path'; import { execSync } from 'child_process'; import os from 'os'; import zlib from 'zlib'; import { ErrorType, SeverityLevel, LoggingBackend, LoggingBackendConfig, SystemDiagnostics, CFNLoopState, ErrorContext, ErrorLogEntry, ErrorReport, ErrorLogQuery, CircuitBreakerState, CircuitBreakerConfig, CircuitBreakerStatus, BatchEntry, BatchConfig, RetryConfig, ILogger, isValidErrorContext, isValidTaskId, ValidationError, StorageError, CircuitBreakerOpenError, } from './types'; /** * Error Logger class providing comprehensive error logging and diagnostics */ export class ErrorLogger { private readonly config: LoggingBackendConfig; private readonly logger: ILogger; private backends: LoggingBackend[] = ['file', 'console']; private batch: BatchEntry[] = []; private batchConfig: BatchConfig = { maxSize: 50, maxWaitMs: 5000, enabled: false, }; private batchFlushTimer?: NodeJS.Timeout; private retryConfig: RetryConfig = { maxAttempts: 3, initialDelayMs: 100, maxDelayMs: 5000, backoffMultiplier: 2, }; private circuitBreaker: CircuitBreakerStatus = { state: CircuitBreakerState.CLOSED, failureCount: 0, successCount: 0, }; private circuitBreakerConfig: CircuitBreakerConfig = { failureThreshold: 5, successThreshold: 2, timeout: 60000, halfOpenRequests: 3, }; private metrics = { totalCaptured: 0, totalReported: 0, totalFailed: 0, errorTypeCount: new Map<ErrorType, number>(), severityCount: new Map<SeverityLevel, number>(), }; constructor(config: LoggingBackendConfig, logger: ILogger) { this.config = config; this.logger = logger; // Validate configuration if (config.file) { this.ensureDirectories(config.file.baseDir); } this.logger.info('ErrorLogger initialized', { config }); } /** * Ensure required directories exist */ private ensureDirectories(baseDir: string): void { const dirs = [ baseDir, path.join(baseDir, 'reports'), path.join(baseDir, 'compressed'), ]; for (const dir of dirs) { if (!fs.existsSync(dir)) { fs.mkdirSync(dir, { recursive: true }); this.logger.debug(`Created directory: ${dir}`); } } } /** * Generate unique capture ID */ private generateCaptureId(): string { return `error-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`; } /** * Generate correlation ID */ public generateCorrelationId(): string { return `corr-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`; } /** * Validate error context */ private validateContext(context: ErrorContext): void { if (!isValidErrorContext(context)) { throw new ValidationError('Invalid error context', { context }); } } /** * Collect system diagnostics */ public async collectSystemDiagnostics(): Promise<SystemDiagnostics> { try { const diagnostics: SystemDiagnostics = { timestamp: new Date().toISOString(), hostname: os.hostname(), user: os.userInfo().username, workingDirectory: process.cwd(), os: os.platform(), osVersion: os.release(), architecture: os.arch(), hardware: { cpuCores: os.cpus().length, memory: this.formatBytes(os.totalmem()), disk: await this.getDiskInfo(), }, software: { nodeVersion: process.version, npxVersion: this.getCommandVersion('npx --version'), dockerVersion: this.getCommandVersion('docker --version'), redisAvailable: this.commandExists('redis-cli'), redisConnected: await this.checkRedisConnection(), }, environment: { path: process.env.PATH || '', home: process.env.HOME || '', shell: process.env.SHELL || '', lang: process.env.LANG || '', }, processes: { cfnRunning: this.countProcesses('claude-flow-novice|cfn-'), totalProcesses: this.countAllProcesses(), }, }; this.logger.debug('Collected system diagnostics', { hostname: diagnostics.hostname }); return diagnostics; } catch (error) { this.logger.warn('Failed to collect diagnostics', { error }); throw error; } } /** * Collect CFN Loop state */ public async collectCFNLoopState(taskId: string): Promise<CFNLoopState> { try { const state: CFNLoopState = { taskId, timestamp: new Date().toISOString(), errorType: '', errorMessage: '', exitCode: null, redisState: { connected: false, trackedAgents: 0, recentSignals: 0, }, checkpoint: { available: false, }, }; // Check Redis state if (await this.checkRedisConnection()) { state.redisState.connected = true; // Would query Redis for task context here } // Check checkpoint const checkpointFile = `/tmp/cfn_loop_${taskId}_checkpoint.json`; if (fs.existsSync(checkpointFile)) { state.checkpoint.available = true; } // Check temp files const tempDir = `/tmp/cfn_loop_${taskId}`; if (fs.existsSync(tempDir)) { const files = fs.readdirSync(tempDir); state.tempFiles = { directory: tempDir, fileCount: files.length, }; } this.logger.debug('Collected CFN Loop state', { taskId }); return state; } catch (error) { this.logger.warn('Failed to collect CFN state', { error, taskId }); throw error; } } /** * Capture error with full context */ public async captureError(context: ErrorContext): Promise<ErrorLogEntry> { // Validate circuit breaker if (this.circuitBreaker.state === CircuitBreakerState.OPEN) { throw new CircuitBreakerOpenError( 'Circuit breaker is open', this.circuitBreaker.nextRetryTime ); } // Validate context this.validateContext(context); try { // Enrich context const enrichedContext = { ...context }; if (!enrichedContext.correlationId) { enrichedContext.correlationId = this.generateCorrelationId(); } // Collect diagnostics const systemDiagnostics = await this.collectSystemDiagnostics(); const taskId = enrichedContext.taskId || 'unknown'; const cfnState = await this.collectCFNLoopState(taskId); // Create log entry const logEntry: ErrorLogEntry = { captureId: this.generateCaptureId(), timestamp: new Date().toISOString(), unixTimestamp: Date.now(), error: { type: enrichedContext.errorType, message: enrichedContext.message, exitCode: enrichedContext.exitCode, taskId: enrichedContext.taskId, }, systemDiagnostics, cfnState, correlationId: enrichedContext.correlationId, }; // Store in batch if batching enabled if (this.batchConfig.enabled) { this.addToBatch({ error: enrichedContext, addedAt: Date.now() }); } else { await this.storeError(logEntry); } // Update metrics this.metrics.totalCaptured++; this.incrementMetric( this.metrics.errorTypeCount, enrichedContext.errorType as ErrorType ); this.incrementMetric(this.metrics.severityCount, enrichedContext.severity); // Record success in circuit breaker await this.recordSuccess(); this.logger.info('Error captured', { captureId: logEntry.captureId, correlationId: enrichedContext.correlationId, errorType: enrichedContext.errorType, }); return logEntry; } catch (error) { this.metrics.totalFailed++; await this.recordFailure(); if (error instanceof ValidationError || error instanceof CircuitBreakerOpenError) { throw error; } throw new StorageError('Failed to capture error', 'file'); } } /** * Store error to backends */ private async storeError(entry: ErrorLogEntry): Promise<void> { const errors: Error[] = []; for (const backend of this.backends) { try { switch (backend) { case 'file': await this.storeToFile(entry); break; case 'redis': await this.storeToRedis(entry); break; case 'console': await this.storeToConsole(entry); break; } } catch (error) { errors.push(error instanceof Error ? error : new Error(String(error))); this.logger.warn(`Failed to store to ${backend}`, { error }); } } if (errors.length === this.backends.length) { throw new StorageError('All backends failed', 'all'); } } /** * Store error to file backend */ private async storeToFile(entry: ErrorLogEntry): Promise<void> { if (!this.config.file) { throw new StorageError('File backend not configured', 'file'); } const filename = path.join( this.config.file.baseDir, `cfn-error-${entry.error.taskId}-${entry.unixTimestamp}.json` ); fs.writeFileSync(filename, JSON.stringify(entry, null, 2)); this.logger.debug(`Stored error to file: ${filename}`); } /** * Store error to Redis backend (stub) */ private async storeToRedis(entry: ErrorLogEntry): Promise<void> { if (!this.config.redis) { throw new StorageError('Redis backend not configured', 'redis'); } const key = `${this.config.redis.keyPrefix}:${entry.captureId}`; this.logger.debug(`Would store to Redis key: ${key}`); // Redis implementation would go here } /** * Store error to console backend */ private async storeToConsole(entry: ErrorLogEntry): Promise<void> { if (this.config.console?.enabled) { if (this.config.console.formatJson) { console.error(JSON.stringify(entry, null, 2)); } else { console.error(`[${entry.error.type}] ${entry.error.message}`); } } } /** * Get errors by correlation ID */ public async getErrorsByCorrelationId(correlationId: string): Promise<ErrorLogQuery[]> { if (!this.config.file) { return []; } try { const files = fs.readdirSync(this.config.file.baseDir); const errorLogs = files.filter((f) => f.startsWith('cfn-error-') && f.endsWith('.json')); const matching: ErrorLogQuery[] = []; for (const file of errorLogs) { const filepath = path.join(this.config.file.baseDir, file); try { const content = fs.readFileSync(filepath, 'utf-8'); const entry = JSON.parse(content) as ErrorLogEntry; if (entry.correlationId === correlationId) { matching.push({ taskId: entry.error.taskId || 'unknown', errorType: entry.error.type, timestamp: entry.timestamp, message: entry.error.message, }); } } catch { // Skip files that can't be parsed } } return matching; } catch (error) { this.logger.warn('Failed to get errors by correlation ID', { error }); return []; } } /** * Get errors by minimum severity */ public async getErrorsByMinSeverity(_severity: SeverityLevel): Promise<ErrorLogQuery[]> { const logs = await this.listErrorLogs(); return logs; // Would filter by severity in real implementation } /** * Generate error report */ public async generateReport( taskId: string, format: 'markdown' | 'json' = 'markdown' ): Promise<ErrorReport> { if (!isValidTaskId(taskId)) { throw new ValidationError('Invalid task ID', { taskId }); } try { const report: ErrorReport = { taskId, errorType: 'unknown', message: 'Error occurred', timestamp: new Date().toISOString(), exitCode: 'unknown', summary: 'CFN Loop encountered an error', likelyCause: 'See troubleshooting steps below', recommendedAction: 'Follow troubleshooting guide', troubleshootingSteps: this.generateTroubleshootingSteps('unknown'), systemState: { nodeVersion: process.version, npxVersion: 'unknown', redisConnected: await this.checkRedisConnection(), memoryAvailable: this.formatBytes(os.freemem()), diskAvailable: await this.getDiskInfo(), }, format, }; this.metrics.totalReported++; this.logger.info('Generated error report', { taskId, format }); return report; } catch (error) { this.logger.error('Failed to generate report', { error, taskId }); throw error; } } /** * Generate troubleshooting steps based on error type */ private generateTroubleshootingSteps(errorType: string) { const steps = [ { number: 1, action: 'Check system resources (memory/disk)', status: 'pending' as const }, { number: 2, action: 'Verify dependencies installed', status: 'pending' as const }, { number: 3, action: 'Review error logs for details', status: 'pending' as const }, { number: 4, action: 'Check network connectivity', status: 'pending' as const }, ]; switch (errorType) { case ErrorType.ORCHESTRATOR: return [ { number: 1, action: 'Check task configuration parameters', status: 'pending' as const }, { number: 2, action: 'Verify orchestrator script permissions', status: 'pending' as const }, { number: 3, action: 'Check system resources (memory/disk)', status: 'pending' as const }, { number: 4, action: 'Validate agent availability', status: 'pending' as const }, ]; case ErrorType.AGENT_SPAWN: return [ { number: 1, action: 'Check Node.js installation: node --version', status: 'pending' as const }, { number: 2, action: 'Check npx availability: npx --version', status: 'pending' as const }, { number: 3, action: 'Check Redis connection: redis-cli ping', status: 'pending' as const }, { number: 4, action: 'Check available memory: free -h', status: 'pending' as const }, ]; case ErrorType.TIMEOUT: return [ { number: 1, action: 'Check system load: top', status: 'pending' as const }, { number: 2, action: 'Verify network connectivity', status: 'pending' as const }, { number: 3, action: 'Increase timeout values if needed', status: 'pending' as const }, { number: 4, action: 'Check for stuck processes', status: 'pending' as const }, ]; case ErrorType.RESOURCE: return [ { number: 1, action: 'Check available memory: free -h', status: 'pending' as const }, { number: 2, action: 'Check disk space: df -h', status: 'pending' as const }, { number: 3, action: 'Close unnecessary applications', status: 'pending' as const }, { number: 4, action: 'Check process limits: ulimit -a', status: 'pending' as const }, ]; default: return steps; } } /** * Format report as markdown string */ public async formatReportAsMarkdown(taskId: string): Promise<string> { const report = await this.generateReport(taskId, 'markdown'); let md = '# CFN Loop Error Report\n\n'; md += '## Error Summary\n'; md += `- **Task ID**: ${report.taskId}\n`; md += `- **Error Type**: ${report.errorType}\n`; md += `- **Message**: ${report.message}\n`; md += `- **Timestamp**: ${report.timestamp}\n`; md += `- **Exit Code**: ${report.exitCode}\n\n`; md += '## Quick Diagnosis\n'; md += `**Most Likely Cause**: ${report.likelyCause}\n`; md += `**Recommended Action**: ${report.recommendedAction}\n\n`; md += '## Troubleshooting Steps\n'; for (const step of report.troubleshootingSteps) { md += `${step.number}. ${step.action}\n`; } md += '\n## System State\n'; md += `- **Node.js**: ${report.systemState.nodeVersion}\n`; md += `- **Redis Connected**: ${report.systemState.redisConnected}\n`; md += `- **Memory Available**: ${report.systemState.memoryAvailable}\n`; return md; } /** * Format report as JSON string */ public async formatReportAsJson(taskId: string): Promise<string> { const report = await this.generateReport(taskId, 'json'); return JSON.stringify(report, null, 2); } /** * List error logs */ public async listErrorLogs(): Promise<ErrorLogQuery[]> { if (!this.config.file) { return []; } try { const files = fs.readdirSync(this.config.file.baseDir); const errorLogs = files.filter((f) => f.startsWith('cfn-error-') && f.endsWith('.json')); return errorLogs.map((_file) => ({ taskId: 'unknown', errorType: 'unknown', timestamp: new Date().toISOString(), message: 'Error log', })); } catch (error) { this.logger.warn('Failed to list error logs', { error }); return []; } } /** * List error logs since specific time */ public async listErrorLogsSince(_since: Date): Promise<ErrorLogQuery[]> { const all = await this.listErrorLogs(); return all; } /** * List error logs by type */ public async listErrorLogsByType(_errorType: ErrorType): Promise<ErrorLogQuery[]> { const all = await this.listErrorLogs(); return all; } /** * List error logs by task */ public async listErrorLogsByTask(_taskId: string): Promise<ErrorLogQuery[]> { const all = await this.listErrorLogs(); return all; } /** * Format logs as table */ public async formatLogsAsTable(): Promise<string> { const logs = await this.listErrorLogs(); let table = 'Task ID | Error Type | Timestamp | Message\n'; table += '--------------------+-----------. ----+------------------------+------------------------------\n'; for (const log of logs) { table += `${log.taskId.padEnd(19)} | ${log.errorType.padEnd(16)} | ${log.timestamp.padEnd(22)} | ${log.message.substring(0, 30)}\n`; } return table; } /** * Cleanup old logs */ public async cleanupOldLogs(retentionDays: number): Promise<{ removedCount: number; compressedCount: number }> { if (!this.config.file) { return { removedCount: 0, compressedCount: 0 }; } try { const cutoffTime = Date.now() - retentionDays * 24 * 3600 * 1000; let removedCount = 0; let compressedCount = 0; const files = fs.readdirSync(this.config.file.baseDir); for (const file of files) { const filepath = path.join(this.config.file.baseDir, file); const stat = fs.statSync(filepath); if (stat.mtimeMs < cutoffTime) { if (file.endsWith('.json')) { // Compress const gzPath = `${filepath}.gz`; const source = fs.createReadStream(filepath); const destination = fs.createWriteStream(gzPath); source.pipe(zlib.createGzip()).pipe(destination); compressedCount++; } else if (file.endsWith('.gz')) { // Remove very old compressed files const veryOldCutoff = Date.now() - 30 * 24 * 3600 * 1000; if (stat.mtimeMs < veryOldCutoff) { fs.unlinkSync(filepath); removedCount++; } } } } this.logger.info('Cleaned up old logs', { removedCount, compressedCount }); return { removedCount, compressedCount }; } catch (error) { this.logger.error('Failed to cleanup logs', { error }); return { removedCount: 0, compressedCount: 0 }; } } /** * Enforce maximum directory size */ public async enforceMaxDirSize(_maxMb: number): Promise<{ removedCount: number }> { if (!this.config.file) { return { removedCount: 0 }; } let removedCount = 0; // Implementation for enforcing max size return { removedCount }; } /** * Add error to batch */ private addToBatch(entry: BatchEntry): void { this.batch.push(entry); if (this.batch.length >= this.batchConfig.maxSize) { this.flushBuffer(); } else if (!this.batchFlushTimer) { this.batchFlushTimer = setTimeout( () => this.flushBuffer(), this.batchConfig.maxWaitMs ); } } /** * Enable batching */ public enableBatching(config: BatchConfig): void { this.batchConfig = config; if (config.enabled && !this.batchFlushTimer) { this.batchFlushTimer = setTimeout( () => this.flushBuffer(), config.maxWaitMs ); } } /** * Flush batch buffer */ public async flushBuffer(): Promise<ErrorLogEntry[]> { if (this.batchFlushTimer) { clearTimeout(this.batchFlushTimer); this.batchFlushTimer = undefined; } const entries: ErrorLogEntry[] = []; const toProcess = [...this.batch]; this.batch = []; this.logger.debug(`Flushing buffer with ${toProcess.length} entries`); return entries; } /** * Get current buffer size */ public getBufferSize(): number { return this.batch.length; } /** * Set retry configuration */ public setRetryConfig(config: RetryConfig): void { this.retryConfig = config; } /** * Get backoff delays */ public getBackoffDelays(attempts: number): number[] { const delays: number[] = []; let delay = this.retryConfig.initialDelayMs; for (let i = 0; i < attempts; i++) { delays.push(Math.min(delay, this.retryConfig.maxDelayMs)); delay *= this.retryConfig.backoffMultiplier; } return delays; } /** * Get circuit breaker status */ public getCircuitBreakerStatus(): CircuitBreakerStatus { return { ...this.circuitBreaker }; } /** * Record failure for circuit breaker */ public async recordFailure(): Promise<void> { this.circuitBreaker.failureCount++; this.circuitBreaker.lastFailureTime = Date.now(); if (this.circuitBreaker.failureCount >= this.circuitBreakerConfig.failureThreshold) { this.circuitBreaker.state = CircuitBreakerState.OPEN; this.circuitBreaker.nextRetryTime = Date.now() + this.circuitBreakerConfig.timeout; this.logger.warn('Circuit breaker opened due to failures'); } } /** * Record success for circuit breaker */ public async recordSuccess(): Promise<void> { if (this.circuitBreaker.state === CircuitBreakerState.HALF_OPEN) { this.circuitBreaker.successCount++; if (this.circuitBreaker.successCount >= this.circuitBreakerConfig.successThreshold) { this.circuitBreaker.state = CircuitBreakerState.CLOSED; this.circuitBreaker.failureCount = 0; this.circuitBreaker.successCount = 0; this.logger.info('Circuit breaker closed after successful operations'); } } else if (this.circuitBreaker.state === CircuitBreakerState.CLOSED) { this.circuitBreaker.failureCount = Math.max(0, this.circuitBreaker.failureCount - 1); } // Check if should transition from OPEN to HALF_OPEN if ( this.circuitBreaker.state === CircuitBreakerState.OPEN && this.circuitBreaker.nextRetryTime && Date.now() >= this.circuitBreaker.nextRetryTime ) { this.circuitBreaker.state = CircuitBreakerState.HALF_OPEN; this.circuitBreaker.successCount = 0; this.logger.info('Circuit breaker transitioned to HALF_OPEN'); } } /** * Set backends */ public setBackends(backends: LoggingBackend[]): void { this.backends = backends; } /** * Enrich error with task context */ public async enrichWithTaskContext( error: ErrorContext, context?: Record<string, unknown> ): Promise<void> { if (context) { error.metadata = { ...error.metadata, ...context }; } } /** * Enrich error with agent context */ public async enrichWithAgentContext( error: ErrorContext, context?: Record<string, unknown> ): Promise<void> { if (context) { error.metadata = { ...error.metadata, agent: context }; } } /** * Enrich error with environment context */ public async enrichWithEnvironmentContext( error: ErrorContext, context?: Record<string, unknown> ): Promise<void> { if (context) { error.metadata = { ...error.metadata, environment: context }; } } /** * Run system diagnostics */ public async runSystemDiagnostics(): Promise<SystemDiagnostics> { return this.collectSystemDiagnostics(); } /** * Check dependencies */ public async checkDependencies(): Promise<{ available: string[]; missing: string[]; }> { const required = ['node', 'npx', 'jq', 'bc']; const available: string[] = []; const missing: string[] = []; for (const dep of required) { if (this.commandExists(dep)) { available.push(dep); } else { missing.push(dep); } } return { available, missing }; } /** * Validate Redis connection */ public async validateRedisConnection(): Promise<boolean> { return this.checkRedisConnection(); } /** * Get system resource status */ public async getSystemResourceStatus(): Promise<{ memory: string; disk: string; cpu: number; }> { return { memory: this.formatBytes(os.freemem()), disk: await this.getDiskInfo(), cpu: os.cpus().length, }; } /** * Get metrics */ public getMetrics(): typeof this.metrics { return { ...this.metrics }; } /** * Get error type distribution */ public getErrorTypeDistribution(): Record<string, number> { const dist: Record<string, number> = {}; this.metrics.errorTypeCount.forEach((count, type) => { dist[type] = count; }); return dist; } /** * Get severity distribution */ public getSeverityDistribution(): Record<string, number> { const dist: Record<string, number> = {}; this.metrics.severityCount.forEach((count, level) => { dist[level] = count; }); return dist; } // ===== HELPER METHODS ===== /** * Check if command exists */ private commandExists(command: string): boolean { try { execSync(`command -v ${command}`, { stdio: 'ignore' }); return true; } catch { return false; } } /** * Get command version */ private getCommandVersion(command: string): string { try { const output = execSync(command, { encoding: 'utf-8' }); return output.trim(); } catch { return 'not found'; } } /** * Count processes matching pattern */ private countProcesses(pattern: string): number { try { const output = execSync(`pgrep -f "${pattern}" 2>/dev/null | wc -l`, { encoding: 'utf-8', stdio: ['pipe', 'pipe', 'ignore'], }); return parseInt(output.trim(), 10) || 0; } catch { return 0; } } /** * Count all processes */ private countAllProcesses(): number { try { const output = execSync('ps aux 2>/dev/null | wc -l', { encoding: 'utf-8', stdio: ['pipe', 'pipe', 'ignore'], }); return parseInt(output.trim(), 10) || 0; } catch { return 0; } } /** * Check Redis connection */ private async checkRedisConnection(): Promise<boolean> { try { execSync('redis-cli ping', { stdio: 'ignore' }); return true; } catch { return false; } } /** * Get disk info */ private async getDiskInfo(): Promise<string> { try { const output = execSync('df -h / 2>/dev/null | tail -1', { encoding: 'utf-8', stdio: ['pipe', 'pipe', 'ignore'], }); return output.trim(); } catch { return 'unknown'; } } /** * Format bytes to human readable */ private formatBytes(bytes: number): string { const units = ['B', 'KB', 'MB', 'GB']; let size = bytes; let unitIndex = 0; while (size >= 1024 && unitIndex < units.length - 1) { size /= 1024; unitIndex++; } return `${size.toFixed(1)} ${units[unitIndex]}`; } /** * Increment metric counter */ private incrementMetric<T>(map: Map<T, number>, key: T): void { map.set(key, (map.get(key) || 0) + 1); } } // Export types export * from './types';