UNPKG

logs-interceptor

Version:

High-performance, production-ready log interceptor for Node.js applications with Loki integration. Built with Clean Architecture principles. Supports Node.js, Browser, and Node-RED.

832 lines 33.2 kB
"use strict"; var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { if (k2 === undefined) k2 = k; var desc = Object.getOwnPropertyDescriptor(m, k); if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { desc = { enumerable: true, get: function() { return m[k]; } }; } Object.defineProperty(o, k2, desc); }) : (function(o, m, k, k2) { if (k2 === undefined) k2 = k; o[k2] = m[k]; })); var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { Object.defineProperty(o, "default", { enumerable: true, value: v }); }) : function(o, v) { o["default"] = v; }); var __importStar = (this && this.__importStar) || function (mod) { if (mod && mod.__esModule) return mod; var result = {}; if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); __setModuleDefault(result, mod); return result; }; var __importDefault = (this && this.__importDefault) || function (mod) { return (mod && mod.__esModule) ? mod : { "default": mod }; }; Object.defineProperty(exports, "__esModule", { value: true }); exports.LogsInterceptor = void 0; const api_1 = require("@opentelemetry/api"); const async_hooks_1 = require("async_hooks"); const axios_1 = __importDefault(require("axios")); const crypto = __importStar(require("crypto")); const events_1 = require("events"); const os = __importStar(require("os")); const perf_hooks_1 = require("perf_hooks"); const utils_1 = require("./utils"); // AsyncLocalStorage para contexto de requisição const asyncLocalStorage = new async_hooks_1.AsyncLocalStorage(); function createDebugLogger(enabled) { const formatMessage = (level, message, context) => { const timestamp = new Date().toISOString(); const prefix = `[${timestamp}] [logs-interceptor:${level.toUpperCase()}]`; if (context && Object.keys(context).length > 0) { return `${prefix} ${message} ${(0, utils_1.safeStringify)(context)}`; } return `${prefix} ${message}`; }; const noop = () => { }; if (!enabled) { return { debug: noop, info: noop, warn: noop, error: noop }; } return { debug: (message, context) => { process.stderr.write(formatMessage('debug', message, context) + '\n'); }, info: (message, context) => { process.stderr.write(formatMessage('info', message, context) + '\n'); }, warn: (message, context) => { process.stderr.write(formatMessage('warn', message, context) + '\n'); }, error: (message, context) => { process.stderr.write(formatMessage('error', message, context) + '\n'); }, }; } class LogsInterceptor extends events_1.EventEmitter { constructor(config) { super(); this.buffer = []; this.flushTimer = null; this.isDestroyed = false; // Circuit Breaker this.circuitBreaker = { state: 'closed', failures: 0, successCount: 0, lastFailure: 0, nextAttempt: 0, }; // Memory Management this.memoryCheckTimer = null; this.lastMemoryUsage = 0; // Worker for background processing (removed - causing issues) // private flushWorker: Worker | null = null; // Integrations this.integrations = new Map(); this.config = this.validateAndNormalizeConfig(config); this.debugLogger = createDebugLogger(this.config.debug); this.startTime = Date.now(); // Initialize metrics with more detail this.metrics = { logsProcessed: 0, logsDropped: 0, logsSanitized: 0, flushCount: 0, errorCount: 0, bufferSize: 0, avgFlushTime: 0, lastFlushTime: 0, memoryUsage: 0, cpuUsage: 0, circuitBreakerTrips: 0, }; // Store original console methods this.originalConsole = { log: console.log.bind(console), info: console.info.bind(console), warn: console.warn.bind(console), error: console.error.bind(console), debug: console.debug.bind(console), }; this.setupHttpClient(); this.setupProcessHandlers(); this.setupMemoryMonitoring(); // this.setupWorker(); // Commented out - not fully implemented if (this.config.interceptConsole) { this.interceptConsole(); } // Setup integrations if (this.config.integrations?.winston) { this.setupWinstonIntegration(); } if (this.config.integrations?.pino) { this.setupPinoIntegration(); } if (this.config.integrations?.morgan) { this.setupMorganIntegration(); } this.debugLogger.info('LogsInterceptor initialized', { appName: this.config.appName, environment: this.config.environment, bufferSize: this.config.buffer.maxSize, flushInterval: this.config.buffer.flushInterval, integrations: Object.keys(this.config.integrations || {}), }); } validateAndNormalizeConfig(config) { if (!config.transport?.url) { throw new Error('Transport URL is required'); } if (!config.transport?.tenantId) { throw new Error('Tenant ID is required'); } if (!config.appName) { throw new Error('App name is required'); } return { transport: { url: config.transport.url, tenantId: config.transport.tenantId, authToken: config.transport.authToken ?? '', timeout: config.transport.timeout ?? 5000, maxRetries: config.transport.maxRetries ?? 3, retryDelay: config.transport.retryDelay ?? 1000, compression: config.transport.compression ?? true, }, appName: config.appName, version: config.version ?? '1.0.0', environment: config.environment ?? 'production', labels: config.labels ?? {}, dynamicLabels: { trace_id: () => { const span = api_1.trace.getSpan(api_1.context.active()); return span?.spanContext().traceId ?? 'undefined'; }, span_id: () => { const span = api_1.trace.getSpan(api_1.context.active()); return span?.spanContext().spanId ?? 'undefined'; }, request_id: () => { const store = asyncLocalStorage.getStore(); return store?.requestId ?? 'undefined'; }, ...(config.dynamicLabels ?? {}), }, buffer: { maxSize: config.buffer?.maxSize ?? 100, flushInterval: config.buffer?.flushInterval ?? 5000, maxAge: config.buffer?.maxAge ?? 30000, autoFlush: config.buffer?.autoFlush ?? true, maxMemoryMB: config.buffer?.maxMemoryMB ?? 50, }, filter: { levels: config.filter?.levels ?? ['debug', 'info', 'warn', 'error', 'fatal'], patterns: config.filter?.patterns ?? [], samplingRate: config.filter?.samplingRate ?? 1.0, maxMessageLength: config.filter?.maxMessageLength ?? 8192, sanitize: config.filter?.sanitize ?? true, sensitivePatterns: config.filter?.sensitivePatterns ?? [ /password/i, /token/i, /secret/i, /api[_-]?key/i, /authorization/i, /credit[_-]?card/i, /ssn/i, /cpf/i, ], }, circuitBreaker: { enabled: config.circuitBreaker?.enabled ?? true, failureThreshold: config.circuitBreaker?.failureThreshold ?? 5, resetTimeout: config.circuitBreaker?.resetTimeout ?? 60000, halfOpenRequests: config.circuitBreaker?.halfOpenRequests ?? 3, }, integrations: config.integrations ?? {}, performance: { useWorkers: config.performance?.useWorkers ?? true, maxConcurrentFlushes: config.performance?.maxConcurrentFlushes ?? 3, compressionLevel: config.performance?.compressionLevel ?? 6, }, enableMetrics: config.enableMetrics ?? true, enableHealthCheck: config.enableHealthCheck ?? true, interceptConsole: config.interceptConsole ?? false, preserveOriginalConsole: config.preserveOriginalConsole ?? true, debug: config.debug ?? false, silentErrors: config.silentErrors ?? false, }; } setupHttpClient() { const headers = { 'Content-Type': 'application/json', 'X-Scope-OrgID': this.config.transport.tenantId, 'User-Agent': `logs-interceptor/${this.config.version}`, }; if (this.config.transport.authToken) { headers['Authorization'] = `Bearer ${this.config.transport.authToken}`; } this.httpClient = axios_1.default.create({ baseURL: this.config.transport.url, timeout: this.config.transport.timeout, headers, maxContentLength: 100 * 1024 * 1024, maxBodyLength: 100 * 1024 * 1024, }); // Add request/response interceptors with timing this.httpClient.interceptors.request.use((config) => { config.metadata = { startTime: perf_hooks_1.performance.now() }; return config; }); this.httpClient.interceptors.response.use((response) => { const duration = perf_hooks_1.performance.now() - response.config.metadata?.startTime; this.debugLogger.debug('Request successful', { duration }); return response; }, (error) => { this.handleTransportError(error); throw error; }); } setupProcessHandlers() { const gracefulShutdown = async (signal) => { this.debugLogger.info(`Graceful shutdown initiated (${signal})`); // Stop accepting new logs this.isDestroyed = true; // Clear timers if (this.flushTimer) { clearTimeout(this.flushTimer); } if (this.memoryCheckTimer) { clearTimeout(this.memoryCheckTimer); } // Final flush with timeout const flushTimeout = new Promise((resolve) => setTimeout(resolve, 5000)); await Promise.race([this.flush(), flushTimeout]); process.exit(0); }; process.on('SIGINT', () => gracefulShutdown('SIGINT')); process.on('SIGTERM', () => gracefulShutdown('SIGTERM')); process.on('uncaughtException', (error) => { this.error('Uncaught exception', { error: error.message, stack: error.stack, code: 'UNCAUGHT_EXCEPTION' }); this.flushSync(); process.exit(1); }); process.on('unhandledRejection', (reason, promise) => { this.error('Unhandled rejection', { reason: (0, utils_1.safeStringify)(reason), promise: (0, utils_1.safeStringify)(promise), code: 'UNHANDLED_REJECTION' }); this.flushSync(); }); process.on('warning', (warning) => { this.warn('Process warning', { name: warning.name, message: warning.message, stack: warning.stack, }); }); } setupMemoryMonitoring() { if (!this.config.enableMetrics) return; this.memoryCheckTimer = setInterval(() => { const memUsage = process.memoryUsage(); const heapUsedMB = memUsage.heapUsed / 1024 / 1024; this.metrics.memoryUsage = heapUsedMB; this.metrics.cpuUsage = process.cpuUsage().user / 1000000; // Convert to seconds // Check memory threshold if (heapUsedMB > this.config.buffer.maxMemoryMB) { this.debugLogger.warn('Memory threshold exceeded, forcing flush', { heapUsedMB, threshold: this.config.buffer.maxMemoryMB, }); // Drop oldest logs if buffer is too large const bufferSizeMB = JSON.stringify(this.buffer).length / 1024 / 1024; if (bufferSizeMB > this.config.buffer.maxMemoryMB / 2) { const dropCount = Math.floor(this.buffer.length * 0.3); this.buffer = this.buffer.slice(dropCount); this.metrics.logsDropped += dropCount; this.debugLogger.warn(`Dropped ${dropCount} logs due to memory pressure`); } this.flush().catch(() => { }); } }, 10000); // Check every 10 seconds } setupWorker() { // Worker threads implementation - to be added in production // This would handle background log processing if (!this.config.performance.useWorkers) return; this.debugLogger.debug('Worker threads not implemented yet'); } setupWinstonIntegration() { try { const winston = require('winston'); const customTransport = new winston.transports.Stream({ stream: { write: (message) => { try { const log = JSON.parse(message); this.log(log.level || 'info', log.message || message, { ...log, source: 'winston' }); } catch { this.info(message, { source: 'winston' }); } } } }); this.integrations.set('winston', customTransport); this.debugLogger.info('Winston integration setup complete'); } catch (error) { this.debugLogger.error('Failed to setup Winston integration', { error }); } } setupPinoIntegration() { try { const pino = require('pino'); const stream = pino.destination({ write: (data) => { try { const log = JSON.parse(data); const level = this.mapPinoLevel(log.level); this.log(level, log.msg || '', { ...log, source: 'pino' }); } catch { this.info(data, { source: 'pino' }); } } }); this.integrations.set('pino', stream); this.debugLogger.info('Pino integration setup complete'); } catch (error) { this.debugLogger.error('Failed to setup Pino integration', { error }); } } setupMorganIntegration() { try { // Morgan integration for Express const stream = { write: (message) => { this.info(message.trim(), { source: 'morgan', type: 'http_request' }); } }; this.integrations.set('morgan', stream); this.debugLogger.info('Morgan integration setup complete'); } catch (error) { this.debugLogger.error('Failed to setup Morgan integration', { error }); } } mapPinoLevel(level) { if (level <= 20) return 'debug'; if (level <= 30) return 'info'; if (level <= 40) return 'warn'; if (level <= 50) return 'error'; return 'fatal'; } interceptConsole() { const methodMap = { log: 'info', info: 'info', warn: 'warn', error: 'error', debug: 'debug', }; ['log', 'info', 'warn', 'error', 'debug'].forEach((method) => { const original = this.originalConsole[method]; console[method] = (...args) => { // Skip logs from logs-interceptor itself const stack = new Error().stack || ''; if (stack.includes('logs-interceptor')) { if (this.config.preserveOriginalConsole) { original(...args); } return; } const message = args .map((arg) => (typeof arg === 'string' ? arg : (0, utils_1.safeStringify)(arg))) .join(' '); this.log(methodMap[method] || 'info', message, { source: 'console' }); if (this.config.preserveOriginalConsole) { original(...args); } }; }); } restoreConsole() { console.log = this.originalConsole.log; console.info = this.originalConsole.info; console.warn = this.originalConsole.warn; console.error = this.originalConsole.error; console.debug = this.originalConsole.debug; } handleTransportError(error) { this.metrics.errorCount++; this.circuitBreaker.failures++; this.circuitBreaker.lastFailure = Date.now(); // Circuit breaker logic if (this.config.circuitBreaker.enabled) { if (this.circuitBreaker.failures >= this.config.circuitBreaker.failureThreshold) { this.circuitBreaker.state = 'open'; this.circuitBreaker.nextAttempt = Date.now() + this.config.circuitBreaker.resetTimeout; this.metrics.circuitBreakerTrips++; this.debugLogger.error('Circuit breaker opened', { failures: this.circuitBreaker.failures, nextAttempt: new Date(this.circuitBreaker.nextAttempt).toISOString(), }); } } this.emit('error', error); if (!this.config.silentErrors) { this.debugLogger.error('Transport error', { error: error.message, status: error.response?.status, statusText: error.response?.statusText, data: error.response?.data, }); } } isCircuitOpen() { if (!this.config.circuitBreaker.enabled) return false; if (this.circuitBreaker.state === 'open') { if (Date.now() >= this.circuitBreaker.nextAttempt) { this.circuitBreaker.state = 'half-open'; this.circuitBreaker.successCount = 0; this.debugLogger.info('Circuit breaker half-open'); } else { return true; } } return false; } onSuccessfulFlush() { if (this.circuitBreaker.state === 'half-open') { this.circuitBreaker.successCount++; if (this.circuitBreaker.successCount >= this.config.circuitBreaker.halfOpenRequests) { this.circuitBreaker.state = 'closed'; this.circuitBreaker.failures = 0; this.circuitBreaker.successCount = 0; this.debugLogger.info('Circuit breaker closed'); } } else if (this.circuitBreaker.state === 'closed') { this.circuitBreaker.failures = 0; } } shouldLog(level, message) { // Check if level is enabled if (!this.config.filter.levels.includes(level)) { return false; } // Check message patterns if (this.config.filter.patterns.length > 0) { const shouldInclude = this.config.filter.patterns.some(pattern => pattern.test(message)); if (!shouldInclude) { return false; } } // Apply sampling if (!(0, utils_1.shouldSample)(this.config.filter.samplingRate)) { this.metrics.logsDropped++; return false; } return true; } createLogEntry(level, message, context) { // Truncate message if too long const truncatedMessage = message.length > this.config.filter.maxMessageLength ? message.substring(0, this.config.filter.maxMessageLength) + '...[truncated]' : message; // Sanitize sensitive data let sanitizedContext = context; if (this.config.filter.sanitize && context) { sanitizedContext = (0, utils_1.sanitizeData)(context, this.config.filter.sensitivePatterns); if ((0, utils_1.detectSensitiveData)(message, this.config.filter.sensitivePatterns)) { this.metrics.logsSanitized++; } } // Get async context const asyncContext = asyncLocalStorage.getStore() || {}; // Compute dynamic labels const dynamicLabels = Object.entries(this.config.dynamicLabels).reduce((acc, [key, fn]) => { try { acc[key] = String(fn()); } catch (error) { acc[key] = 'error'; this.debugLogger.warn(`Failed to compute dynamic label ${key}`, { error }); } return acc; }, {}); // Generate unique log ID const logId = crypto.randomBytes(8).toString('hex'); return { id: logId, timestamp: new Date().toISOString(), level, message: truncatedMessage, context: { ...asyncContext, ...sanitizedContext, }, traceId: dynamicLabels.trace_id, spanId: dynamicLabels.span_id, requestId: dynamicLabels.request_id, labels: { app: this.config.appName, version: this.config.version, environment: this.config.environment, level, hostname: os.hostname(), pid: String(process.pid), ...this.config.labels, ...dynamicLabels, }, metadata: { memoryUsage: this.metrics.memoryUsage, cpuUsage: this.metrics.cpuUsage, } }; } scheduleFlush() { if (this.flushTimer || !this.config.buffer.autoFlush) return; this.flushTimer = setTimeout(() => { this.flush().catch((error) => { this.debugLogger.error('Scheduled flush failed', { error }); }); }, this.config.buffer.flushInterval); } async retryOperation(operation, maxRetries = this.config.transport.maxRetries, delay = this.config.transport.retryDelay) { let lastError; for (let attempt = 1; attempt <= maxRetries; attempt++) { try { return await operation(); } catch (error) { lastError = error; if (attempt === maxRetries) { throw lastError; } // Exponential backoff const backoffDelay = delay * Math.pow(2, attempt - 1); const jitter = Math.random() * 1000; this.debugLogger.warn(`Attempt ${attempt} failed, retrying in ${backoffDelay + jitter}ms`, { error: lastError.message, }); await new Promise(resolve => setTimeout(resolve, backoffDelay + jitter)); } } throw lastError; } formatForLoki(entries) { const streamMap = new Map(); entries.forEach((entry) => { const streamKey = JSON.stringify(entry.labels); const timestamp = String(Date.parse(entry.timestamp) * 1000000); // nanoseconds // Format log line with structure const logData = { id: entry.id, level: entry.level, message: entry.message, context: entry.context, traceId: entry.traceId !== 'undefined' ? entry.traceId : undefined, spanId: entry.spanId !== 'undefined' ? entry.spanId : undefined, requestId: entry.requestId !== 'undefined' ? entry.requestId : undefined, metadata: entry.metadata, }; // Remove undefined values Object.keys(logData).forEach(key => { if (logData[key] === undefined) delete logData[key]; }); const logLine = JSON.stringify(logData); if (!streamMap.has(streamKey)) { streamMap.set(streamKey, []); } streamMap.get(streamKey).push([timestamp, logLine]); }); return { streams: Array.from(streamMap.entries()).map(([streamKey, values]) => ({ stream: JSON.parse(streamKey), values: values.sort((a, b) => a[0].localeCompare(b[0])), // Sort by timestamp })), }; } flushSync() { if (this.buffer.length === 0) return; try { // Attempt to write to disk as fallback const fs = require('fs'); const path = require('path'); const logFile = path.join(process.cwd(), `logs-interceptor-emergency-${Date.now()}.json`); fs.writeFileSync(logFile, JSON.stringify({ timestamp: new Date().toISOString(), logs: this.buffer, metadata: { appName: this.config.appName, environment: this.config.environment, reason: 'emergency_flush', } })); this.debugLogger.warn(`Emergency logs written to ${logFile}`); } catch (error) { this.debugLogger.error('Failed to write emergency logs', { error }); } } // Public API methods log(level, message, context) { if (this.isDestroyed || !this.shouldLog(level, message)) { return; } const entry = this.createLogEntry(level, message, context); this.buffer.push(entry); this.metrics.logsProcessed++; this.metrics.bufferSize = this.buffer.length; this.emit('log', entry); // Auto-flush if buffer is full if (this.buffer.length >= this.config.buffer.maxSize) { this.flush().catch((error) => { this.debugLogger.error('Auto-flush failed', { error }); }); } else { this.scheduleFlush(); } } debug(message, context) { this.log('debug', message, context); } info(message, context) { this.log('info', message, context); } warn(message, context) { this.log('warn', message, context); } error(message, context) { this.log('error', message, context); } fatal(message, context) { this.log('fatal', message, context); // Force immediate flush for fatal errors this.flush().catch(() => { // Silent fail for fatal errors }); } trackEvent(eventName, properties) { this.info(`[EVENT] ${eventName}`, properties); } async flush() { // Check circuit breaker if (this.isCircuitOpen()) { this.debugLogger.warn('Circuit breaker is open, skipping flush'); return; } if (this.flushTimer) { clearTimeout(this.flushTimer); this.flushTimer = null; } if (this.buffer.length === 0) { return; } const entries = this.buffer.splice(0); const startTime = perf_hooks_1.performance.now(); try { const payload = this.formatForLoki(entries); // Debug payload if (this.config.debug) { this.debugLogger.debug('Sending payload to Loki', { url: this.config.transport.url, streamCount: payload.streams.length, totalLogs: payload.streams.reduce((sum, s) => sum + s.values.length, 0), }); } await this.retryOperation(async () => { let data = JSON.stringify(payload); const headers = {}; // Add gzip compression if enabled if (this.config.transport.compression) { const zlib = require('zlib'); data = zlib.gzipSync(data, { level: this.config.performance.compressionLevel }); headers['Content-Encoding'] = 'gzip'; } const response = await this.httpClient.post('', data, { headers }); return response; }); const flushTime = perf_hooks_1.performance.now() - startTime; this.metrics.flushCount++; this.metrics.lastFlushTime = Date.now(); this.metrics.avgFlushTime = (this.metrics.avgFlushTime * (this.metrics.flushCount - 1) + flushTime) / this.metrics.flushCount; this.metrics.bufferSize = this.buffer.length; this.onSuccessfulFlush(); this.emit('flush', { count: entries.length, duration: flushTime }); this.debugLogger.debug(`Flushed ${entries.length} logs in ${flushTime.toFixed(2)}ms`); } catch (error) { // Re-add entries to buffer on failure (with limit) if (this.buffer.length < this.config.buffer.maxSize * 2) { this.buffer.unshift(...entries.slice(0, this.config.buffer.maxSize)); } else { this.metrics.logsDropped += entries.length; this.debugLogger.error('Buffer overflow, dropping logs', { dropped: entries.length }); } this.handleTransportError(error); throw error; } } // Add context for async operations runWithContext(context, fn) { return asyncLocalStorage.run(context, fn); } // Add async version for better support async runWithContextAsync(context, fn) { return asyncLocalStorage.run(context, fn); } getMetrics() { return { ...this.metrics, bufferSize: this.buffer.length }; } getHealth() { const now = Date.now(); const timeSinceLastFlush = now - this.metrics.lastFlushTime; const bufferUtilization = this.buffer.length / this.config.buffer.maxSize; return { healthy: this.metrics.errorCount < 10 && timeSinceLastFlush < this.config.buffer.flushInterval * 5 && bufferUtilization < 0.9 && this.circuitBreaker.state !== 'open', lastSuccessfulFlush: this.metrics.lastFlushTime, consecutiveErrors: this.metrics.errorCount, bufferUtilization, uptime: now - this.startTime, memoryUsageMB: this.metrics.memoryUsage, circuitBreakerState: this.circuitBreaker.state, }; } // Get integration streams for external loggers getWinstonTransport() { return this.integrations.get('winston'); } getPinoStream() { return this.integrations.get('pino'); } getMorganStream() { return this.integrations.get('morgan'); } async destroy() { if (this.isDestroyed) return; this.debugLogger.info('Destroying LogsInterceptor'); this.isDestroyed = true; if (this.flushTimer) { clearTimeout(this.flushTimer); this.flushTimer = null; } if (this.memoryCheckTimer) { clearTimeout(this.memoryCheckTimer); this.memoryCheckTimer = null; } if (this.config.interceptConsole) { this.restoreConsole(); } // Final flush with timeout try { const flushPromise = this.flush(); const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('Final flush timeout')), 5000)); await Promise.race([flushPromise, timeoutPromise]); } catch (error) { this.debugLogger.error('Final flush failed during destroy', { error }); } this.removeAllListeners(); this.debugLogger.info('LogsInterceptor destroyed'); } } exports.LogsInterceptor = LogsInterceptor; //# sourceMappingURL=logger.js.map