log-vista/src/logCollector.js

LogVista Agent - Lightweight system monitoring and log collection for any project/language

const fs = require('fs').promises;
const path = require('path');
const chokidar = require('chokidar');
const logger = require('./logger');

class LogCollector {
  constructor() {
    this.watchers = new Map();
    this.logBuffer = [];
    this.lastReadPositions = new Map();
  }

  async startWatching(projectConfigs) {
    for (const project of projectConfigs) {
      if (!project.enabled) continue;

      // Watch custom log paths
      for (const logPath of project.custom_log_paths || []) {
        await this.watchLogFile(logPath, project.project_name);
      }

      // Watch common log locations in project directory
      await this.watchProjectLogs(project.pwd_path, project.project_name);
    }
  }

  async watchLogFile(logPath, projectName) {
    try {
      // Check if file exists
      await fs.access(logPath);
      
      // Initialize read position
      const stats = await fs.stat(logPath);
      this.lastReadPositions.set(logPath, stats.size);

      // Watch for changes
      const watcher = chokidar.watch(logPath, {
        persistent: true,
        usePolling: true,
        interval: 1000
      });

      watcher.on('change', async () => {
        await this.readNewLogContent(logPath, projectName);
      });

      this.watchers.set(logPath, watcher);
      logger.info(`Started watching log file: ${logPath} for project: ${projectName}`);

    } catch (error) {
      logger.warn(`Cannot watch log file ${logPath}:`, error.message);
    }
  }

  async watchProjectLogs(projectPath, projectName) {
    try {
      // Common log directories and patterns
      const logPatterns = [
        path.join(projectPath, 'logs', '*.log'),
        path.join(projectPath, 'log', '*.log'),
        path.join(projectPath, '*.log'),
        path.join(projectPath, 'var', 'log', '*.log')
      ];

      for (const pattern of logPatterns) {
        const watcher = chokidar.watch(pattern, {
          persistent: true,
          usePolling: true,
          interval: 1000,
          ignoreInitial: false
        });

        watcher.on('add', async (filePath) => {
          logger.info(`New log file detected: ${filePath}`);
          await this.initializeLogFile(filePath, projectName);
        });

        watcher.on('change', async (filePath) => {
          await this.readNewLogContent(filePath, projectName);
        });

        this.watchers.set(pattern, watcher);
      }

      logger.info(`Started watching project logs in: ${projectPath}`);

    } catch (error) {
      logger.error(`Error watching project logs in ${projectPath}:`, error);
    }
  }

  async initializeLogFile(filePath, projectName) {
    try {
      const stats = await fs.stat(filePath);
      this.lastReadPositions.set(filePath, stats.size);
      logger.info(`Initialized log file: ${filePath} for project: ${projectName}`);
    } catch (error) {
      logger.error(`Error initializing log file ${filePath}:`, error);
    }
  }

  async readNewLogContent(filePath, projectName) {
    try {
      const stats = await fs.stat(filePath);
      const lastPosition = this.lastReadPositions.get(filePath) || 0;
      
      if (stats.size <= lastPosition) {
        // File might have been truncated
        this.lastReadPositions.set(filePath, 0);
        return;
      }

      // Read new content
      const fileHandle = await fs.open(filePath, 'r');
      const buffer = Buffer.alloc(stats.size - lastPosition);
      
      const { bytesRead } = await fileHandle.read(
        buffer, 
        0, 
        buffer.length, 
        lastPosition
      );
      
      await fileHandle.close();

      if (bytesRead > 0) {
        const newContent = buffer.slice(0, bytesRead).toString('utf8');
        await this.processLogContent(newContent, filePath, projectName);
        this.lastReadPositions.set(filePath, stats.size);
      }

    } catch (error) {
      logger.error(`Error reading log file ${filePath}:`, error);
    }
  }

  async processLogContent(content, source, projectName) {
    const lines = content.split('\n').filter(line => line.trim());
    
    for (const line of lines) {
      const logEntry = this.parseLogLine(line, source, projectName);
      if (logEntry) {
        this.logBuffer.push(logEntry);
      }
    }
  }

  parseLogLine(line, source, projectName) {
    try {
      // Try to parse as JSON first
      const jsonLog = JSON.parse(line);
      return {
        timestamp: new Date(jsonLog.timestamp || Date.now()),
        level: this.normalizeLogLevel(jsonLog.level || 'INFO'),
        message: jsonLog.message || line,
        source: source,
        projectName: projectName,
        metadata: jsonLog
      };
    } catch (jsonError) {
      // Fall back to text parsing
      return this.parseTextLog(line, source, projectName);
    }
  }

  parseTextLog(line, source, projectName) {
    // Simple regex patterns for common log formats
    const patterns = [
      // ISO timestamp with level
      /^(\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d{3})?(?:Z|[+-]\d{2}:\d{2})?)\s+\[?(\w+)\]?\s*(.+)$/,
      // Syslog format
      /^(\w{3}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2})\s+\w+\s+\w+\[?\d*\]?:\s*(.+)$/,
      // Simple timestamp
      /^(\d{2}\/\d{2}\/\d{4}\s+\d{2}:\d{2}:\d{2})\s+\[?(\w+)\]?\s*(.+)$/
    ];

    for (const pattern of patterns) {
      const match = line.match(pattern);
      if (match) {
        return {
          timestamp: new Date(match[1]),
          level: this.normalizeLogLevel(match[2] || 'INFO'),
          message: match[3] || match[2] || line,
          source: source,
          projectName: projectName,
          metadata: { rawLine: line }
        };
      }
    }

    // If no pattern matches, treat as info log
    return {
      timestamp: new Date(),
      level: 'INFO',
      message: line,
      source: source,
      projectName: projectName,
      metadata: { rawLine: line }
    };
  }

  normalizeLogLevel(level) {
    const levelMap = {
      'TRACE': 'DEBUG',
      'DEBUG': 'DEBUG',
      'INFO': 'INFO',
      'INFORMATION': 'INFO',
      'WARN': 'WARN',
      'WARNING': 'WARN',
      'ERROR': 'ERROR',
      'ERR': 'ERROR',
      'FATAL': 'FATAL',
      'CRITICAL': 'FATAL'
    };

    return levelMap[level?.toUpperCase()] || 'INFO';
  }

  getBufferedLogs() {
    const logs = [...this.logBuffer];
    this.logBuffer = [];
    return logs;
  }

  stopWatching() {
    for (const [path, watcher] of this.watchers) {
      watcher.close();
      logger.info(`Stopped watching: ${path}`);
    }
    this.watchers.clear();
  }
}

module.exports = LogCollector;