claude-coordination-system/src/coordinator-core.js

🤖 Multi-Claude Parallel Processing Coordination System - Organize multiple Claude AI instances to work together seamlessly on complex development tasks

/**
 * Multi-Claude Coordinator Core Engine
 * Central coordination system for managing multiple Claude workers
 */

const fs = require('fs-extra');
const path = require('path');
const EventEmitter = require('events');
const chalk = require('chalk');
const { logCoordinator, logError, logPerformance } = require('./development-logger');
const WebDashboard = require('./web-dashboard');
const FileProtectionManager = require('./file-protection-manager');
const CoordinationEngine = require('./coordination-engine');
const { SecurityManager, SecurityError } = require('./security-manager');
const AuthManager = require('./auth-manager');

class CoordinatorCore extends EventEmitter {
  constructor(projectRoot, options = {}) {
    super();
    
    this.projectRoot = projectRoot;
    this.coordinationDir = path.join(projectRoot, '.claude-coord');
    this.stateFile = path.join(this.coordinationDir, 'system-state.json');
    this.messagesFile = path.join(this.coordinationDir, 'messages.json');
    
    this.options = {
      port: 7777,
      mode: 'prod',
      maxWorkers: 6,
      heartbeatInterval: 15000,
      staleWorkerTimeout: 60000,
      ...options
    };

    this.isRunning = false;
    this.workers = new Map();
    this.fileLocks = new Map();
    this.messageQueue = [];
    
    // Fast cache for terminal interface performance
    this.stateCache = null;
    this.stateCacheTime = 0;
    this.cacheMaxAge = 100; // 100ms cache for terminal interface responsiveness
    this.webDashboard = new WebDashboard(this, this.options.port);
    this.fileProtection = new FileProtectionManager(projectRoot, {
      enabled: options.fileProtection !== false
    });
    // EMERGENCY FIX: Disable coordination engine to prevent memory leak
    this.coordinationEngine = null;
    
    // Initialize security and authentication
    this.securityManager = new SecurityManager(projectRoot, {
      logSecurityEvents: options.verbose || false
    });
    
    this.authManager = new AuthManager(projectRoot, {
      requireAuth: options.requireAuth || false,
      jwtSecret: options.jwtSecret,
      tokenExpiry: options.tokenExpiry || '24h'
    });
    
    console.log(`🤖 Coordinator initialized for: ${path.basename(projectRoot)}`);
  }

  /**
   * Initialize coordination system state
   */
  async initializeSystem() {
    // Fast initialization - parallel operations
    const [_, existingState] = await Promise.all([
      fs.ensureDir(this.coordinationDir),
      this.loadExistingState()
    ]);
    
    if (existingState) {
      console.log(`⚡ System state loaded from existing file`);
      return existingState;
    }
    
    const initialState = {
      system_info: {
        initialized_at: new Date().toISOString(),
        version: '2.1.0',
        project_root: this.projectRoot,
        max_workers: this.options.maxWorkers,
        coordination_mode: 'file_based',
        coordinator_pid: process.pid
      },
      active_workers: {},
      file_locks: {},
      dependencies: {}, // Load lazily when needed
      task_progress: {
        total_groups: 0,
        completed_groups: 0,
        active_groups: 0
      },
      messages: []
    };

    await this.saveSystemState(initialState);
    console.log(`⚡ System state initialized quickly`);
    
    // Load dependencies asynchronously in background
    this.loadDependenciesAsync();
    
    return initialState;
  }
  
  async loadExistingState() {
    try {
      if (await fs.pathExists(this.stateFile)) {
        const data = await fs.readFile(this.stateFile, 'utf8');
        return JSON.parse(data);
      }
    } catch (error) {
      // Invalid state file, will create new one
    }
    return null;
  }
  
  async loadDependenciesAsync() {
    // Load dependencies in background without blocking
    setTimeout(async () => {
      try {
        const dependencies = await this.loadDependencies();
        const state = await this.loadSystemState();
        state.dependencies = dependencies;
        await this.saveSystemState(state);
      } catch (error) {
        // Silently fail - not critical for startup
      }
    }, 100);
  }

  /**
   * Load project dependencies configuration
   */
  async loadDependencies() {
    const configFile = path.join(this.projectRoot, 'claude-coord.json');
    
    try {
      if (await fs.pathExists(configFile)) {
        const config = await fs.readJson(configFile);
        return config.groups || {};
      }
    } catch (error) {
      console.warn(`⚠️  Could not load project config: ${error.message}`);
    }

    // Return default dependencies based on project type
    return this.getDefaultDependencies();
  }

  /**
   * Get default dependency structure for common project types
   */
  getDefaultDependencies() {
    // Basic Next.js project structure
    return {
      GRUP1_TYPESCRIPT: {
        name: 'TypeScript & Build System',
        priority: 1,
        blocks: ['GRUP2_ESLINT', 'GRUP6_UI'],
        blocked_by: [],
        files: [
          'tsconfig.json',
          'src/**/*.ts',
          'src/**/*.tsx'
        ]
      },
      GRUP2_ESLINT: {
        name: 'ESLint & Code Quality',
        priority: 2,
        blocks: ['GRUP6_UI'],
        blocked_by: ['GRUP1_TYPESCRIPT'],
        files: [
          'eslint.config.*',
          'src/**/*'
        ]
      },
      GRUP3_BUNDLE: {
        name: 'Bundle & Dependencies',
        priority: 2,
        blocks: [],
        blocked_by: [],
        files: [
          'package.json',
          'package-lock.json',
          'next.config.*'
        ]
      }
    };
  }

  /**
   * Start coordination server
   */
  async start() {
    if (this.isRunning) {
      throw new Error('Coordinator already running');
    }

    console.log(chalk.blue('🚀 Starting Multi-Claude Coordination System...'));
    
    await logCoordinator('Starting System', {
      description: 'Initializing Multi-Claude coordination system',
      projectRoot: this.projectRoot,
      maxWorkers: this.options.maxWorkers,
      port: this.options.port
    });
    
    // Initialize system state
    await this.initializeSystem();
    
    // Initialize file protection system
    if (this.fileProtection) {
      await this.fileProtection.initialize();
    }
    
    // Start coordination engine for real-time collaboration (DISABLED for memory leak fix)
    if (this.coordinationEngine) {
      await this.coordinationEngine.start();
    }
    
    // Start background processes
    this.isRunning = true;
    this.startHeartbeatMonitor();
    this.startMessageProcessor();
    this.startFileWatcher();
    
    // Start web dashboard
    await this.webDashboard.start();
    
    console.log(chalk.green('✅ Coordinator started successfully'));
    console.log(chalk.blue(`📊 Dashboard: http://localhost:${this.options.port}`));
    
    await logCoordinator('System Started', {
      result: 'SUCCESS',
      dashboardUrl: `http://localhost:${this.options.port}`,
      notes: 'Coordination system fully operational'
    });
    
    // Clean up any corrupted sync sessions from previous runs
    await this.cleanupCorruptedSessions();
    
    this.emit('started');
    return true;
  }

  /**
   * Get web dashboard instance
   */
  getWebDashboard() {
    return this.webDashboard;
  }

  /**
   * Clean up corrupted sync sessions
   */
  async cleanupCorruptedSessions() {
    const state = await this.loadSystemState();
    
    if (state.coordination?.sessions) {
      let cleaned = false;
      
      // Remove sessions with undefined or invalid values
      Object.keys(state.coordination.sessions).forEach(sessionId => {
        const session = state.coordination.sessions[sessionId];
        if (!session || !session.id || session.group === undefined || session.status === undefined) {
          console.log(chalk.yellow(`🧹 Removing corrupted sync session: ${sessionId}`));
          delete state.coordination.sessions[sessionId];
          cleaned = true;
        }
      });
      
      if (cleaned) {
        await this.saveSystemState(state);
        console.log(chalk.green('✅ Corrupted sync sessions cleaned up'));
      }
    }
  }

  /**
   * Stop coordination system
   */
  async stop() {
    if (!this.isRunning) return;

    console.log(chalk.yellow('🛑 Stopping coordination system...'));
    
    this.isRunning = false;
    
    // Clear all timers
    if (this.heartbeatTimer) clearInterval(this.heartbeatTimer);
    if (this.messageTimer) clearInterval(this.messageTimer);
    if (this.fileWatcher) this.fileWatcher.close();
    
    // Stop web dashboard
    if (this.webDashboard) await this.webDashboard.stop();
    
    // Stop coordination engine (DISABLED for memory leak fix)
    if (this.coordinationEngine) {
      await this.coordinationEngine.stop();
    }
    
    // Stop file protection system
    if (this.fileProtection) {
      await this.fileProtection.stop();
    }
    
    // Stop all workers
    await this.stopAllWorkers();
    
    console.log(chalk.green('✅ Coordinator stopped'));
    this.emit('stopped');
  }

  /**
   * Register a new worker with authentication
   */
  async registerWorker(workerId, groupId, metadata = {}, authToken = null) {
    // Validate authentication if required
    if (this.authManager.options.requireAuth) {
      try {
        this.authManager.requireAuth('worker')(authToken);
      } catch (error) {
        throw new SecurityError(`Worker registration denied: ${error.message}`);
      }
    }
    
    // Validate input parameters for security
    const safeWorkerId = this.securityManager.sanitizeCommandArgument(workerId);
    const safeGroupId = this.securityManager.sanitizeCommandArgument(groupId);
    
    const state = await this.loadSystemState();
    
    if (state.active_workers[workerId]) {
      throw new Error(`Worker ${workerId} already registered`);
    }

    const workerInfo = {
      id: workerId,
      group: groupId,
      status: 'initializing',
      started_at: new Date().toISOString(),
      last_heartbeat: new Date().toISOString(),
      current_files: [],
      metadata,
      progress: {
        total_tasks: 0,
        completed_tasks: 0,
        current_task: null
      }
    };

    state.active_workers[workerId] = workerInfo;
    state.task_progress.active_groups++;
    
    await this.saveSystemState(state);
    
    console.log(chalk.green(`👥 Worker registered: ${workerId} (${groupId})`));
    this.emit('worker:registered', workerInfo);
    
    return workerInfo;
  }

  /**
   * Update worker status
   */
  async updateWorkerStatus(workerId, updates) {
    const state = await this.loadSystemState();
    
    if (!state.active_workers[workerId]) {
      throw new Error(`Worker ${workerId} not found`);
    }

    // Update worker info
    Object.assign(state.active_workers[workerId], {
      ...updates,
      last_heartbeat: new Date().toISOString()
    });

    await this.saveSystemState(state);
    this.emit('worker:updated', state.active_workers[workerId]);
  }

  /**
   * File locking system
   */
  async acquireFileLock(workerId, filePath) {
    const state = await this.loadSystemState();
    
    // Check if file is already locked
    if (state.file_locks[filePath] && state.file_locks[filePath] !== workerId) {
      return { success: false, lockedBy: state.file_locks[filePath] };
    }

    // Acquire lock
    state.file_locks[filePath] = workerId;
    await this.saveSystemState(state);
    
    console.log(`🔒 File locked: ${filePath} → ${workerId}`);
    this.emit('file:locked', { filePath, workerId });
    
    return { success: true };
  }

  /**
   * Release file lock
   */
  async releaseFileLock(workerId, filePath) {
    const state = await this.loadSystemState();
    
    if (state.file_locks[filePath] === workerId) {
      delete state.file_locks[filePath];
      await this.saveSystemState(state);
      
      console.log(`🔓 File unlocked: ${filePath} ← ${workerId}`);
      this.emit('file:unlocked', { filePath, workerId });
    }
  }

  /**
   * Check dependencies for a worker group
   */
  async checkDependencies(groupId) {
    const state = await this.loadSystemState();
    const groupConfig = state.dependencies[groupId];
    
    if (!groupConfig) return { canProceed: true, blockers: [] };
    
    const blockers = [];
    
    // Check blocked_by dependencies
    for (const blockingGroup of groupConfig.blocked_by || []) {
      const blockingWorker = Object.values(state.active_workers).find(
        w => w.group === blockingGroup
      );
      
      if (!blockingWorker || blockingWorker.status !== 'completed') {
        blockers.push(blockingGroup);
      }
    }
    
    return {
      canProceed: blockers.length === 0,
      blockers
    };
  }

  /**
   * Get system status (Enhanced with coordination engine)
   */
  async getSystemStatus() {
    try {
      const state = await this.loadSystemState();
      // EMERGENCY FIX: Skip coordination status to prevent memory issues
      const coordinationStatus = null;
      
      return {
        healthy: true,
        activeWorkers: Object.keys(state.active_workers).length,
        completedTasks: state.task_progress.completed_groups,
        totalTasks: state.task_progress.total_groups,
        fileLocks: Object.keys(state.file_locks).length,
        workers: Object.values(state.active_workers),
        uptime: Date.now() - new Date(state.system_info.initialized_at).getTime(),
        
        // Enhanced coordination info
        // EMERGENCY FIX: Simplified coordination (engine disabled)
        coordination: {
          engine: 'disabled',
          sessions: {},
          groups: {},
          realTimeFileLocks: {},
          messageQueue: []
        },
        
        // File protection status
        fileProtection: this.fileProtection ? this.fileProtection.getStatus() : null
      };
    } catch (error) {
      return {
        healthy: false,
        error: error.message
      };
    }
  }

  /**
   * System state management
   */
  async loadSystemState(useCache = true) {
    // Use cache for terminal interface performance
    if (useCache && this.stateCache && (Date.now() - this.stateCacheTime) < this.cacheMaxAge) {
      return this.stateCache;
    }
    
    try {
      const data = await fs.readFile(this.stateFile, 'utf8');
      const state = JSON.parse(data);
      
      // Update cache
      this.stateCache = state;
      this.stateCacheTime = Date.now();
      
      return state;
    } catch (error) {
      console.warn('⚠️  Could not load system state, initializing...');
      const state = await this.initializeSystem();
      
      // Cache the initialized state too
      this.stateCache = state;
      this.stateCacheTime = Date.now();
      
      return state;
    }
  }

  async saveSystemState(state) {
    try {
      await fs.writeFile(this.stateFile, JSON.stringify(state, null, 2));
      
      // Update cache with new state
      this.stateCache = state;
      this.stateCacheTime = Date.now();
      
    } catch (error) {
      console.error('❌ Failed to save system state:', error.message);
      throw error;
    }
  }

  /**
   * Background monitoring processes
   */
  startHeartbeatMonitor() {
    this.heartbeatTimer = setInterval(async () => {
      try {
        await this.checkWorkerHealth();
      } catch (error) {
        console.error('❌ Heartbeat check failed:', error.message);
      }
    }, this.options.heartbeatInterval);
    
    console.log('💓 Heartbeat monitor started');
  }

  async checkWorkerHealth() {
    const state = await this.loadSystemState();
    const now = Date.now();
    const staleWorkers = [];
    
    for (const [workerId, worker] of Object.entries(state.active_workers)) {
      const lastHeartbeat = new Date(worker.last_heartbeat).getTime();
      
      if (now - lastHeartbeat > this.options.staleWorkerTimeout) {
        staleWorkers.push(workerId);
      }
    }
    
    // Mark stale workers as failed
    for (const workerId of staleWorkers) {
      await this.updateWorkerStatus(workerId, { status: 'stale' });
      // Only log stale workers internally, don't show to user unless verbose mode
      if (this.options.verbose) {
        console.log(chalk.gray(`🔇 Worker ${workerId} marked as stale (internal)`));
      }
      
      await logCoordinator('Worker Stale', {
        description: `Worker marked as stale due to missed heartbeat`,
        notes: `Worker: ${workerId}, Last heartbeat: ${new Date(worker.last_heartbeat).toISOString()}`
      });
    }
  }

  startMessageProcessor() {
    this.messageTimer = setInterval(async () => {
      try {
        await this.processMessageQueue();
      } catch (error) {
        console.error('❌ Message processing failed:', error.message);
      }
    }, 1000);
  }

  async processMessageQueue() {
    if (this.messageQueue.length === 0) return;
    
    const messages = [...this.messageQueue];
    this.messageQueue = [];
    
    // Process messages and emit events
    for (const message of messages) {
      this.emit('message', message);
    }
  }

  startFileWatcher() {
    // File watching implementation would go here
    // For now, we'll use polling
    console.log('👁️  File watcher initialized');
  }

  async stopAllWorkers() {
    const state = await this.loadSystemState();
    
    // Send stop signals to all active workers
    for (const workerId of Object.keys(state.active_workers)) {
      await this.updateWorkerStatus(workerId, { status: 'stopping' });
    }
    
    // Clear all file locks
    state.file_locks = {};
    state.active_workers = {};
    await this.saveSystemState(state);
  }

  /**
   * Remove a specific worker from the system
   */
  async removeWorker(workerId) {
    const state = await this.loadSystemState();
    
    if (!state.active_workers[workerId]) {
      return { success: false, error: 'Worker not found' };
    }

    const worker = state.active_workers[workerId];
    const releasedFiles = [];

    // Release all file locks held by this worker
    for (const [filePath, lockedBy] of Object.entries(state.file_locks)) {
      if (lockedBy === workerId) {
        delete state.file_locks[filePath];
        releasedFiles.push(filePath);
      }
    }

    // Remove worker from active workers
    delete state.active_workers[workerId];
    
    // Update task progress
    if (state.task_progress.active_groups > 0) {
      state.task_progress.active_groups--;
    }

    await this.saveSystemState(state);
    
    console.log(chalk.yellow(`👋 Worker removed: ${workerId} (${worker.group})`));
    this.emit('worker:removed', { workerId, group: worker.group, releasedFiles });

    await logCoordinator('Worker Removed', {
      description: `Successfully removed worker from coordination system`,
      result: 'SUCCESS',
      files: releasedFiles,
      notes: `Worker: ${workerId} (${worker.group}), Released ${releasedFiles.length} file locks`
    });

    return { success: true, releasedFiles };
  }

  /**
   * Reassign a worker to a different group
   */
  async reassignWorker(workerId, newGroupId) {
    const state = await this.loadSystemState();
    
    if (!state.active_workers[workerId]) {
      return { success: false, error: 'Worker not found' };
    }

    // Check if new group exists
    if (!state.dependencies[newGroupId]) {
      return { success: false, error: `Group '${newGroupId}' not found` };
    }

    const worker = state.active_workers[workerId];
    const oldGroup = worker.group;
    
    // Release current file locks
    const releasedFiles = [];
    for (const [filePath, lockedBy] of Object.entries(state.file_locks)) {
      if (lockedBy === workerId) {
        delete state.file_locks[filePath];
        releasedFiles.push(filePath);
      }
    }

    // Update worker group
    state.active_workers[workerId] = {
      ...worker,
      group: newGroupId,
      status: 'reassigned',
      reassigned_at: new Date().toISOString(),
      previous_group: oldGroup,
      current_files: [],
      progress: {
        total_tasks: 0,
        completed_tasks: 0,
        current_task: null
      }
    };

    await this.saveSystemState(state);
    
    console.log(chalk.blue(`🔄 Worker reassigned: ${workerId} (${oldGroup} → ${newGroupId})`));
    this.emit('worker:reassigned', { workerId, oldGroup, newGroupId, releasedFiles });

    return { success: true };
  }

  /**
   * Stop all coordination (alias for stopAllWorkers for CLI compatibility)
   */
  async stopAll() {
    return await this.stopAllWorkers();
  }
}

module.exports = CoordinatorCore;