@clipwhisperer/common/src/infrastructure/services/ServiceManager.ts

ClipWhisperer Common - Shared library providing core utilities, database schemas, authentication, bucket management, and common functionality across all ClipWhisperer microservices

import axios, { AxiosError } from 'axios';
import { spawn } from 'child_process';
import { EventEmitter } from 'events';
import { ServiceConfig } from '../../schemas/services';
import {
  DependencyError,
  HealthCheckError,
  HealthCheckResult,
  IEventBus,
  IHealthChecker,
  IProcessManager,
  IServiceOrchestrator,
  IServiceRegistry,
  ProcessError,
  ProcessInfo,
  ServiceError,
  ServiceEvent,
  ServiceInfo,
  ServiceStatus
} from '../../types/services';
import { Logger } from '../services/Logger';

/**
 * Enterprise Service Manager - Core orchestration component
 * 
 * This class provides enterprise-grade service management capabilities including:
 * - Dependency-aware startup/shutdown
 * - Health monitoring with circuit breaker patterns
 * - Event-driven architecture
 * - Auto-recovery and restart policies
 * - Process lifecycle management
 * - Service discovery and registry
 * 
 * @example
 * ```typescript
 * const serviceManager = new EnterpriseServiceManager();
 * await serviceManager.start();
 * ```
 */

/**
 * Event Bus Implementation
 * Handles inter-service communication and event propagation
 */
class EventBus implements IEventBus {
  private emitter = new EventEmitter();
  private eventHistory: ServiceEvent[] = [];
  private logger = Logger.getInstance().child({ component: 'EventBus' });

  emit(event: ServiceEvent): void {
    this.eventHistory.push({
      ...event,
      timestamp: new Date()
    });

    // Keep only last 1000 events to prevent memory leaks
    if (this.eventHistory.length > 1000) {
      this.eventHistory = this.eventHistory.slice(-1000);
    }

    this.logger.info('Event emitted', {
      type: event.type,
      serviceName: event.serviceName,
      data: event.data
    });

    this.emitter.emit(event.type, event);
    this.emitter.emit('*', event); // Wildcard listener
  }

  on(eventType: string, listener: (event: ServiceEvent) => void): void {
    this.emitter.on(eventType, listener);
  }

  off(eventType: string, listener: (event: ServiceEvent) => void): void {
    this.emitter.off(eventType, listener);
  }

  getEventHistory(): ServiceEvent[] {
    return [...this.eventHistory];
  }

  clearHistory(): void {
    this.eventHistory = [];
    this.logger.info('Event history cleared');
  }
}

/**
 * Service Registry Implementation
 * Manages service discovery and configuration
 */
class ServiceRegistry implements IServiceRegistry {
  private services = new Map<string, ServiceInfo>();
  private logger = Logger.getInstance().child({ component: 'ServiceRegistry' });

  register(service: ServiceInfo): void {
    this.services.set(service.name, {
      ...service,
      registeredAt: new Date()
    });

    this.logger.info('Service registered', {
      serviceName: service.name,
      port: service.port,
      dependencies: service.dependencies
    });
  }

  get(serviceName: string): ServiceInfo | undefined {
    return this.services.get(serviceName);
  }

  getAll(): ServiceInfo[] {
    return Array.from(this.services.values());
  }

  unregister(serviceName: string): boolean {
    const removed = this.services.delete(serviceName);
    if (removed) {
      this.logger.info('Service unregistered', { serviceName });
    }
    return removed;
  }

  /**
   * Get services in dependency order using topological sorting
   */
  getDependencyOrder(): string[] {
    const services = Array.from(this.services.values());
    const visited = new Set<string>();
    const visiting = new Set<string>();
    const result: string[] = [];

    const visit = (serviceName: string): void => {
      if (visited.has(serviceName)) return;
      if (visiting.has(serviceName)) {
        throw new DependencyError(`Circular dependency detected involving ${serviceName}`);
      }

      visiting.add(serviceName);

      const service = this.services.get(serviceName);
      if (service?.dependencies) {
        for (const dep of service.dependencies) {
          visit(dep);
        }
      }

      visiting.delete(serviceName);
      visited.add(serviceName);
      result.push(serviceName);
    };

    for (const service of services) {
      visit(service.name);
    }

    return result;
  }
}

/**
 * Health Checker Implementation
 * Monitors service health with circuit breaker patterns
 */
class HealthChecker implements IHealthChecker {
  private healthStatus = new Map<string, HealthCheckResult>();
  private checkIntervals = new Map<string, NodeJS.Timeout>();
  private circuitBreakers = new Map<string, { failures: number; lastFailure: Date; isOpen: boolean }>();
  private logger = Logger.getInstance().child({ component: 'HealthChecker' });

  private readonly CIRCUIT_BREAKER_THRESHOLD = 3;
  private readonly CIRCUIT_BREAKER_TIMEOUT = 30000; // 30 seconds

  async checkHealth(serviceName: string, url: string): Promise<HealthCheckResult> {
    const startTime = Date.now();
    const correlationId = Logger.generateCorrelationId();

    try {
      // Check circuit breaker
      const breaker = this.circuitBreakers.get(serviceName);
      if (breaker?.isOpen) {
        const timeSinceLastFailure = Date.now() - breaker.lastFailure.getTime();
        if (timeSinceLastFailure < this.CIRCUIT_BREAKER_TIMEOUT) {
          const result: HealthCheckResult = {
            serviceName,
            status: 'unhealthy',
            timestamp: new Date(),
            responseTime: 0,
            error: 'Circuit breaker is open'
          };
          this.healthStatus.set(serviceName, result);
          return result;
        } else {
          // Reset circuit breaker
          this.circuitBreakers.set(serviceName, { failures: 0, lastFailure: new Date(), isOpen: false });
        }
      }

      const response = await axios.get(url, {
        timeout: 5000,
        headers: { 'X-Correlation-ID': correlationId }
      });

      const responseTime = Date.now() - startTime;

      const result: HealthCheckResult = {
        serviceName,
        status: response.status === 200 ? 'healthy' : 'unhealthy',
        timestamp: new Date(),
        responseTime,
        details: response.data
      };

      // Reset circuit breaker on success
      this.circuitBreakers.set(serviceName, { failures: 0, lastFailure: new Date(), isOpen: false });

      this.healthStatus.set(serviceName, result);
      this.logger.info('Health check completed', {
        serviceName,
        status: result.status,
        responseTime,
        correlationId
      });

      return result;
    } catch (error) {
      const responseTime = Date.now() - startTime;

      // Update circuit breaker
      const currentBreaker = this.circuitBreakers.get(serviceName) || { failures: 0, lastFailure: new Date(), isOpen: false };
      currentBreaker.failures++;
      currentBreaker.lastFailure = new Date();

      if (currentBreaker.failures >= this.CIRCUIT_BREAKER_THRESHOLD) {
        currentBreaker.isOpen = true;
        this.logger.warn('Circuit breaker opened', { serviceName, failures: currentBreaker.failures });
      }

      this.circuitBreakers.set(serviceName, currentBreaker);

      const errorMessage = error instanceof AxiosError
        ? `HTTP ${error.response?.status}: ${error.message}`
        : error instanceof Error ? error.message : String(error);

      const result: HealthCheckResult = {
        serviceName,
        status: 'unhealthy',
        timestamp: new Date(),
        responseTime,
        error: errorMessage
      };

      this.healthStatus.set(serviceName, result);
      this.logger.error('Health check failed', {
        serviceName,
        error: errorMessage,
        responseTime,
        correlationId
      });

      return result;
    }
  }

  startMonitoring(serviceName: string, url: string, intervalMs: number = 5000): void {
    this.stopMonitoring(serviceName); // Stop any existing monitoring

    const interval = setInterval(async () => {
      await this.checkHealth(serviceName, url);
    }, intervalMs);

    this.checkIntervals.set(serviceName, interval);
    this.logger.info('Health monitoring started', { serviceName, intervalMs });
  }

  stopMonitoring(serviceName: string): void {
    const interval = this.checkIntervals.get(serviceName);
    if (interval) {
      clearInterval(interval);
      this.checkIntervals.delete(serviceName);
      this.logger.info('Health monitoring stopped', { serviceName });
    }
  }

  getStatus(serviceName: string): HealthCheckResult | undefined {
    return this.healthStatus.get(serviceName);
  }

  getAllStatus(): Map<string, HealthCheckResult> {
    return new Map(this.healthStatus);
  }

  resetCircuitBreaker(serviceName: string): void {
    this.circuitBreakers.set(serviceName, { failures: 0, lastFailure: new Date(), isOpen: false });
    this.logger.info('Circuit breaker reset', { serviceName });
  }
}

/**
 * Process Manager Implementation
 * Handles process lifecycle management
 */
class ProcessManager implements IProcessManager {
  private processes = new Map<string, ProcessInfo>();
  private logger = Logger.getInstance().child({ component: 'ProcessManager' });

  async start(serviceName: string, command: string, args: string[], cwd?: string): Promise<ProcessInfo> {
    try {
      // Kill existing process if running
      await this.stop(serviceName);

      const childProcess = spawn(command, args, {
        cwd: cwd || process.cwd(),
        stdio: ['pipe', 'pipe', 'pipe'],
        shell: true
      });

      const processInfo: ProcessInfo = {
        serviceName,
        pid: childProcess.pid!,
        command,
        args,
        startTime: new Date(),
        status: 'running',
        process: childProcess
      };

      // Setup process event handlers
      childProcess.on('error', (error) => {
        this.logger.error('Process error', { serviceName, error: error.message });
        processInfo.status = 'failed';
        processInfo.exitCode = -1;
        processInfo.exitTime = new Date();
      });

      childProcess.on('exit', (code, signal) => {
        this.logger.info('Process exited', { serviceName, code, signal });
        processInfo.status = code === 0 ? 'stopped' : 'failed';
        processInfo.exitCode = code ?? undefined;
        processInfo.exitSignal = signal ?? undefined;
        processInfo.exitTime = new Date();
      });

      // Log output
      childProcess.stdout?.on('data', (data) => {
        const output = data.toString().trim();
        if (output) {
          this.logger.info(`[${serviceName}] ${output}`);
        }
      });

      childProcess.stderr?.on('data', (data) => {
        const output = data.toString().trim();
        if (output) {
          this.logger.error(`[${serviceName}] ${output}`);
        }
      });

      this.processes.set(serviceName, processInfo);
      this.logger.info('Process started', { serviceName, pid: childProcess.pid, command });

      return processInfo;
    } catch (error) {
      const errorMessage = error instanceof Error ? error.message : String(error);
      this.logger.error('Failed to start process', { serviceName, error: errorMessage });
      throw new ProcessError(`Failed to start ${serviceName}: ${errorMessage}`, serviceName);
    }
  }

  async stop(serviceName: string, signal: NodeJS.Signals = 'SIGTERM'): Promise<boolean> {
    const processInfo = this.processes.get(serviceName);
    if (!processInfo || !processInfo.process) {
      return false;
    }

    try {
      return new Promise<boolean>((resolve) => {
        const childProcess = processInfo.process!;
        let resolved = false;

        const cleanup = () => {
          if (!resolved) {
            resolved = true;
            processInfo.status = 'stopped';
            processInfo.exitTime = new Date();
            this.logger.info('Process stopped', { serviceName, signal });
            resolve(true);
          }
        };

        childProcess.on('exit', cleanup);

        // Force kill after timeout
        const forceKillTimer = setTimeout(() => {
          if (!resolved && !childProcess.killed) {
            this.logger.warn('Force killing process', { serviceName });
            childProcess.kill('SIGKILL');
            cleanup();
          }
        }, 10000);

        childProcess.kill(signal);

        // Clean up timer if process exits normally
        childProcess.on('exit', () => clearTimeout(forceKillTimer));
      });
    } catch (error) {
      const errorMessage = error instanceof Error ? error.message : String(error);
      this.logger.error('Failed to stop process', { serviceName, error: errorMessage });
      return false;
    }
  }

  getProcess(serviceName: string): ProcessInfo | undefined {
    return this.processes.get(serviceName);
  }

  getAllProcesses(): ProcessInfo[] {
    return Array.from(this.processes.values());
  }

  isRunning(serviceName: string): boolean {
    const processInfo = this.processes.get(serviceName);
    return !!(processInfo?.status === 'running' && processInfo.process && !(processInfo.process.killed ?? false));
  }

  async restart(serviceName: string): Promise<ProcessInfo> {
    const processInfo = this.processes.get(serviceName);
    if (!processInfo) {
      throw new ProcessError(`Service ${serviceName} not found`, serviceName);
    }

    await this.stop(serviceName);
    return this.start(serviceName, processInfo.command, processInfo.args);
  }
}

/**
 * Enterprise Service Manager
 * Main orchestrator implementing dependency injection and enterprise patterns
 */
export class EnterpriseServiceManager implements IServiceOrchestrator {
  private eventBus: IEventBus;
  private serviceRegistry: IServiceRegistry;
  private healthChecker: IHealthChecker;
  private processManager: IProcessManager;
  private logger = Logger.getInstance().child({ component: 'EnterpriseServiceManager' });
  private isRunning = false;
  private config: ServiceConfig;

  constructor(config?: Partial<any>) {
    this.config = new ServiceConfig(config);
    this.eventBus = new EventBus();
    this.serviceRegistry = new ServiceRegistry();
    this.healthChecker = new HealthChecker();
    this.processManager = new ProcessManager();

    // Setup event listeners
    this.setupEventListeners();

    this.logger.info('EnterpriseServiceManager initialized', {
      environment: this.config.getEnvironment(),
      servicesCount: this.config.getServices().length
    });
  }

  private setupEventListeners(): void {
    this.eventBus.on('service:unhealthy', (event) => {
      this.handleUnhealthyService(event);
    });

    this.eventBus.on('service:failed', (event) => {
      this.handleFailedService(event);
    });

    this.eventBus.on('process:exit', (event) => {
      this.handleProcessExit(event);
    });
  }

  private async handleUnhealthyService(event: ServiceEvent): Promise<void> {
    const serviceName = event.serviceName;
    const serviceConfig = this.config.getService(serviceName);

    if (serviceConfig?.restartPolicy?.enabled) {
      this.logger.warn('Attempting service recovery', { serviceName });

      try {
        await this.processManager.restart(serviceName);
        this.eventBus.emit({
          type: 'service:recovered',
          serviceName,
          data: { reason: 'unhealthy_restart' }
        });
      } catch (error) {
        this.logger.error('Service recovery failed', {
          serviceName,
          error: error instanceof Error ? error.message : String(error)
        });
      }
    }
  }

  private async handleFailedService(event: ServiceEvent): Promise<void> {
    this.logger.error('Service failed', { serviceName: event.serviceName, data: event.data });
  }

  private async handleProcessExit(event: ServiceEvent): Promise<void> {
    this.logger.info('Process exited', { serviceName: event.serviceName, data: event.data });
  }

  async start(): Promise<void> {
    if (this.isRunning) {
      throw new ServiceError('Service manager is already running');
    }

    this.logger.info('Starting Enterprise Service Manager');

    try {
      // Register all services
      const services = this.config.getServices();
      for (const service of services) {
        this.serviceRegistry.register({
          name: service.name,
          port: service.port,
          healthEndpoint: service.healthEndpoint,
          dependencies: service.dependencies,
          command: service.startCommand
        });
      }

      // Start services in dependency order
      const startOrder = this.serviceRegistry.getDependencyOrder();
      this.logger.info('Starting services in order', { order: startOrder });

      for (const serviceName of startOrder) {
        await this.startService(serviceName);
        // Wait a bit between service starts
        await new Promise(resolve => setTimeout(resolve, 2000));
      }

      // Start health monitoring
      this.startHealthMonitoring();

      this.isRunning = true;
      this.eventBus.emit({
        type: 'manager:started',
        serviceName: 'enterprise-manager',
        data: { servicesCount: services.length }
      });

      this.logger.info('Enterprise Service Manager started successfully');
    } catch (error) {
      this.logger.error('Failed to start Enterprise Service Manager', {
        error: error instanceof Error ? error.message : String(error)
      });
      throw error;
    }
  }

  async stop(): Promise<void> {
    if (!this.isRunning) {
      return;
    }

    this.logger.info('Stopping Enterprise Service Manager');

    try {
      // Stop health monitoring
      this.stopHealthMonitoring();

      // Stop services in reverse dependency order
      const stopOrder = this.serviceRegistry.getDependencyOrder().reverse();
      this.logger.info('Stopping services in order', { order: stopOrder });

      for (const serviceName of stopOrder) {
        await this.stopService(serviceName);
      }

      this.isRunning = false;
      this.eventBus.emit({
        type: 'manager:stopped',
        serviceName: 'enterprise-manager',
        data: {}
      });

      this.logger.info('Enterprise Service Manager stopped successfully');
    } catch (error) {
      this.logger.error('Error during shutdown', {
        error: error instanceof Error ? error.message : String(error)
      });
      throw error;
    }
  }

  async startService(serviceName: string): Promise<void> {
    const serviceInfo = this.serviceRegistry.get(serviceName);
    if (!serviceInfo) {
      throw new ServiceError(`Service ${serviceName} not found in registry`, serviceName);
    }

    const serviceConfig = this.config.getService(serviceName);
    if (!serviceConfig) {
      throw new ServiceError(`Service configuration for ${serviceName} not found`, serviceName);
    }

    try {
      this.logger.info('Starting service', { serviceName });

      // Check dependencies are healthy
      if (serviceInfo.dependencies) {
        for (const dep of serviceInfo.dependencies) {
          const depHealth = this.healthChecker.getStatus(dep);
          if (!depHealth || depHealth.status !== 'healthy') {
            throw new DependencyError(`Dependency ${dep} is not healthy`, serviceName, dep);
          }
        }
      }

      // Start the process
      const [command, ...args] = serviceConfig.startCommand.split(' ');
      await this.processManager.start(serviceName, command, args, serviceConfig.workingDirectory);

      // Wait for service to be ready
      await this.waitForServiceReady(serviceName, serviceInfo.healthEndpoint);

      this.eventBus.emit({
        type: 'service:started',
        serviceName,
        data: { port: serviceInfo.port }
      });

      this.logger.info('Service started successfully', { serviceName });
    } catch (error) {
      this.logger.error('Failed to start service', {
        serviceName,
        error: error instanceof Error ? error.message : String(error)
      });
      throw error;
    }
  }

  async stopService(serviceName: string): Promise<void> {
    try {
      this.logger.info('Stopping service', { serviceName });

      // Stop health monitoring for this service
      this.healthChecker.stopMonitoring(serviceName);

      // Stop the process
      await this.processManager.stop(serviceName);

      this.eventBus.emit({
        type: 'service:stopped',
        serviceName,
        data: {}
      });

      this.logger.info('Service stopped successfully', { serviceName });
    } catch (error) {
      this.logger.error('Failed to stop service', {
        serviceName,
        error: error instanceof Error ? error.message : String(error)
      });
      throw error;
    }
  }

  private async waitForServiceReady(serviceName: string, healthEndpoint: string, maxAttempts: number = 30): Promise<void> {
    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
      try {
        const result = await this.healthChecker.checkHealth(serviceName, healthEndpoint);
        if (result.status === 'healthy') {
          return;
        }
      } catch (error) {
        // Expected during startup
      }

      if (attempt === maxAttempts) {
        throw new HealthCheckError(`Service ${serviceName} failed to become healthy after ${maxAttempts} attempts`, serviceName, healthEndpoint);
      }

      await new Promise(resolve => setTimeout(resolve, 2000));
    }
  }

  private startHealthMonitoring(): void {
    const services = this.serviceRegistry.getAll();
    for (const service of services) {
      this.healthChecker.startMonitoring(service.name, service.healthEndpoint);
    }
  }

  private stopHealthMonitoring(): void {
    const services = this.serviceRegistry.getAll();
    for (const service of services) {
      this.healthChecker.stopMonitoring(service.name);
    }
  }

  getStatus(): { [serviceName: string]: ServiceStatus } {
    const status: { [serviceName: string]: ServiceStatus } = {};
    const services = this.serviceRegistry.getAll();

    for (const service of services) {
      const processInfo = this.processManager.getProcess(service.name);
      const healthInfo = this.healthChecker.getStatus(service.name);

      if (!processInfo) {
        status[service.name] = ServiceStatus.STOPPED;
      } else if (processInfo.status === 'failed') {
        status[service.name] = ServiceStatus.FAILED;
      } else if (healthInfo?.status === 'healthy') {
        status[service.name] = ServiceStatus.RUNNING;
      } else {
        status[service.name] = ServiceStatus.STARTING;
      }
    }

    return status;
  }

  getServiceInfo(): ServiceInfo[] {
    return this.serviceRegistry.getAll();
  }

  getEventHistory(): ServiceEvent[] {
    return this.eventBus.getEventHistory();
  }

  // Dependency injection getters
  getEventBus(): IEventBus {
    return this.eventBus;
  }

  getServiceRegistry(): IServiceRegistry {
    return this.serviceRegistry;
  }

  getHealthChecker(): IHealthChecker {
    return this.healthChecker;
  }

  getProcessManager(): IProcessManager {
    return this.processManager;
  }

  getConfig(): ServiceConfig {
    return this.config;
  }
}

export default EnterpriseServiceManager;