auto-publishing-mcp-server/src/tools/deploy/canary-deploy.js

Enterprise-grade MCP Server for Auto-Publishing with pre-publish validation, multi-cloud deployment, and monitoring

/**
 * Canary Deployment Strategy
 * Gradual rollout with traffic splitting and automated rollback
 */

import Docker from 'dockerode';
import fs from 'fs/promises';
import path from 'path';
import { execSync } from 'child_process';

export class CanaryDeployment {
  constructor(config = {}) {
    this.docker = new Docker();
    this.deploymentPath = config.deploymentPath || '/root/projects/auto-publishing/deployments';
    this.configPath = config.configPath || '/root/projects/auto-publishing/config';
    
    // Canary defaults
    this.defaultCanaryConfig = {
      initialTrafficPercentage: 10,
      incrementPercentage: 20,
      incrementInterval: 300000, // 5 minutes
      successThreshold: 0.95, // 95% success rate
      errorThreshold: 0.05, // 5% error rate
      responseTimeThreshold: 1000, // 1 second
      minObservationTime: 60000 // 1 minute
    };
  }

  /**
   * Deploy using canary strategy
   */
  async deployCanary(args) {
    const {
      dockerId,
      newImage,
      environment = 'prod',
      canaryConfig = {},
      healthEndpoint = '/health',
      metricsEndpoint = '/metrics'
    } = args;

    const config = { ...this.defaultCanaryConfig, ...canaryConfig };
    const deploymentId = `canary-${dockerId}-${Date.now()}`;
    
    try {
      // Validate prerequisites
      await this.validateCanaryPrerequisites(dockerId, environment);
      
      // Create canary container
      console.log(`🐤 Starting canary deployment for ${dockerId}`);
      const canaryContainer = await this.createCanaryContainer({
        dockerId,
        newImage,
        environment,
        trafficPercentage: config.initialTrafficPercentage
      });
      
      // Monitor and gradually increase traffic
      const result = await this.performCanaryRollout({
        dockerId,
        canaryContainer,
        config,
        healthEndpoint,
        metricsEndpoint,
        deploymentId
      });
      
      if (result.success) {
        // Complete deployment
        await this.completeCanaryDeployment(dockerId, canaryContainer.id, environment);
        
        return {
          output: `Canary deployment completed successfully for ${dockerId}`,
          data: {
            deploymentId,
            status: 'completed',
            duration: result.duration,
            finalTrafficPercentage: 100,
            metrics: result.finalMetrics
          }
        };
      } else {
        // Rollback
        await this.rollbackCanary(canaryContainer.id, dockerId);
        
        return {
          output: `Canary deployment rolled back for ${dockerId}`,
          data: {
            deploymentId,
            status: 'rolled_back',
            reason: result.reason,
            lastTrafficPercentage: result.lastTrafficPercentage,
            metrics: result.metrics
          }
        };
      }
    } catch (error) {
      throw new Error(`Canary deployment failed: ${error.message}`);
    }
  }

  /**
   * Validate prerequisites for canary deployment
   */
  async validateCanaryPrerequisites(dockerId, environment) {
    // Check if main container exists
    const containers = await this.docker.listContainers({
      all: true,
      filters: { name: [dockerId] }
    });
    
    if (containers.length === 0) {
      throw new Error(`No existing container found with ID ${dockerId}`);
    }
    
    // Check if load balancer is configured
    const nginxConfigPath = `/etc/nginx/sites-available/${dockerId}`;
    try {
      await fs.access(nginxConfigPath);
    } catch {
      throw new Error('Nginx load balancer not configured for canary deployment');
    }
    
    return true;
  }

  /**
   * Create canary container
   */
  async createCanaryContainer(params) {
    const { dockerId, newImage, environment, trafficPercentage } = params;
    
    // Get existing container config
    const existingContainer = await this.docker.getContainer(dockerId).inspect();
    const canaryName = `${dockerId}-canary`;
    
    // Create canary container with same config but new image
    const container = await this.docker.createContainer({
      name: canaryName,
      Image: newImage,
      Env: [
        ...existingContainer.Config.Env,
        `CANARY=true`,
        `DEPLOYMENT_TYPE=canary`,
        `TRAFFIC_PERCENTAGE=${trafficPercentage}`
      ],
      HostConfig: {
        ...existingContainer.HostConfig,
        PortBindings: {} // No direct port binding for canary
      },
      Labels: {
        ...existingContainer.Config.Labels,
        'canary': 'true',
        'canary.parent': dockerId,
        'canary.traffic': String(trafficPercentage)
      }
    });
    
    await container.start();
    
    // Update nginx configuration for traffic splitting
    await this.updateLoadBalancerConfig(dockerId, canaryName, trafficPercentage);
    
    return container;
  }

  /**
   * Update load balancer configuration for traffic splitting
   */
  async updateLoadBalancerConfig(mainContainerId, canaryContainerId, percentage) {
    const mainWeight = 100 - percentage;
    const canaryWeight = percentage;
    
    const nginxConfig = `
upstream ${mainContainerId}_backend {
    server ${mainContainerId}:3000 weight=${mainWeight};
    server ${canaryContainerId}:3000 weight=${canaryWeight};
}

server {
    listen 80;
    server_name ${mainContainerId}.local;
    
    location / {
        proxy_pass http://${mainContainerId}_backend;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
        
        # Add canary header for tracking
        add_header X-Canary-Status $upstream_addr;
    }
    
    location /nginx_status {
        stub_status on;
        access_log off;
    }
}`;
    
    const configPath = `/etc/nginx/sites-available/${mainContainerId}`;
    await fs.writeFile(configPath, nginxConfig);
    
    // Reload nginx (보안 개선)
    const { spawn } = require('child_process');
    await new Promise((resolve, reject) => {
      const nginx = spawn('nginx', ['-s', 'reload']);
      nginx.on('close', (code) => {
        if (code === 0) {
          resolve();
        } else {
          reject(new Error(`nginx reload failed with code ${code}`));
        }
      });
      nginx.on('error', reject);
    });
  }

  /**
   * Perform canary rollout with monitoring
   */
  async performCanaryRollout(params) {
    const {
      dockerId,
      canaryContainer,
      config,
      healthEndpoint,
      metricsEndpoint,
      deploymentId
    } = params;
    
    const startTime = Date.now();
    let currentPercentage = config.initialTrafficPercentage;
    let rolloutStep = 0;
    
    while (currentPercentage < 100) {
      console.log(`📊 Canary at ${currentPercentage}% traffic`);
      
      // Wait for observation period
      await new Promise(resolve => setTimeout(resolve, config.minObservationTime));
      
      // Collect metrics
      const metrics = await this.collectCanaryMetrics({
        mainContainerId: dockerId,
        canaryContainerId: canaryContainer.id,
        healthEndpoint,
        metricsEndpoint
      });
      
      // Analyze metrics
      const analysis = this.analyzeCanaryMetrics(metrics, config);
      
      if (!analysis.healthy) {
        return {
          success: false,
          reason: analysis.reason,
          lastTrafficPercentage: currentPercentage,
          metrics: metrics,
          duration: Date.now() - startTime
        };
      }
      
      // Increase traffic if healthy
      currentPercentage = Math.min(100, currentPercentage + config.incrementPercentage);
      await this.updateLoadBalancerConfig(dockerId, canaryContainer.id, currentPercentage);
      
      // Log progress
      await this.logCanaryProgress({
        deploymentId,
        step: ++rolloutStep,
        percentage: currentPercentage,
        metrics,
        timestamp: new Date().toISOString()
      });
      
      // Wait before next increment
      if (currentPercentage < 100) {
        await new Promise(resolve => setTimeout(resolve, config.incrementInterval));
      }
    }
    
    return {
      success: true,
      duration: Date.now() - startTime,
      finalMetrics: await this.collectCanaryMetrics({
        mainContainerId: dockerId,
        canaryContainerId: canaryContainer.id,
        healthEndpoint,
        metricsEndpoint
      })
    };
  }

  /**
   * Collect metrics from both main and canary containers
   */
  async collectCanaryMetrics(params) {
    const { mainContainerId, canaryContainerId, healthEndpoint, metricsEndpoint } = params;
    
    const metrics = {
      main: {
        healthy: true,
        responseTime: 0,
        errorRate: 0,
        requestCount: 0
      },
      canary: {
        healthy: true,
        responseTime: 0,
        errorRate: 0,
        requestCount: 0
      }
    };
    
    try {
      // Check health endpoints
      const mainHealth = await this.checkContainerHealth(mainContainerId, healthEndpoint);
      const canaryHealth = await this.checkContainerHealth(canaryContainerId, healthEndpoint);
      
      metrics.main.healthy = mainHealth.healthy;
      metrics.canary.healthy = canaryHealth.healthy;
      
      // Get nginx statistics
      const nginxStats = await this.getNginxStats(mainContainerId);
      
      // Parse upstream statistics
      if (nginxStats.upstreams) {
        const mainStats = nginxStats.upstreams.find(u => u.server.includes(mainContainerId));
        const canaryStats = nginxStats.upstreams.find(u => u.server.includes(canaryContainerId));
        
        if (mainStats) {
          metrics.main.requestCount = mainStats.requests || 0;
          metrics.main.responseTime = mainStats.response_time || 0;
          metrics.main.errorRate = mainStats.fails / (mainStats.requests || 1);
        }
        
        if (canaryStats) {
          metrics.canary.requestCount = canaryStats.requests || 0;
          metrics.canary.responseTime = canaryStats.response_time || 0;
          metrics.canary.errorRate = canaryStats.fails / (canaryStats.requests || 1);
        }
      }
    } catch (error) {
      console.error('Error collecting metrics:', error);
    }
    
    return metrics;
  }

  /**
   * Check container health
   */
  async checkContainerHealth(containerId, endpoint) {
    try {
      const container = this.docker.getContainer(containerId);
      const exec = await container.exec({
        AttachStdout: true,
        AttachStderr: true,
        Cmd: ['curl', '-f', `http://localhost:3000${endpoint}`]
      });
      
      const stream = await exec.start();
      const output = await new Promise((resolve, reject) => {
        let data = '';
        stream.on('data', chunk => data += chunk.toString());
        stream.on('end', () => resolve(data));
        stream.on('error', reject);
      });
      
      return { healthy: true, output };
    } catch (error) {
      return { healthy: false, error: error.message };
    }
  }

  /**
   * Get nginx statistics
   */
  async getNginxStats(mainContainerId) {
    try {
      // 보안 개선: execSync 대신 spawn 사용
      const { spawn } = require('child_process');
      const response = await new Promise((resolve, reject) => {
        let data = '';
        const curl = spawn('curl', ['-s', 'http://localhost/nginx_status']);
        curl.stdout.on('data', chunk => data += chunk.toString());
        curl.on('close', (code) => {
          if (code === 0) {
            resolve(data);
          } else {
            reject(new Error(`curl failed with code ${code}`));
          }
        });
        curl.on('error', reject);
      });
      // Parse nginx stub_status output
      const stats = {
        activeConnections: 0,
        accepts: 0,
        handled: 0,
        requests: 0,
        upstreams: []
      };
      
      const lines = response.split('\n');
      const activeMatch = lines[0].match(/Active connections: (\d+)/);
      if (activeMatch) stats.activeConnections = parseInt(activeMatch[1]);
      
      return stats;
    } catch (error) {
      return { error: error.message };
    }
  }

  /**
   * Analyze canary metrics
   */
  analyzeCanaryMetrics(metrics, config) {
    const analysis = {
      healthy: true,
      reason: null
    };
    
    // Check if canary is healthy
    if (!metrics.canary.healthy) {
      analysis.healthy = false;
      analysis.reason = 'Canary health check failed';
      return analysis;
    }
    
    // Check error rate
    if (metrics.canary.errorRate > config.errorThreshold) {
      analysis.healthy = false;
      analysis.reason = `Canary error rate ${(metrics.canary.errorRate * 100).toFixed(2)}% exceeds threshold ${config.errorThreshold * 100}%`;
      return analysis;
    }
    
    // Check response time
    if (metrics.canary.responseTime > config.responseTimeThreshold) {
      analysis.healthy = false;
      analysis.reason = `Canary response time ${metrics.canary.responseTime}ms exceeds threshold ${config.responseTimeThreshold}ms`;
      return analysis;
    }
    
    // Compare with main container
    if (metrics.canary.errorRate > metrics.main.errorRate * 1.5) {
      analysis.healthy = false;
      analysis.reason = 'Canary error rate significantly higher than main container';
      return analysis;
    }
    
    return analysis;
  }

  /**
   * Complete canary deployment
   */
  async completeCanaryDeployment(mainContainerId, canaryContainerId, environment) {
    console.log(`✅ Completing canary deployment for ${mainContainerId}`);
    
    // Stop old container
    const oldContainer = this.docker.getContainer(mainContainerId);
    await oldContainer.stop();
    
    // Rename canary to main
    const canaryContainer = this.docker.getContainer(canaryContainerId);
    await canaryContainer.rename({ name: mainContainerId });
    
    // Update labels (보안 개선)
    const { spawn } = require('child_process');
    await new Promise((resolve, reject) => {
      const docker = spawn('docker', ['update', '--label', 'canary=false', mainContainerId]);
      docker.on('close', (code) => {
        if (code === 0) {
          resolve();
        } else {
          reject(new Error(`docker update failed with code ${code}`));
        }
      });
      docker.on('error', reject);
    });
    
    // Remove old container
    await oldContainer.remove();
    
    // Update nginx to single upstream
    const nginxConfig = `
upstream ${mainContainerId}_backend {
    server ${mainContainerId}:3000;
}

server {
    listen 80;
    server_name ${mainContainerId}.local;
    
    location / {
        proxy_pass http://${mainContainerId}_backend;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
    }
}`;
    
    await fs.writeFile(`/etc/nginx/sites-available/${mainContainerId}`, nginxConfig);
    execSync('nginx -s reload');
  }

  /**
   * Rollback canary deployment
   */
  async rollbackCanary(canaryContainerId, mainContainerId) {
    console.log(`⚠️ Rolling back canary deployment for ${mainContainerId}`);
    
    try {
      // Stop and remove canary container
      const canaryContainer = this.docker.getContainer(canaryContainerId);
      await canaryContainer.stop();
      await canaryContainer.remove();
      
      // Restore nginx configuration
      const nginxConfig = `
upstream ${mainContainerId}_backend {
    server ${mainContainerId}:3000;
}

server {
    listen 80;
    server_name ${mainContainerId}.local;
    
    location / {
        proxy_pass http://${mainContainerId}_backend;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
    }
}`;
      
      await fs.writeFile(`/etc/nginx/sites-available/${mainContainerId}`, nginxConfig);
      execSync('nginx -s reload');
    } catch (error) {
      console.error('Error during rollback:', error);
    }
  }

  /**
   * Log canary progress
   */
  async logCanaryProgress(data) {
    const logPath = path.join(this.deploymentPath, 'canary-logs', `${data.deploymentId}.json`);
    await fs.mkdir(path.dirname(logPath), { recursive: true });
    
    let logs = [];
    try {
      const existing = await fs.readFile(logPath, 'utf8');
      logs = JSON.parse(existing);
    } catch {
      // File doesn't exist yet
    }
    
    logs.push(data);
    await fs.writeFile(logPath, JSON.stringify(logs, null, 2));
  }

  /**
   * Get canary deployment status
   */
  async getCanaryStatus(deploymentId) {
    const logPath = path.join(this.deploymentPath, 'canary-logs', `${deploymentId}.json`);
    
    try {
      const logs = JSON.parse(await fs.readFile(logPath, 'utf8'));
      const latest = logs[logs.length - 1];
      
      return {
        output: `Canary deployment ${deploymentId} status`,
        data: {
          deploymentId,
          currentPercentage: latest.percentage,
          steps: logs.length,
          metrics: latest.metrics,
          startTime: logs[0].timestamp,
          lastUpdate: latest.timestamp
        }
      };
    } catch (error) {
      throw new Error(`Canary deployment ${deploymentId} not found`);
    }
  }
}

export default CanaryDeployment;