shipdeck/lib/ultimate/quality-gates/rollback-trigger.js

Ship MVPs in 48 hours. Fix bugs in 30 seconds. The command deck for developers who ship.

/**
 * Rollback Trigger for Quality Gates
 * 
 * Automatic rollback system for critical failures:
 * - Critical security vulnerabilities
 * - Breaking changes detection
 * - Performance regression beyond threshold
 * - Test coverage drops below critical level
 * - Build/deployment failures
 */

const EventEmitter = require('events');

class RollbackTrigger extends EventEmitter {
  constructor(config = {}) {
    super();
    
    this.config = {
      // Rollback triggers
      enableAutoRollback: config.enableAutoRollback !== false,
      rollbackOnCriticalSecurity: config.rollbackOnCriticalSecurity !== false,
      rollbackOnBreakingChanges: config.rollbackOnBreakingChanges !== false,
      rollbackOnPerformanceRegression: config.rollbackOnPerformanceRegression || false,
      rollbackOnTestFailures: config.rollbackOnTestFailures || false,
      
      // Rollback thresholds
      criticalVulnerabilityThreshold: config.criticalVulnerabilityThreshold || 1,
      performanceRegressionThreshold: config.performanceRegressionThreshold || 0.3, // 30%
      testCoverageDropThreshold: config.testCoverageDropThreshold || 0.2, // 20%
      failedTestThreshold: config.failedTestThreshold || 0.1, // 10%
      
      // Rollback strategies
      rollbackStrategy: config.rollbackStrategy || 'git_revert', // git_revert, backup_restore, feature_toggle
      maxRollbackAttempts: config.maxRollbackAttempts || 3,
      rollbackTimeout: config.rollbackTimeout || 300000, // 5 minutes
      
      // Notification settings
      notifyOnRollback: config.notifyOnRollback !== false,
      escalateAfterRollback: config.escalateAfterRollback !== false,
      
      ...config
    };
    
    // Rollback state tracking
    this.rollbackHistory = [];
    this.activeRollbacks = new Map();
    this.rollbackStrategies = this._initializeRollbackStrategies();
    
    // Statistics
    this.stats = {
      totalRollbacks: 0,
      successfulRollbacks: 0,
      failedRollbacks: 0,
      rollbackReasons: {
        security: 0,
        breaking: 0,
        performance: 0,
        tests: 0,
        build: 0,
        manual: 0
      },
      averageRollbackTime: 0
    };
  }
  
  /**
   * Trigger rollback for a quality gate failure
   */
  async triggerRollback(gateId, failureResult, reason = 'quality_gate_failure') {
    const rollbackId = this._generateRollbackId();
    const startTime = Date.now();
    
    if (!this.config.enableAutoRollback) {
      console.log(`⚠️ Auto-rollback disabled - would rollback ${gateId} due to: ${reason}`);
      return {
        triggered: false,
        reason: 'auto_rollback_disabled',
        rollbackId: null
      };
    }
    
    console.log(`🚨 Triggering rollback ${rollbackId} for gate ${gateId}: ${reason}`);
    
    try {
      // Determine if rollback is required
      const rollbackDecision = await this._shouldTriggerRollback(failureResult, reason);
      
      if (!rollbackDecision.shouldRollback) {
        console.log(`ℹ️ Rollback not required: ${rollbackDecision.reason}`);
        return {
          triggered: false,
          reason: rollbackDecision.reason,
          rollbackId: null
        };
      }
      
      // Create rollback record
      const rollbackRecord = {
        id: rollbackId,
        gateId,
        reason,
        failureResult,
        startTime,
        status: 'in_progress',
        strategy: rollbackDecision.strategy,
        attempts: 0,
        maxAttempts: this.config.maxRollbackAttempts
      };
      
      this.activeRollbacks.set(rollbackId, rollbackRecord);
      
      // Send notifications
      await this._notifyRollbackStarted(rollbackRecord);
      
      // Execute rollback
      const rollbackResult = await this._executeRollback(rollbackRecord);
      
      // Update record
      rollbackRecord.status = rollbackResult.success ? 'completed' : 'failed';
      rollbackRecord.completedAt = Date.now();
      rollbackRecord.duration = rollbackRecord.completedAt - rollbackRecord.startTime;
      rollbackRecord.result = rollbackResult;
      
      // Move to history
      this.rollbackHistory.push(rollbackRecord);
      this.activeRollbacks.delete(rollbackId);
      
      // Update statistics
      this.stats.totalRollbacks++;
      this.stats.rollbackReasons[this._categorizRollbackReason(reason)]++;
      
      if (rollbackResult.success) {
        this.stats.successfulRollbacks++;
        console.log(`✅ Rollback completed successfully: ${rollbackId} (${rollbackRecord.duration}ms)`);
        
        this.emit('rollback:success', {
          rollbackId,
          gateId,
          reason,
          duration: rollbackRecord.duration
        });
        
      } else {
        this.stats.failedRollbacks++;
        console.error(`❌ Rollback failed: ${rollbackId} - ${rollbackResult.error}`);
        
        this.emit('rollback:failed', {
          rollbackId,
          gateId,
          reason,
          error: rollbackResult.error
        });
        
        // Escalate if configured
        if (this.config.escalateAfterRollback) {
          await this._escalateFailedRollback(rollbackRecord);
        }
      }
      
      // Send completion notifications
      await this._notifyRollbackCompleted(rollbackRecord);
      
      this._updateAverageRollbackTime(rollbackRecord.duration);
      
      return {
        triggered: true,
        success: rollbackResult.success,
        rollbackId,
        duration: rollbackRecord.duration,
        result: rollbackResult
      };
      
    } catch (error) {
      console.error(`❌ Rollback trigger failed for ${rollbackId}: ${error.message}`);
      
      // Clean up active rollback
      if (this.activeRollbacks.has(rollbackId)) {
        const record = this.activeRollbacks.get(rollbackId);
        record.status = 'failed';
        record.error = error.message;
        this.rollbackHistory.push(record);
        this.activeRollbacks.delete(rollbackId);
      }
      
      this.stats.failedRollbacks++;
      
      this.emit('rollback:error', {
        rollbackId,
        gateId,
        error: error.message
      });
      
      return {
        triggered: false,
        success: false,
        rollbackId,
        error: error.message
      };
    }
  }
  
  /**
   * Determine if rollback should be triggered
   */
  async _shouldTriggerRollback(failureResult, reason) {
    const decision = {
      shouldRollback: false,
      reason: '',
      strategy: this.config.rollbackStrategy
    };
    
    // Check security triggers
    if (this.config.rollbackOnCriticalSecurity) {
      const criticalSecurityIssues = this._countCriticalSecurityIssues(failureResult);
      if (criticalSecurityIssues >= this.config.criticalVulnerabilityThreshold) {
        decision.shouldRollback = true;
        decision.reason = `Critical security vulnerabilities found: ${criticalSecurityIssues}`;
        decision.strategy = 'immediate_revert';
        return decision;
      }
    }
    
    // Check breaking changes
    if (this.config.rollbackOnBreakingChanges) {
      const hasBreakingChanges = this._detectBreakingChanges(failureResult);
      if (hasBreakingChanges) {
        decision.shouldRollback = true;
        decision.reason = 'Breaking changes detected';
        decision.strategy = 'git_revert';
        return decision;
      }
    }
    
    // Check performance regression
    if (this.config.rollbackOnPerformanceRegression) {
      const performanceRegression = this._analyzePerformanceRegression(failureResult);
      if (performanceRegression.severity >= this.config.performanceRegressionThreshold) {
        decision.shouldRollback = true;
        decision.reason = `Performance regression: ${Math.round(performanceRegression.severity * 100)}%`;
        decision.strategy = 'feature_toggle';
        return decision;
      }
    }
    
    // Check test failures
    if (this.config.rollbackOnTestFailures) {
      const testFailures = this._analyzeTestFailures(failureResult);
      if (testFailures.failureRate >= this.config.failedTestThreshold) {
        decision.shouldRollback = true;
        decision.reason = `High test failure rate: ${Math.round(testFailures.failureRate * 100)}%`;
        decision.strategy = 'git_revert';
        return decision;
      }
    }
    
    // Check manual rollback triggers
    if (reason === 'manual_trigger') {
      decision.shouldRollback = true;
      decision.reason = 'Manual rollback requested';
      decision.strategy = this.config.rollbackStrategy;
      return decision;
    }
    
    decision.reason = 'No rollback triggers activated';
    return decision;
  }
  
  /**
   * Execute the rollback using the selected strategy
   */
  async _executeRollback(rollbackRecord) {
    const { strategy, id: rollbackId } = rollbackRecord;
    
    console.log(`🔄 Executing rollback ${rollbackId} using strategy: ${strategy}`);
    
    const strategyExecutor = this.rollbackStrategies[strategy];
    if (!strategyExecutor) {
      throw new Error(`Unknown rollback strategy: ${strategy}`);
    }
    
    let lastError = null;
    
    // Attempt rollback with retries
    for (let attempt = 1; attempt <= rollbackRecord.maxAttempts; attempt++) {
      rollbackRecord.attempts = attempt;
      
      try {
        console.log(`  🔄 Rollback attempt ${attempt}/${rollbackRecord.maxAttempts}`);
        
        const result = await this._executeWithTimeout(
          () => strategyExecutor(rollbackRecord),
          this.config.rollbackTimeout
        );
        
        if (result.success) {
          console.log(`  ✅ Rollback successful on attempt ${attempt}`);
          return result;
        } else {
          lastError = result.error;
          console.warn(`  ⚠️ Rollback attempt ${attempt} failed: ${result.error}`);
        }
        
      } catch (error) {
        lastError = error.message;
        console.warn(`  ⚠️ Rollback attempt ${attempt} failed: ${error.message}`);
      }
      
      // Wait before retry (except on last attempt)
      if (attempt < rollbackRecord.maxAttempts) {
        const delay = Math.min(5000 * attempt, 30000); // Exponential backoff, max 30s
        await new Promise(resolve => setTimeout(resolve, delay));
      }
    }
    
    return {
      success: false,
      error: lastError || 'All rollback attempts failed',
      attempts: rollbackRecord.attempts
    };
  }
  
  /**
   * Execute function with timeout
   */
  async _executeWithTimeout(func, timeout) {
    return Promise.race([
      func(),
      new Promise((_, reject) => 
        setTimeout(() => reject(new Error('Rollback timeout')), timeout)
      )
    ]);
  }
  
  /**
   * Initialize rollback strategies
   */
  _initializeRollbackStrategies() {
    return {
      git_revert: this._executeGitRevert.bind(this),
      backup_restore: this._executeBackupRestore.bind(this),
      feature_toggle: this._executeFeatureToggle.bind(this),
      immediate_revert: this._executeImmediateRevert.bind(this),
      database_rollback: this._executeDatabaseRollback.bind(this)
    };
  }
  
  /**
   * Git revert strategy
   */
  async _executeGitRevert(rollbackRecord) {
    console.log('  📝 Executing git revert strategy');
    
    try {
      // In a real implementation, this would execute git commands
      // For now, we'll simulate the process
      
      const steps = [
        'Identifying commits to revert',
        'Creating revert commit',
        'Pushing revert to repository',
        'Triggering deployment pipeline'
      ];
      
      for (const step of steps) {
        console.log(`    ${step}...`);
        await new Promise(resolve => setTimeout(resolve, 500)); // Simulate work
      }
      
      return {
        success: true,
        strategy: 'git_revert',
        details: {
          revertCommit: 'abc123',
          pushedToRemote: true,
          deploymentTriggered: true
        }
      };
      
    } catch (error) {
      return {
        success: false,
        error: `Git revert failed: ${error.message}`,
        strategy: 'git_revert'
      };
    }
  }
  
  /**
   * Backup restore strategy
   */
  async _executeBackupRestore(rollbackRecord) {
    console.log('  💾 Executing backup restore strategy');
    
    try {
      const steps = [
        'Identifying latest stable backup',
        'Stopping current services',
        'Restoring from backup',
        'Restarting services',
        'Verifying restoration'
      ];
      
      for (const step of steps) {
        console.log(`    ${step}...`);
        await new Promise(resolve => setTimeout(resolve, 800)); // Simulate work
      }
      
      return {
        success: true,
        strategy: 'backup_restore',
        details: {
          backupTimestamp: new Date(Date.now() - 3600000).toISOString(), // 1 hour ago
          restoredFiles: ['app/', 'config/', 'data/'],
          servicesRestarted: ['web', 'api', 'database']
        }
      };
      
    } catch (error) {
      return {
        success: false,
        error: `Backup restore failed: ${error.message}`,
        strategy: 'backup_restore'
      };
    }
  }
  
  /**
   * Feature toggle strategy
   */
  async _executeFeatureToggle(rollbackRecord) {
    console.log('  🎛️  Executing feature toggle strategy');
    
    try {
      const steps = [
        'Identifying affected features',
        'Disabling problematic features',
        'Updating feature flag configuration',
        'Verifying feature disablement'
      ];
      
      for (const step of steps) {
        console.log(`    ${step}...`);
        await new Promise(resolve => setTimeout(resolve, 300)); // Simulate work
      }
      
      return {
        success: true,
        strategy: 'feature_toggle',
        details: {
          disabledFeatures: ['new-payment-flow', 'experimental-ui'],
          configurationUpdated: true,
          rolloutPercentage: 0
        }
      };
      
    } catch (error) {
      return {
        success: false,
        error: `Feature toggle failed: ${error.message}`,
        strategy: 'feature_toggle'
      };
    }
  }
  
  /**
   * Immediate revert strategy (for critical security issues)
   */
  async _executeImmediateRevert(rollbackRecord) {
    console.log('  🚨 Executing immediate revert strategy');
    
    try {
      const steps = [
        'Emergency: Stopping all traffic',
        'Reverting to last known secure state',
        'Updating security configurations',
        'Gradually restoring traffic'
      ];
      
      for (const step of steps) {
        console.log(`    ${step}...`);
        await new Promise(resolve => setTimeout(resolve, 200)); // Quick execution
      }
      
      return {
        success: true,
        strategy: 'immediate_revert',
        details: {
          trafficStopped: true,
          secureStateRestored: true,
          gradualRestoreStarted: true,
          emergencyMode: true
        }
      };
      
    } catch (error) {
      return {
        success: false,
        error: `Immediate revert failed: ${error.message}`,
        strategy: 'immediate_revert'
      };
    }
  }
  
  /**
   * Database rollback strategy
   */
  async _executeDatabaseRollback(rollbackRecord) {
    console.log('  🗄️ Executing database rollback strategy');
    
    try {
      const steps = [
        'Creating database backup',
        'Identifying migration to rollback',
        'Executing rollback migration',
        'Verifying data integrity'
      ];
      
      for (const step of steps) {
        console.log(`    ${step}...`);
        await new Promise(resolve => setTimeout(resolve, 1000)); // Simulate DB work
      }
      
      return {
        success: true,
        strategy: 'database_rollback',
        details: {
          backupCreated: true,
          migrationsRolledBack: ['20231201_add_new_column'],
          dataIntegrityVerified: true
        }
      };
      
    } catch (error) {
      return {
        success: false,
        error: `Database rollback failed: ${error.message}`,
        strategy: 'database_rollback'
      };
    }
  }
  
  // Analysis methods
  
  /**
   * Count critical security issues in failure result
   */
  _countCriticalSecurityIssues(failureResult) {
    if (!failureResult.issues) return 0;
    
    return failureResult.issues.filter(issue => 
      issue.severity === 'critical' && 
      (issue.type?.includes('security') || issue.owaspCategory)
    ).length;
  }
  
  /**
   * Detect breaking changes in failure result
   */
  _detectBreakingChanges(failureResult) {
    if (!failureResult.issues) return false;
    
    return failureResult.issues.some(issue =>
      issue.type?.includes('breaking') ||
      issue.description?.toLowerCase().includes('breaking') ||
      issue.title?.toLowerCase().includes('breaking')
    );
  }
  
  /**
   * Analyze performance regression
   */
  _analyzePerformanceRegression(failureResult) {
    if (!failureResult.issues) {
      return { severity: 0 };
    }
    
    const performanceIssues = failureResult.issues.filter(issue =>
      issue.type?.includes('performance') ||
      issue.type?.includes('regression')
    );
    
    if (performanceIssues.length === 0) {
      return { severity: 0 };
    }
    
    // Calculate severity based on performance degradation
    let maxSeverity = 0;
    
    performanceIssues.forEach(issue => {
      if (issue.change && typeof issue.change === 'number') {
        const severityRatio = Math.abs(issue.change) / 100; // Convert percentage to ratio
        maxSeverity = Math.max(maxSeverity, severityRatio);
      } else if (issue.severity === 'high') {
        maxSeverity = Math.max(maxSeverity, 0.5);
      } else if (issue.severity === 'critical') {
        maxSeverity = Math.max(maxSeverity, 0.8);
      }
    });
    
    return { severity: maxSeverity };
  }
  
  /**
   * Analyze test failures
   */
  _analyzeTestFailures(failureResult) {
    if (!failureResult.testMetrics && !failureResult.issues) {
      return { failureRate: 0 };
    }
    
    // Check for test-related issues
    const testIssues = failureResult.issues?.filter(issue =>
      issue.type?.includes('test') ||
      issue.title?.toLowerCase().includes('test')
    ) || [];
    
    // If we have test metrics, calculate failure rate
    if (failureResult.testMetrics?.totalTests) {
      const failedTests = testIssues.length;
      const totalTests = failureResult.testMetrics.totalTests;
      return { failureRate: failedTests / totalTests };
    }
    
    // Otherwise, estimate based on issue severity
    const criticalTestIssues = testIssues.filter(issue => issue.severity === 'critical').length;
    const highTestIssues = testIssues.filter(issue => issue.severity === 'high').length;
    
    // Rough estimation
    return { failureRate: (criticalTestIssues * 0.8 + highTestIssues * 0.5) / Math.max(testIssues.length, 1) };
  }
  
  // Notification methods
  
  /**
   * Notify that rollback has started
   */
  async _notifyRollbackStarted(rollbackRecord) {
    if (!this.config.notifyOnRollback) return;
    
    const notification = {
      type: 'rollback_started',
      rollbackId: rollbackRecord.id,
      gateId: rollbackRecord.gateId,
      reason: rollbackRecord.reason,
      strategy: rollbackRecord.strategy,
      timestamp: rollbackRecord.startTime
    };
    
    console.log(`🔔 ROLLBACK STARTED: ${rollbackRecord.id}`);
    console.log(`   Gate: ${rollbackRecord.gateId}`);
    console.log(`   Reason: ${rollbackRecord.reason}`);
    console.log(`   Strategy: ${rollbackRecord.strategy}`);
    
    this.emit('notification:rollback_started', notification);
  }
  
  /**
   * Notify that rollback has completed
   */
  async _notifyRollbackCompleted(rollbackRecord) {
    if (!this.config.notifyOnRollback) return;
    
    const notification = {
      type: 'rollback_completed',
      rollbackId: rollbackRecord.id,
      success: rollbackRecord.status === 'completed',
      duration: rollbackRecord.duration,
      attempts: rollbackRecord.attempts,
      result: rollbackRecord.result
    };
    
    const status = rollbackRecord.status === 'completed' ? '✅ SUCCESS' : '❌ FAILED';
    console.log(`🔔 ROLLBACK COMPLETED: ${rollbackRecord.id} - ${status}`);
    
    this.emit('notification:rollback_completed', notification);
  }
  
  /**
   * Escalate failed rollback
   */
  async _escalateFailedRollback(rollbackRecord) {
    console.log(`🚨 ESCALATING FAILED ROLLBACK: ${rollbackRecord.id}`);
    
    const escalation = {
      rollbackId: rollbackRecord.id,
      gateId: rollbackRecord.gateId,
      reason: rollbackRecord.reason,
      failureReason: rollbackRecord.result?.error,
      attempts: rollbackRecord.attempts,
      timestamp: Date.now()
    };
    
    this.emit('escalation:failed_rollback', escalation);
    
    // In a real implementation, this might:
    // - Send alerts to on-call engineers
    // - Create high-priority incidents
    // - Trigger manual intervention workflows
  }
  
  // Utility methods
  
  _generateRollbackId() {
    return `rollback-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
  }
  
  _categorizRollbackReason(reason) {
    if (reason.includes('security')) return 'security';
    if (reason.includes('breaking')) return 'breaking';
    if (reason.includes('performance')) return 'performance';
    if (reason.includes('test')) return 'tests';
    if (reason.includes('build')) return 'build';
    if (reason.includes('manual')) return 'manual';
    return 'security'; // Default category
  }
  
  _updateAverageRollbackTime(newTime) {
    if (this.stats.totalRollbacks === 1) {
      this.stats.averageRollbackTime = newTime;
    } else {
      const currentTotal = this.stats.averageRollbackTime * (this.stats.totalRollbacks - 1);
      this.stats.averageRollbackTime = (currentTotal + newTime) / this.stats.totalRollbacks;
    }
  }
  
  /**
   * Get rollback status
   */
  getRollbackStatus(rollbackId) {
    // Check active rollbacks
    if (this.activeRollbacks.has(rollbackId)) {
      return {
        found: true,
        active: true,
        ...this.activeRollbacks.get(rollbackId)
      };
    }
    
    // Check history
    const historical = this.rollbackHistory.find(r => r.id === rollbackId);
    if (historical) {
      return {
        found: true,
        active: false,
        ...historical
      };
    }
    
    return { found: false };
  }
  
  /**
   * Get all active rollbacks
   */
  getActiveRollbacks() {
    return Array.from(this.activeRollbacks.values()).map(rollback => ({
      id: rollback.id,
      gateId: rollback.gateId,
      reason: rollback.reason,
      strategy: rollback.strategy,
      status: rollback.status,
      attempts: rollback.attempts,
      maxAttempts: rollback.maxAttempts,
      startTime: rollback.startTime,
      duration: rollback.completedAt ? rollback.completedAt - rollback.startTime : Date.now() - rollback.startTime
    }));
  }
  
  /**
   * Get rollback history
   */
  getRollbackHistory(limit = 50) {
    return this.rollbackHistory
      .slice(-limit)
      .map(rollback => ({
        id: rollback.id,
        gateId: rollback.gateId,
        reason: rollback.reason,
        strategy: rollback.strategy,
        status: rollback.status,
        success: rollback.status === 'completed',
        attempts: rollback.attempts,
        duration: rollback.duration,
        startTime: rollback.startTime,
        completedAt: rollback.completedAt
      }));
  }
  
  /**
   * Get rollback statistics
   */
  getStatistics() {
    return {
      ...this.stats,
      averageRollbackTime: Math.round(this.stats.averageRollbackTime),
      successRate: this.stats.totalRollbacks > 0 ? 
        Math.round((this.stats.successfulRollbacks / this.stats.totalRollbacks) * 100) : 0,
      activeRollbacks: this.activeRollbacks.size,
      historySize: this.rollbackHistory.length,
      config: {
        enableAutoRollback: this.config.enableAutoRollback,
        rollbackOnCriticalSecurity: this.config.rollbackOnCriticalSecurity,
        rollbackOnBreakingChanges: this.config.rollbackOnBreakingChanges,
        rollbackOnPerformanceRegression: this.config.rollbackOnPerformanceRegression,
        rollbackOnTestFailures: this.config.rollbackOnTestFailures,
        rollbackStrategy: this.config.rollbackStrategy
      }
    };
  }
  
  /**
   * Manual rollback trigger
   */
  async triggerManualRollback(gateId, reason, strategy = null) {
    const rollbackStrategy = strategy || this.config.rollbackStrategy;
    
    console.log(`🔧 Manual rollback triggered for gate ${gateId}: ${reason}`);
    
    return this.triggerRollback(gateId, {
      issues: [{
        type: 'manual_trigger',
        severity: 'high',
        title: 'Manual Rollback Requested',
        description: reason
      }]
    }, 'manual_trigger');
  }
  
  /**
   * Cancel active rollback
   */
  async cancelRollback(rollbackId, reason = 'Cancelled by user') {
    const rollback = this.activeRollbacks.get(rollbackId);
    
    if (!rollback) {
      throw new Error(`No active rollback found: ${rollbackId}`);
    }
    
    rollback.status = 'cancelled';
    rollback.cancellationReason = reason;
    rollback.completedAt = Date.now();
    rollback.duration = rollback.completedAt - rollback.startTime;
    
    this.rollbackHistory.push(rollback);
    this.activeRollbacks.delete(rollbackId);
    
    console.log(`🚫 Rollback cancelled: ${rollbackId} - ${reason}`);
    
    this.emit('rollback:cancelled', {
      rollbackId,
      reason
    });
  }
}

module.exports = { RollbackTrigger };