shipdeck/lib/ultimate/workflow/dag-workflow.js

Ship MVPs in 48 hours. Fix bugs in 30 seconds. The command deck for developers who ship.

/**
 * DAG-Based Workflow Engine for Shipdeck Ultimate
 * Provides Directed Acyclic Graph workflow execution with parallel task orchestration
 */

const EventEmitter = require('events');
const { AgentExecutor } = require('../anthropic/agent-executor');

/**
 * Represents a single node in the workflow DAG
 */
class WorkflowNode {
  constructor(id, config) {
    this.id = id;
    this.name = config.name || id;
    this.description = config.description || '';
    this.agent = config.agent || 'backend-architect';
    this.prompt = config.prompt || '';
    this.inputs = config.inputs || [];
    this.outputs = config.outputs || [];
    this.dependencies = config.dependencies || [];
    this.retryConfig = config.retry || { maxAttempts: 3, backoffMs: 1000 };
    this.timeout = config.timeout || 300000; // 5 minutes default
    this.condition = config.condition || null; // Optional condition function
    this.parallel = config.parallel !== false; // Allow parallel by default
    
    // Runtime state
    this.status = 'pending'; // pending, ready, running, completed, failed, skipped
    this.result = null;
    this.error = null;
    this.attempts = 0;
    this.startedAt = null;
    this.completedAt = null;
    this.duration = null;
  }

  /**
   * Check if node is ready to execute (all dependencies completed)
   */
  isReady(completedNodes) {
    if (this.status !== 'pending') return false;
    
    return this.dependencies.every(depId => 
      completedNodes.has(depId)
    );
  }

  /**
   * Check if node should be skipped based on condition
   */
  shouldSkip(context) {
    if (!this.condition) return false;
    
    try {
      return !this.condition(context);
    } catch (error) {
      console.warn(`Condition evaluation failed for ${this.id}: ${error.message}`);
      return false;
    }
  }

  /**
   * Reset node to pending state
   */
  reset() {
    this.status = 'pending';
    this.result = null;
    this.error = null;
    this.attempts = 0;
    this.startedAt = null;
    this.completedAt = null;
    this.duration = null;
  }

  /**
   * Get node execution summary
   */
  getSummary() {
    return {
      id: this.id,
      name: this.name,
      status: this.status,
      agent: this.agent,
      attempts: this.attempts,
      duration: this.duration,
      error: this.error?.message || this.error,
      hasResult: !!this.result
    };
  }
}

/**
 * Main DAG Workflow Engine
 */
class DAGWorkflow extends EventEmitter {
  constructor(id, config = {}) {
    super();
    
    this.id = id;
    this.name = config.name || id;
    this.description = config.description || '';
    this.nodes = new Map();
    this.edges = new Map(); // nodeId -> Set of dependency nodeIds
    this.context = config.context || {};
    this.maxConcurrency = config.maxConcurrency || 10;
    this.timeout = config.timeout || 3600000; // 1 hour default
    
    // Runtime state
    this.status = 'pending'; // pending, running, completed, failed, cancelled
    this.startedAt = null;
    this.completedAt = null;
    this.completedNodes = new Set();
    this.failedNodes = new Set();
    this.skippedNodes = new Set();
    this.runningNodes = new Set();
    this.results = new Map();
    this.errors = new Map();
    
    // Agent executor
    if (config.anthropic?.skipInit) {
      this.agentExecutor = null; // Will be set manually in tests
    } else {
      this.agentExecutor = new AgentExecutor(config.anthropic || {});
    }
    
    // Execution control
    this.cancelled = false;
    this.executionPromise = null;
  }

  /**
   * Add a node to the workflow
   */
  addNode(nodeConfig) {
    if (typeof nodeConfig === 'string') {
      nodeConfig = { id: nodeConfig };
    }
    
    const node = new WorkflowNode(nodeConfig.id, nodeConfig);
    this.nodes.set(node.id, node);
    
    // Initialize edge tracking
    if (!this.edges.has(node.id)) {
      this.edges.set(node.id, new Set());
    }
    
    // Add dependencies as edges
    if (node.dependencies.length > 0) {
      for (const depId of node.dependencies) {
        this.addEdge(depId, node.id);
      }
    }
    
    return node;
  }

  /**
   * Add an edge (dependency) between nodes
   */
  addEdge(fromNodeId, toNodeId) {
    if (!this.nodes.has(fromNodeId)) {
      throw new Error(`Source node ${fromNodeId} does not exist`);
    }
    if (!this.nodes.has(toNodeId)) {
      throw new Error(`Target node ${toNodeId} does not exist`);
    }
    
    // Add to target node's dependencies
    if (!this.edges.has(toNodeId)) {
      this.edges.set(toNodeId, new Set());
    }
    this.edges.get(toNodeId).add(fromNodeId);
    
    // Update node's dependency list
    const toNode = this.nodes.get(toNodeId);
    if (!toNode.dependencies.includes(fromNodeId)) {
      toNode.dependencies.push(fromNodeId);
    }
  }

  /**
   * Validate the DAG (check for cycles and missing dependencies)
   */
  validate() {
    const issues = [];
    
    // Check for cycles using DFS
    const visited = new Set();
    const recursionStack = new Set();
    
    const hasCycle = (nodeId) => {
      if (recursionStack.has(nodeId)) {
        return true; // Back edge found - cycle detected
      }
      if (visited.has(nodeId)) {
        return false; // Already processed
      }
      
      visited.add(nodeId);
      recursionStack.add(nodeId);
      
      // Check all dependencies
      const dependencies = this.edges.get(nodeId) || new Set();
      for (const depId of dependencies) {
        if (hasCycle(depId)) {
          return true;
        }
      }
      
      recursionStack.delete(nodeId);
      return false;
    };
    
    // Check each node for cycles
    for (const nodeId of this.nodes.keys()) {
      if (hasCycle(nodeId)) {
        issues.push(`Cycle detected involving node: ${nodeId}`);
        break;
      }
    }
    
    // Check for missing dependencies
    for (const [nodeId, node] of this.nodes) {
      for (const depId of node.dependencies) {
        if (!this.nodes.has(depId)) {
          issues.push(`Node ${nodeId} depends on missing node: ${depId}`);
        }
      }
    }
    
    // Check for isolated nodes (no dependencies and no dependents)
    const hasIncoming = new Set();
    const hasOutgoing = new Set();
    
    for (const [nodeId, dependencies] of this.edges) {
      if (dependencies.size > 0) {
        hasIncoming.add(nodeId);
        for (const depId of dependencies) {
          hasOutgoing.add(depId);
        }
      }
    }
    
    for (const nodeId of this.nodes.keys()) {
      if (!hasIncoming.has(nodeId) && !hasOutgoing.has(nodeId) && this.nodes.size > 1) {
        issues.push(`Isolated node detected: ${nodeId}`);
      }
    }
    
    return {
      isValid: issues.length === 0,
      issues
    };
  }

  /**
   * Get nodes in topological order
   */
  getTopologicalOrder() {
    const visited = new Set();
    const stack = [];
    
    const visit = (nodeId) => {
      if (visited.has(nodeId)) return;
      
      visited.add(nodeId);
      
      // Visit all dependencies first
      const dependencies = this.edges.get(nodeId) || new Set();
      for (const depId of dependencies) {
        visit(depId);
      }
      
      stack.push(nodeId);
    };
    
    // Visit all nodes
    for (const nodeId of this.nodes.keys()) {
      visit(nodeId);
    }
    
    return stack;
  }

  /**
   * Get nodes that are ready to run
   */
  getReadyNodes() {
    const ready = [];
    
    for (const [nodeId, node] of this.nodes) {
      if (node.isReady(this.completedNodes) && !this.runningNodes.has(nodeId)) {
        // Check if should be skipped
        if (node.shouldSkip(this.context)) {
          this.skipNode(nodeId, 'Condition not met');
          continue;
        }
        
        ready.push(node);
      }
    }
    
    return ready;
  }

  /**
   * Execute the workflow
   */
  async execute() {
    if (this.status === 'running') {
      throw new Error('Workflow is already running');
    }
    
    // Validate workflow before execution
    const validation = this.validate();
    if (!validation.isValid) {
      throw new Error(`Workflow validation failed: ${validation.issues.join(', ')}`);
    }
    
    this.status = 'running';
    this.startedAt = new Date();
    this.cancelled = false;
    
    console.log(`🚀 Starting DAG workflow: ${this.name} (${this.nodes.size} nodes)`);
    this.emit('workflow:started', { id: this.id, nodeCount: this.nodes.size });
    
    try {
      // Create execution promise for timeout handling
      this.executionPromise = this._executeNodes();
      
      // Setup timeout
      const timeoutPromise = new Promise((_, reject) => {
        setTimeout(() => reject(new Error('Workflow timeout')), this.timeout);
      });
      
      await Promise.race([this.executionPromise, timeoutPromise]);
      
      // Check final status
      if (this.failedNodes.size > 0) {
        this.status = 'failed';
        throw new Error(`Workflow failed: ${this.failedNodes.size} node(s) failed`);
      } else if (this.cancelled) {
        this.status = 'cancelled';
        console.log(`🛑 Workflow cancelled: ${this.id}`);
      } else {
        this.status = 'completed';
        console.log(`✅ Workflow completed: ${this.id} (${this.completedNodes.size} nodes)`);
      }
      
    } catch (error) {
      this.status = 'failed';
      console.error(`❌ Workflow execution failed: ${error.message}`);
      throw error;
      
    } finally {
      this.completedAt = new Date();
      this.emit('workflow:finished', {
        id: this.id,
        status: this.status,
        duration: this.completedAt - this.startedAt,
        completed: this.completedNodes.size,
        failed: this.failedNodes.size,
        skipped: this.skippedNodes.size
      });
    }
    
    return this.getExecutionSummary();
  }

  /**
   * Internal method to execute nodes with parallel processing
   */
  async _executeNodes() {
    const runningPromises = new Map();
    
    while (!this.cancelled && 
           (this.completedNodes.size + this.failedNodes.size + this.skippedNodes.size < this.nodes.size)) {
      
      // Get nodes ready to execute
      const readyNodes = this.getReadyNodes();
      
      if (readyNodes.length === 0 && runningPromises.size === 0) {
        // No ready nodes and nothing running - check for deadlock
        const pendingNodes = Array.from(this.nodes.values())
          .filter(node => node.status === 'pending');
        
        if (pendingNodes.length > 0) {
          throw new Error(`Workflow deadlock: ${pendingNodes.map(n => n.id).join(', ')} cannot proceed`);
        }
        break;
      }
      
      // Start execution of ready nodes (respecting concurrency limit)
      const slotsAvailable = this.maxConcurrency - runningPromises.size;
      const nodesToStart = readyNodes.slice(0, slotsAvailable);
      
      for (const node of nodesToStart) {
        const executionPromise = this._executeNode(node);
        runningPromises.set(node.id, executionPromise);
        
        // Handle completion
        executionPromise
          .then(() => {
            runningPromises.delete(node.id);
          })
          .catch(() => {
            runningPromises.delete(node.id);
          });
      }
      
      // Wait for at least one node to complete if we're at capacity
      if (runningPromises.size >= this.maxConcurrency && runningPromises.size > 0) {
        await Promise.race(runningPromises.values());
      }
      
      // Small delay to prevent tight loop
      await new Promise(resolve => setTimeout(resolve, 10));
    }
    
    // Wait for all remaining executions to complete
    if (runningPromises.size > 0) {
      await Promise.allSettled(runningPromises.values());
    }
  }

  /**
   * Execute a single node
   */
  async _executeNode(node) {
    const nodeId = node.id;
    
    try {
      console.log(`⚡ Starting node: ${node.name} (${node.agent})`);
      
      node.status = 'running';
      node.startedAt = new Date();
      this.runningNodes.add(nodeId);
      
      this.emit('node:started', { nodeId, name: node.name, agent: node.agent });
      
      // Prepare node context with dependency results
      const nodeContext = {
        ...this.context,
        dependencies: {}
      };
      
      // Add dependency results to context
      for (const depId of node.dependencies) {
        if (this.results.has(depId)) {
          nodeContext.dependencies[depId] = this.results.get(depId);
        }
      }
      
      // Prepare prompt with context
      let prompt = node.prompt;
      if (node.inputs.length > 0) {
        const inputData = {};
        for (const inputKey of node.inputs) {
          if (nodeContext[inputKey] !== undefined) {
            inputData[inputKey] = nodeContext[inputKey];
          }
        }
        
        if (Object.keys(inputData).length > 0) {
          prompt += `\n\nInput Data:\n${JSON.stringify(inputData, null, 2)}`;
        }
      }
      
      // Add dependency context
      if (Object.keys(nodeContext.dependencies).length > 0) {
        prompt += `\n\nDependency Results:\n${JSON.stringify(nodeContext.dependencies, null, 2)}`;
      }
      
      // Execute with timeout and retry
      let result = null;
      let lastError = null;
      
      for (let attempt = 1; attempt <= node.retryConfig.maxAttempts; attempt++) {
        node.attempts = attempt;
        
        try {
          // Create timeout promise
          const timeoutPromise = new Promise((_, reject) => {
            setTimeout(() => reject(new Error(`Node timeout after ${node.timeout}ms`)), node.timeout);
          });
          
          // Execute agent
          const executionPromise = this.agentExecutor.executeAgent(node.agent, prompt, {
            sessionId: `${this.id}-${nodeId}-${attempt}`
          });
          
          const response = await Promise.race([executionPromise, timeoutPromise]);
          result = response;
          lastError = null;
          break;
          
        } catch (error) {
          lastError = error;
          console.warn(`Node ${nodeId} attempt ${attempt} failed: ${error.message}`);
          
          if (attempt < node.retryConfig.maxAttempts) {
            const backoffMs = node.retryConfig.backoffMs * Math.pow(2, attempt - 1);
            await new Promise(resolve => setTimeout(resolve, backoffMs));
          }
        }
      }
      
      if (lastError && !result) {
        throw lastError;
      }
      
      // Process result
      node.result = result;
      node.status = 'completed';
      node.completedAt = new Date();
      node.duration = node.completedAt - node.startedAt;
      
      // Store results
      this.results.set(nodeId, result);
      
      // Update context with outputs
      if (node.outputs.length > 0 && result) {
        for (const outputKey of node.outputs) {
          if (result.content && result.content[0]?.text) {
            this.context[outputKey] = result.content[0].text;
          }
        }
      }
      
      // Mark as completed
      this.completedNodes.add(nodeId);
      this.runningNodes.delete(nodeId);
      
      console.log(`✅ Node completed: ${node.name} (${node.duration}ms, ${node.attempts} attempts)`);
      
      this.emit('node:completed', {
        nodeId,
        name: node.name,
        duration: node.duration,
        attempts: node.attempts,
        hasTests: !!result?.tests
      });
      
    } catch (error) {
      // Handle failure
      node.error = error;
      node.status = 'failed';
      node.completedAt = new Date();
      node.duration = node.completedAt - node.startedAt;
      
      this.failedNodes.add(nodeId);
      this.runningNodes.delete(nodeId);
      this.errors.set(nodeId, error);
      
      console.error(`❌ Node failed: ${node.name} - ${error.message}`);
      
      this.emit('node:failed', {
        nodeId,
        name: node.name,
        error: error.message,
        attempts: node.attempts
      });
      
      throw error;
    }
  }

  /**
   * Skip a node with reason
   */
  skipNode(nodeId, reason = 'Skipped') {
    const node = this.nodes.get(nodeId);
    if (!node) return;
    
    node.status = 'skipped';
    node.error = reason;
    node.completedAt = new Date();
    
    this.skippedNodes.add(nodeId);
    
    console.log(`⏭  Node skipped: ${node.name} - ${reason}`);
    this.emit('node:skipped', { nodeId, name: node.name, reason });
  }

  /**
   * Cancel workflow execution
   */
  cancel() {
    this.cancelled = true;
    
    // Cancel running nodes (agents)
    for (const nodeId of this.runningNodes) {
      const node = this.nodes.get(nodeId);
      if (node) {
        node.status = 'cancelled';
        node.error = 'Workflow cancelled';
      }
    }
    
    this.runningNodes.clear();
    console.log(`🛑 Workflow cancellation requested: ${this.id}`);
    
    this.emit('workflow:cancelled', { id: this.id });
  }

  /**
   * Get workflow execution summary
   */
  getExecutionSummary() {
    return {
      id: this.id,
      name: this.name,
      status: this.status,
      startedAt: this.startedAt,
      completedAt: this.completedAt,
      duration: this.completedAt ? this.completedAt - this.startedAt : null,
      nodes: {
        total: this.nodes.size,
        completed: this.completedNodes.size,
        failed: this.failedNodes.size,
        skipped: this.skippedNodes.size,
        running: this.runningNodes.size
      },
      results: Object.fromEntries(this.results),
      errors: Object.fromEntries(
        Array.from(this.errors.entries()).map(([k, v]) => [k, v.message])
      )
    };
  }

  /**
   * Get workflow progress
   */
  getProgress() {
    const total = this.nodes.size;
    const completed = this.completedNodes.size + this.failedNodes.size + this.skippedNodes.size;
    
    return {
      total,
      completed,
      running: this.runningNodes.size,
      percentage: total > 0 ? Math.round((completed / total) * 100) : 0,
      nodes: Array.from(this.nodes.values()).map(node => node.getSummary())
    };
  }

  /**
   * Reset workflow to initial state
   */
  reset() {
    if (this.status === 'running') {
      throw new Error('Cannot reset running workflow');
    }
    
    // Reset all nodes
    for (const node of this.nodes.values()) {
      node.reset();
    }
    
    // Reset workflow state
    this.status = 'pending';
    this.startedAt = null;
    this.completedAt = null;
    this.completedNodes.clear();
    this.failedNodes.clear();
    this.skippedNodes.clear();
    this.runningNodes.clear();
    this.results.clear();
    this.errors.clear();
    this.cancelled = false;
    
    console.log(`🔄 Workflow reset: ${this.id}`);
    this.emit('workflow:reset', { id: this.id });
  }

  /**
   * Create a workflow from configuration
   */
  static fromConfig(id, config) {
    const workflow = new DAGWorkflow(id, config);
    
    // Add nodes from config
    if (config.nodes) {
      for (const nodeConfig of config.nodes) {
        workflow.addNode(nodeConfig);
      }
    }
    
    // Add edges from config
    if (config.edges) {
      for (const edge of config.edges) {
        workflow.addEdge(edge.from, edge.to);
      }
    }
    
    return workflow;
  }
}

module.exports = { DAGWorkflow, WorkflowNode };