devin-workflow/test/complex-workflow-definitions.js

~Devin AI workflow automation

import { TestRunner } from './test-runner.js';
import { readFileSync } from 'fs';
import { join, dirname } from 'path';
import { fileURLToPath } from 'url';

const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);

/**
 * Complex workflow test definitions
 * This class contains the actual test implementations
 */
export class ComplexWorkflowTests {
  constructor() {
    this.testRunner = new TestRunner();
    this.workflowsPath = join(__dirname, 'workflows');
  }

  loadWorkflow(filename) {
    try {
      const workflowPath = join(this.workflowsPath, filename);
      return readFileSync(workflowPath, 'utf8');
    } catch (error) {
      console.warn(`⚠️ Could not load workflow file ${filename}: ${error.message}`);
      return null;
    }
  }

  extractWorkflowFromMarkdown(markdownContent) {
    if (!markdownContent) return null;
    
    // Extract step sections from markdown
    const stepMatches = markdownContent.match(/### Step \d+:.*?\n```markdown\n([\s\S]*?)\n```/g);
    
    if (stepMatches && stepMatches.length > 0) {
      const extractedSteps = stepMatches.map(match => {
        const stepContent = match.match(/```markdown\n([\s\S]*?)\n```/);
        return stepContent ? stepContent[1].trim() : '';
      }).filter(step => step.length > 0);
      
      return extractedSteps.join('\n\n');
    }
    
    // Fallback: look for workflow in code blocks
    const workflowMatch = markdownContent.match(/```markdown\n([\s\S]*?)\n```/);
    if (workflowMatch) {
      return workflowMatch[1];
    }
    
    return null;
  }

  async runComplexWorkflowSuite() {
    console.log('🏗️ Starting Complex Workflow Test Suite\n');
    
    const complexTests = [
      this.testECommerceWorkflow,
      this.testMicroservicesWorkflow,
      this.testDevOpsWorkflow
    ];

    const results = [];
    for (const test of complexTests) {
      try {
        const result = await test.call(this);
        results.push({ test: test.name, status: 'PASS', result });
      } catch (error) {
        results.push({ test: test.name, status: 'FAIL', error: error.message });
        console.error(`❌ ${test.name} failed: ${error.message}`);
      }
    }

    this.printComplexTestSummary(results);
    return results;
  }

  async testECommerceWorkflow() {
    console.log('🛒 Testing E-Commerce Platform Development Workflow (10 steps)...');
    
    // Configure faster polling for testing
    this.testRunner.handoffManager.setPollingInterval(1500);
    
    // Try to load from markdown file first
    const markdownContent = this.loadWorkflow('ecommerce-platform-workflow.md');
    let ecommerceWorkflow = null;
    
    if (markdownContent) {
      ecommerceWorkflow = this.extractWorkflowFromMarkdown(markdownContent);
      console.log(`   📄 Loaded workflow from: ecommerce-platform-workflow.md`);
    }
    
    // Fallback to hardcoded workflow if markdown extraction fails
    if (!ecommerceWorkflow) {
      console.log(`   📝 Using fallback hardcoded workflow`);
      ecommerceWorkflow = `## Step 1 ##
- Playbook: requirements-analysis
- Prompt: Analyze requirements for a modern e-commerce platform with user management, product catalog, shopping cart, and payment processing
- Handoff: Provide detailed requirements document with technical specifications

## Step 2 ##
- Playbook: architecture-design
- RelyPreviousStep: yes
- Prompt: Design the system architecture including microservices, database design, and API structure
- Handoff: Deliver complete architecture documentation with diagrams

## Step 3 ##
- Playbook: database-development
- RelyPreviousStep: yes
- Prompt: Implement the database schema with user tables, product catalog, orders, and payment records
- Handoff: Provide database migration scripts and seed data

## Step 4 ##
- Playbook: auth-service
- RelyPreviousStep: yes
- Prompt: Build authentication and authorization service with JWT tokens and role-based access
- Handoff: Deliver working authentication API with middleware

## Step 5 ##
- Playbook: product-service
- RelyPreviousStep: no
- Prompt: Create product catalog service with search, filtering, and inventory management
- Handoff: Provide product management API with admin interface

## Step 6 ##
- Playbook: cart-service
- RelyPreviousStep: yes
- Prompt: Implement shopping cart service with session management and cart persistence
- Handoff: Deliver cart API with real-time updates

## Step 7 ##
- Playbook: payment-service
- RelyPreviousStep: no
- Prompt: Integrate payment processing with Stripe, including webhooks and transaction handling
- Handoff: Deliver secure payment API with fraud detection

## Step 8 ##
- Playbook: frontend-app
- RelyPreviousStep: yes
- Prompt: Build React frontend with product browsing, cart management, and checkout flow
- Handoff: Deliver complete e-commerce web application

## Step 9 ##
- Playbook: testing-qa
- RelyPreviousStep: yes
- Prompt: Create comprehensive test suite including unit, integration, and e2e tests
- Handoff: Provide test automation with CI/CD pipeline

## Step 10 ##
- Playbook: deployment-monitoring
- RelyPreviousStep: yes
- Prompt: Deploy to production with monitoring, logging, and performance tracking
- Handoff: Deliver production-ready platform with monitoring dashboard`;
    }
    
    const results = await this.testRunner.handoffManager.executeWorkflow(
      this.testRunner.parser.parse(ecommerceWorkflow)
    );

    // Validate complex workflow execution
    this.validateComplexWorkflow(results, 10, [1, 5, 7]); // Steps 1, 5, 7 are independent
    
    console.log(`✅ E-Commerce workflow completed with ${results.length} steps`);
    return results;
  }

  async testMicroservicesWorkflow() {
    console.log('🔧 Testing Microservices Architecture Workflow (8 steps)...');
    
    // Configure faster polling for testing
    this.testRunner.handoffManager.setPollingInterval(1500);
    
    // Try to load from markdown file first
    const markdownContent = this.loadWorkflow('microservices-architecture-workflow.md');
    let microservicesWorkflow = null;
    
    if (markdownContent) {
      microservicesWorkflow = this.extractWorkflowFromMarkdown(markdownContent);
      console.log(`   📄 Loaded workflow from: microservices-architecture-workflow.md`);
    }
    
    // Fallback to hardcoded workflow if markdown extraction fails
    if (!microservicesWorkflow) {
      console.log(`   📝 Using fallback hardcoded workflow`);
      microservicesWorkflow = `## Step 1 ##
- Playbook: service-discovery
- Prompt: Set up service discovery and API gateway for microservices architecture
- Handoff: Provide configured service mesh with load balancing

## Step 2 ##
- Playbook: user-service
- RelyPreviousStep: yes
- Prompt: Create user management microservice with CRUD operations and authentication
- Handoff: Deliver containerized user service with API documentation

## Step 3 ##
- Playbook: order-service
- RelyPreviousStep: no
- Prompt: Build order processing microservice with event-driven architecture
- Handoff: Provide order service with message queue integration

## Step 4 ##
- Playbook: notification-service
- RelyPreviousStep: yes
- Prompt: Implement notification service for email, SMS, and push notifications
- Handoff: Deliver notification service with template management

## Step 5 ##
- Playbook: data-pipeline
- RelyPreviousStep: no
- Prompt: Create data processing pipeline for analytics and reporting
- Handoff: Provide ETL pipeline with data warehouse integration

## Step 6 ##
- Playbook: security-service
- RelyPreviousStep: yes
- Prompt: Implement centralized security service with OAuth2 and rate limiting
- Handoff: Deliver security middleware for all microservices

## Step 7 ##
- Playbook: monitoring-observability
- RelyPreviousStep: yes
- Prompt: Set up distributed tracing, metrics collection, and log aggregation
- Handoff: Provide complete observability stack with dashboards

## Step 8 ##
- Playbook: orchestration
- RelyPreviousStep: yes
- Prompt: Deploy microservices using Kubernetes with auto-scaling and health checks
- Handoff: Deliver production-ready microservices platform`;
    }

    const results = await this.testRunner.handoffManager.executeWorkflow(
      this.testRunner.parser.parse(microservicesWorkflow)
    );

    this.validateComplexWorkflow(results, 8, [1, 3, 5]); // Steps 1, 3, 5 are independent
    
    console.log(`✅ Microservices workflow completed with ${results.length} steps`);
    return results;
  }

  async testDevOpsWorkflow() {
    console.log('🚀 Testing DevOps CI/CD Pipeline Workflow (6 steps)...');
    
    // Configure faster polling for testing
    this.testRunner.handoffManager.setPollingInterval(1500);
    
    // Try to load from markdown file first
    const markdownContent = this.loadWorkflow('devops-cicd-pipeline-workflow.md');
    let devopsWorkflow = null;
    
    if (markdownContent) {
      devopsWorkflow = this.extractWorkflowFromMarkdown(markdownContent);
      console.log(`   📄 Loaded workflow from: devops-cicd-pipeline-workflow.md`);
    }
    
    // Fallback to hardcoded workflow if markdown extraction fails
    if (!devopsWorkflow) {
      console.log(`   📝 Using fallback hardcoded workflow`);
      devopsWorkflow = `## Step 1 ##
- Playbook: infrastructure-setup
- Prompt: Set up cloud infrastructure with Terraform for development, staging, and production environments
- Handoff: Provide infrastructure as code with environment configurations

## Step 2 ##
- Playbook: ci-pipeline
- RelyPreviousStep: yes
- Prompt: Create CI pipeline with automated testing, code quality checks, and security scanning
- Handoff: Deliver GitHub Actions workflow with quality gates

## Step 3 ##
- Playbook: containerization
- RelyPreviousStep: no
- Prompt: Containerize applications with Docker and create optimized images
- Handoff: Provide Docker images with multi-stage builds and security scanning

## Step 4 ##
- Playbook: cd-pipeline
- RelyPreviousStep: yes
- Prompt: Implement CD pipeline with automated deployment to staging and production
- Handoff: Deliver automated deployment pipeline with rollback capabilities

## Step 5 ##
- Playbook: monitoring-alerting
- RelyPreviousStep: yes
- Prompt: Set up comprehensive monitoring, alerting, and incident response
- Handoff: Provide monitoring stack with SLA dashboards and alerting rules

## Step 6 ##
- Playbook: backup-disaster-recovery
- RelyPreviousStep: yes
- Prompt: Implement backup strategies and disaster recovery procedures
- Handoff: Deliver backup automation and disaster recovery playbook`;
    }

    const results = await this.testRunner.handoffManager.executeWorkflow(
      this.testRunner.parser.parse(devopsWorkflow)
    );

    this.validateComplexWorkflow(results, 6, [1, 3]); // Steps 1, 3 are independent
    
    console.log(`✅ DevOps workflow completed with ${results.length} steps`);
    return results;
  }

  validateComplexWorkflow(results, expectedSteps, independentStepNumbers) {
    // Basic validation
    if (results.length !== expectedSteps) {
      throw new Error(`Expected ${expectedSteps} steps, got ${results.length}`);
    }

    if (!results.every(r => r.success)) {
      throw new Error('Not all steps succeeded');
    }

    // Validate independent steps
    const independentSteps = results.filter(r => !r.relied_on_previous);
    if (independentSteps.length !== independentStepNumbers.length) {
      throw new Error(`Expected ${independentStepNumbers.length} independent steps, got ${independentSteps.length}`);
    }

    // Validate timing progression
    for (let i = 1; i < results.length; i++) {
      if (results[i].total_elapsed_time_ms <= results[i-1].total_elapsed_time_ms) {
        throw new Error(`Step ${i+1} total elapsed time should be greater than step ${i}`);
      }
    }

    // Validate unique session IDs
    const sessionIds = results.map(r => r.session_id);
    const uniqueSessionIds = [...new Set(sessionIds)];
    if (uniqueSessionIds.length !== results.length) {
      throw new Error('Not all steps have unique session IDs');
    }
  }

  printComplexTestSummary(results) {
    console.log('\n🎯 Complex Workflow Test Summary:');
    console.log('=================================');
    
    const passed = results.filter(r => r.status === 'PASS').length;
    const failed = results.filter(r => r.status === 'FAIL').length;
    
    console.log(`Total Complex Tests: ${results.length}`);
    console.log(`Passed: ${passed}`);
    console.log(`Failed: ${failed}`);
    
    // Calculate total steps executed
    const totalSteps = results
      .filter(r => r.status === 'PASS')
      .reduce((sum, r) => sum + (r.result ? r.result.length : 0), 0);
    
    console.log(`Total Steps Executed: ${totalSteps}`);
    
    if (failed === 0) {
      console.log('\n🎉 All complex workflow tests passed!');
      console.log(`📊 Successfully executed ${totalSteps} workflow steps across ${passed} complex scenarios`);
    } else {
      console.log('\n💥 Some complex tests failed:');
      results
        .filter(r => r.status === 'FAIL')
        .forEach(r => console.log(`   - ${r.test}: ${r.error}`));
    }
  }
}