devin-workflow/test/test-execute-workflow.js

~Devin AI workflow automation

#!/usr/bin/env node

import { startWorkflow, startWorkflowMock, startWorkflowQuiet, validateWorkflow } from '../src/workflow-executor.js';
import { readFileSync } from 'fs';
import { join, dirname } from 'path';
import { fileURLToPath } from 'url';
import { spawn } from 'child_process';

const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);

class ExecuteWorkflowUnitTest {
  constructor() {
    this.mockServer = null;
    this.testResults = [];
  }

  async startMockServer() {
    console.log('🔧 Starting Mock Server for executeWorkflow tests...');
    
    return new Promise((resolve, reject) => {
      const mockServerPath = join(__dirname, 'mock-devin-api.js');
      this.mockServer = spawn('node', [mockServerPath], {
        stdio: ['inherit', 'pipe', 'pipe']
      });

      this.mockServer.stdout.on('data', (data) => {
        const output = data.toString();
        if (output.includes('Mock Devin API running')) {
          resolve();
        }
      });

      this.mockServer.stderr.on('data', (data) => {
        console.error(`Mock API Error: ${data}`);
      });

      this.mockServer.on('error', (error) => {
        reject(error);
      });

      setTimeout(() => resolve(), 2000);
    });
  }

  async stopMockServer() {
    if (this.mockServer) {
      this.mockServer.kill();
    }
  }

  loadWorkflow(filename) {
    const workflowPath = join(__dirname, 'workflows', filename);
    return readFileSync(workflowPath, 'utf8');
  }

  extractWorkflowFromMarkdown(markdownContent) {
    // Use the real parser for modern markdown format
    const { WorkflowParser } = require('../src/workflow-parser.js');
    const parser = new WorkflowParser();
    return parser.parse(markdownContent);
  }

  async runAllTests() {
    console.log('🧪 Starting executeWorkflow Unit Tests');
    console.log('=====================================\n');

    try {
      await this.startMockServer();
      await this.sleep(2000); // Wait for mock server

      const tests = [
        this.testBasicExecuteWorkflow,
        this.testExecuteWorkflowWithECommerceMarkdown,
        this.testExecuteWorkflowMock,
        this.testExecuteWorkflowQuiet,
        this.testValidateWorkflow,
        this.testExecuteWorkflowWithCustomOptions,
        this.testExecuteWorkflowWithInvalidWorkflow,
        this.testExecuteWorkflowWithEnvironmentVariables
      ];

      for (const test of tests) {
        try {
          await test.call(this);
        } catch (error) {
          this.logError(test.name, error);
        }
      }

      this.printSummary();

    } finally {
      await this.stopMockServer();
    }
  }

  async testBasicExecuteWorkflow() {
    console.log('🔍 Testing basic executeWorkflow functionality...');

    const simpleWorkflow = `## Step 1: Create Feature Branch\n- Prompt: Test basic workflow execution\n- Handoff: Provide test results\n\n## Step 2: Create and Commit Test Document\n- RelyPreviousStep: yes\n- Prompt: Generate summary\n- Handoff: Create final report`;

    // Use mock mode for testing
    const result = await startWorkflowMock(simpleWorkflow);

    this.assert(result.success === true, 'Basic workflow should succeed');
    this.assert(result.total_steps === 2, 'Should execute 2 steps');
    this.assert(result.successful_steps === 2, 'Both steps should succeed');
    this.assert(result.failed_steps === 0, 'No steps should fail');
    this.assert(result.completion_rate === 100, 'Completion rate should be 100%');
    this.assert(result.session_ids.length === 2, 'Should generate 2 session IDs');
    this.assert(result.total_execution_time_ms > 0, 'Should have execution time');
    this.assert(result.workflow_config.mock_mode === true, 'Should use mock mode');

    this.logSuccess('testBasicExecuteWorkflow', 'Basic workflow execution works correctly');
  }

  async testExecuteWorkflowWithECommerceMarkdown() {
    console.log('🛒 Testing executeWorkflow with E-Commerce markdown file...');

    const markdownContent = this.loadWorkflow('ecommerce-platform-workflow.md');
    let workflowObj = this.extractWorkflowFromMarkdown(markdownContent);
    let workflow = workflowObj.steps;
    // Always use mock mode for this test
    const result = await startWorkflowMock(
      workflow.map(step => {
        // Reconstruct markdown for each step for mock
        let md = `## Step ${step.step_number}: ${step.title || ''}`.trim() + '\n';
        if (step.playbook) md += `- Playbook: ${step.playbook}\n`;
        if (step.rely_previous_step !== undefined) md += `- RelyPreviousStep: ${step.rely_previous_step ? 'yes' : 'no'}\n`;
        if (step.prompt) md += `- Prompt: ${step.prompt.split('\n')[0]}\n`;
        if (step.handoff) md += `- Handoff: ${step.handoff.split('\n')[0]}\n`;
        return md;
      }).join('\n\n'), {
      pollingInterval: 1, // Fast for testing
      timeout: 30
    });

    this.assert(result.success === true, 'E-commerce workflow should succeed');
    this.assert(result.total_steps === 3, 'Should execute 3 steps');
    this.assert(result.step_results.length === 3, 'Should have 3 step results');
    this.assert(result.step_results[0].relied_on_previous === false, 'First step should not rely on previous');
    this.assert(result.step_results[1].relied_on_previous === true, 'Second step should rely on previous');
    this.assert(result.step_results[2].relied_on_previous === true, 'Third step should rely on previous');
    
    // Validate session IDs are unique
    const uniqueSessionIds = [...new Set(result.session_ids)];
    this.assert(uniqueSessionIds.length === 3, 'All session IDs should be unique');

    this.logSuccess('testExecuteWorkflowWithECommerceMarkdown', 'E-commerce workflow from markdown executed successfully');
  }

  async testExecuteWorkflowMock() {
    console.log('🎭 Testing executeWorkflowMock function...');

    const workflow = `## Step 1 ##
- Prompt: Test mock execution
- Handoff: Provide mock results`;

    const result = await startWorkflowMock(workflow);

    this.assert(result.success === true, 'Mock workflow should succeed');
    this.assert(result.workflow_config.mock_mode === true, 'Should be in mock mode');
    this.assert(result.workflow_config.polling_interval === 2, 'Should use fast polling (2s)');
    this.assert(result.workflow_config.timeout === 60, 'Should use short timeout (60s)');

    this.logSuccess('testExecuteWorkflowMock', 'Mock workflow execution works correctly');
  }

  async testExecuteWorkflowQuiet() {
    console.log('🤫 Testing executeWorkflowQuiet function...');

    const workflow = `## Step 1 ##
- Prompt: Test quiet execution
- Handoff: Provide results quietly`;

    // Capture console output to verify quiet mode
    const originalLog = console.log;
    let consoleOutput = '';
    console.log = (...args) => {
      consoleOutput += args.join(' ') + '\n';
    };

    const result = await startWorkflowQuiet(workflow, { useMockMode: true });

    console.log = originalLog; // Restore console.log

    this.assert(result.success === true, 'Quiet workflow should succeed');
    this.assert(!consoleOutput.includes('🚀 Starting One-Click'), 'Should not show verbose output');

    this.logSuccess('testExecuteWorkflowQuiet', 'Quiet workflow execution works correctly');
  }

  async testValidateWorkflow() {
    console.log('✅ Testing validateWorkflow function...');

    const validWorkflow = `## Step 1 ##
- Prompt: Valid workflow step
- Handoff: Provide validation results`;

    const invalidWorkflow = `## Step 1 ##
- Handoff: Missing prompt field`;

    const validResult = validateWorkflow(validWorkflow);
    const invalidResult = validateWorkflow(invalidWorkflow);

    this.assert(validResult.valid === true, 'Valid workflow should pass validation');
    this.assert(validResult.steps.length === 1, 'Should parse 1 step');
    this.assert(validResult.summary !== null, 'Should provide summary');

    this.assert(invalidResult.valid === false, 'Invalid workflow should fail validation');
    this.assert(invalidResult.errors.length > 0, 'Should have validation errors');

    this.logSuccess('testValidateWorkflow', 'Workflow validation works correctly');
  }

  async testExecuteWorkflowWithCustomOptions() {
    console.log('⚙️ Testing executeWorkflow with custom options...');

    const workflow = `## Step 1 ##
- Prompt: Test custom options
- Handoff: Provide customized results`;

    const result = await startWorkflow(workflow, {
      useMockMode: true,
      apiKey: 'custom-test-key',
      pollingInterval: 3,
      timeout: 120,
      verbose: false
    });

    this.assert(result.success === true, 'Custom options workflow should succeed');
    this.assert(result.workflow_config.mock_mode === true, 'Should respect mock mode option');
    this.assert(result.workflow_config.polling_interval === 3, 'Should use custom polling interval');
    this.assert(result.workflow_config.timeout === 120, 'Should use custom timeout');

    this.logSuccess('testExecuteWorkflowWithCustomOptions', 'Custom options work correctly');
  }

  async testExecuteWorkflowWithInvalidWorkflow() {
    console.log('❌ Testing executeWorkflow with invalid workflow...');

    const invalidWorkflow = `## Step 1 ##
- Handoff: Missing required prompt field`;

    const result = await startWorkflowMock(invalidWorkflow);

    this.assert(result.success === false, 'Invalid workflow should fail');
    this.assert(result.error.includes('validation failed'), 'Should have validation error');
    this.assert(result.total_execution_time_ms > 0, 'Should track execution time even on failure');

    this.logSuccess('testExecuteWorkflowWithInvalidWorkflow', 'Invalid workflow handling works correctly');
  }

  async testExecuteWorkflowWithEnvironmentVariables() {
    console.log('🌍 Testing executeWorkflow with environment variables...');

    // Temporarily set environment variables
    const originalNodeEnv = process.env.NODE_ENV;
    const originalDevinApiKey = process.env.DEVIN_API_KEY;

    process.env.NODE_ENV = 'test';
    delete process.env.DEVIN_API_KEY;

    const workflow = `## Step 1 ##
- Prompt: Test environment detection
- Handoff: Provide environment results`;

    const result = await startWorkflow(workflow);

    // Restore environment variables
    process.env.NODE_ENV = originalNodeEnv;
    if (originalDevinApiKey) {
      process.env.DEVIN_API_KEY = originalDevinApiKey;
    }

    this.assert(result.success === true, 'Environment-based workflow should succeed');
    this.assert(result.workflow_config.mock_mode === true, 'Should auto-enable mock mode when no API key');

    this.logSuccess('testExecuteWorkflowWithEnvironmentVariables', 'Environment variable detection works correctly');
  }

  assert(condition, message) {
    if (!condition) {
      throw new Error(`Assertion failed: ${message}`);
    }
  }

  logSuccess(testName, message) {
    console.log(`   ✅ ${testName}: ${message}`);
    this.testResults.push({ test: testName, status: 'PASS', message });
  }

  logError(testName, error) {
    console.log(`   ❌ ${testName}: ${error.message}`);
    this.testResults.push({ test: testName, status: 'FAIL', message: error.message });
  }

  printSummary() {
    console.log('\n📊 executeWorkflow Unit Test Summary:');
    console.log('====================================');
    
    const passed = this.testResults.filter(r => r.status === 'PASS').length;
    const failed = this.testResults.filter(r => r.status === 'FAIL').length;
    
    console.log(`Total Tests: ${this.testResults.length}`);
    console.log(`Passed: ${passed}`);
    console.log(`Failed: ${failed}`);
    
    if (failed === 0) {
      console.log('\n🎉 All executeWorkflow unit tests passed!');
    } else {
      console.log('\n💥 Some tests failed:');
      this.testResults
        .filter(r => r.status === 'FAIL')
        .forEach(r => console.log(`   - ${r.test}: ${r.message}`));
    }

    console.log(`\n📈 Success Rate: ${passed}/${this.testResults.length} (${((passed / this.testResults.length) * 100).toFixed(1)}%)`);
  }

  sleep(ms) {
    return new Promise(resolve => setTimeout(resolve, ms));
  }
}

// Run tests if executed directly
if (import.meta.url === `file://${process.argv[1]}`) {
  const tester = new ExecuteWorkflowUnitTest();
  
  process.on('SIGINT', async () => {
    console.log('\n🛑 Stopping tests...');
    await tester.stopMockServer();
    process.exit(0);
  });

  // Check for command line arguments to run specific tests
  const args = process.argv.slice(2);
  const testName = args[0];

  if (testName) {
    // Run specific test
    console.log(`🎯 Running specific test: ${testName}`);
    console.log('=======================================\n');
    
    try {
      await tester.startMockServer();
      await tester.sleep(2000);
      
      const testMethods = {
        'basic': tester.testBasicExecuteWorkflow,
        'ecommerce': tester.testExecuteWorkflowWithECommerceMarkdown,
        'mock': tester.testExecuteWorkflowMock,
        'quiet': tester.testExecuteWorkflowQuiet,
        'validate': tester.testValidateWorkflow,
        'options': tester.testExecuteWorkflowWithCustomOptions,
        'invalid': tester.testExecuteWorkflowWithInvalidWorkflow,
        'env': tester.testExecuteWorkflowWithEnvironmentVariables
      };

      if (testMethods[testName]) {
        try {
          await testMethods[testName].call(tester);
        } catch (error) {
          tester.logError(testMethods[testName].name, error);
        }
        tester.printSummary();
      } else {
        console.log('❌ Unknown test name. Available tests:');
        console.log('  basic      - Basic executeWorkflow functionality');
        console.log('  ecommerce  - E-Commerce markdown workflow');
        console.log('  mock       - Mock mode testing');
        console.log('  quiet      - Quiet mode testing');
        console.log('  validate   - Workflow validation');
        console.log('  options    - Custom options testing');
        console.log('  invalid    - Invalid workflow handling');
        console.log('  env        - Environment variables testing');
        console.log('\nUsage: node test/test-execute-workflow.js [test-name]');
        console.log('       node test/test-execute-workflow.js (runs all tests)');
      }
    } finally {
      await tester.stopMockServer();
    }
  } else {
    // Run all tests (default behavior)
    tester.runAllTests().catch(console.error);
  }
}

export { ExecuteWorkflowUnitTest };