agentic-data-stack-community/tools/lib/task-orchestrator.js

AI Agentic Data Stack Framework - Community Edition. Open source data engineering framework with 4 core agents, essential templates, and 3-dimensional quality validation.

/**
 * Task Orchestrator - Executable Task Management System
 * 
 * Manages task discovery, execution, progress tracking, and validation.
 * Implements interactive task execution with user collaboration.
 */

const chalk = require('chalk');
const inquirer = require('inquirer');
const fs = require('fs-extra');
const path = require('path');

class TaskOrchestrator {
  constructor(options = {}) {
    this.rootDir = options.rootDir || process.cwd();
    this.dataCore = path.join(this.rootDir, 'data-core');
    this.taskCache = new Map();
    this.executionState = new Map();
  }

  async executeTask(taskName, context = {}) {
    console.log(chalk.blue(`\n📋 Starting task: ${taskName}`));
    
    try {
      const task = await this.loadTask(taskName);
      return await this.executeTaskSteps(task, context);
    } catch (error) {
      console.error(chalk.red(`❌ Task execution failed: ${error.message}`));
      return { success: false, error: error.message };
    }
  }

  async executeTaskContent(content, context = {}) {
    const task = this.parseTaskContent(content);
    return await this.executeTaskSteps(task, context);
  }

  async loadTask(taskName) {
    // Check cache first
    if (this.taskCache.has(taskName)) {
      return this.taskCache.get(taskName);
    }

    const taskPath = path.join(this.dataCore, 'tasks', `${taskName}.md`);
    
    if (!await fs.pathExists(taskPath)) {
      throw new Error(`Task file not found: ${taskName}.md`);
    }

    const content = await fs.readFile(taskPath, 'utf8');
    const task = this.parseTaskContent(content);
    
    // Cache the task
    this.taskCache.set(taskName, task);
    
    return task;
  }

  parseTaskContent(content) {
    const lines = content.split('\n');
    const task = {
      title: this.extractTitle(content),
      overview: this.extractOverview(content),
      prerequisites: this.extractPrerequisites(content),
      dependencies: this.extractDependencies(content),
      steps: this.extractSteps(content),
      outputs: this.extractOutputs(content),
      successCriteria: this.extractSuccessCriteria(content),
      validationRequirements: this.extractValidationRequirements(content)
    };

    return task;
  }

  extractTitle(content) {
    const match = content.match(/^#\s+(.+)/m);
    return match ? match[1] : 'Untitled Task';
  }

  extractOverview(content) {
    const match = content.match(/## Overview\n(.*?)(?=\n##|\n$)/s);
    return match ? match[1].trim() : '';
  }

  extractPrerequisites(content) {
    const match = content.match(/## Prerequisites\n(.*?)(?=\n##|\n$)/s);
    if (!match) return [];
    
    return match[1]
      .split('\n')
      .filter(line => line.trim().startsWith('-'))
      .map(line => line.replace(/^-\s*/, '').trim());
  }

  extractDependencies(content) {
    const match = content.match(/## Dependencies\n(.*?)(?=\n##|\n$)/s);
    if (!match) return {};
    
    const deps = {};
    const lines = match[1].split('\n');
    
    for (const line of lines) {
      const depMatch = line.match(/- ([\w-]+):\s*`?([^`]+)`?/);
      if (depMatch) {
        const [, type, items] = depMatch;
        deps[type] = items.split(',').map(item => item.trim());
      }
    }
    
    return deps;
  }

  extractSteps(content) {
    const stepsMatch = content.match(/## Steps\n(.*?)(?=\n##[^#]|\n$)/s);
    if (!stepsMatch) return [];

    const steps = [];
    const stepRegex = /###\s+(\d+)\.\s+\*\*(.+?)\*\*\n(.*?)(?=###|\n##|\n$)/gs;
    let match;

    while ((match = stepRegex.exec(stepsMatch[1])) !== null) {
      const [, number, title, content] = match;
      steps.push({
        number: parseInt(number),
        title: title.trim(),
        content: content.trim(),
        validation: this.extractStepValidation(content),
        qualityCheck: this.extractStepQualityCheck(content)
      });
    }

    return steps;
  }

  extractStepValidation(stepContent) {
    const match = stepContent.match(/\*\*Validation\*\*:\s*(.+)/);
    return match ? match[1].trim() : null;
  }

  extractStepQualityCheck(stepContent) {
    const match = stepContent.match(/\*\*Quality Check\*\*:\s*(.+)/);
    return match ? match[1].trim() : null;
  }

  extractOutputs(content) {
    const match = content.match(/## Outputs\n(.*?)(?=\n##|\n$)/s);
    if (!match) return {};
    
    const outputs = {};
    const sections = match[1].split(/###\s+(.+)/);
    
    for (let i = 1; i < sections.length; i += 2) {
      const sectionTitle = sections[i].trim();
      const sectionContent = sections[i + 1] ? sections[i + 1].trim() : '';
      outputs[sectionTitle] = sectionContent;
    }
    
    return outputs;
  }

  extractSuccessCriteria(content) {
    const match = content.match(/## Success Criteria\n(.*?)(?=\n##|\n$)/s);
    if (!match) return {};
    
    const criteria = {};
    const sections = match[1].split(/###\s+(.+)/);
    
    for (let i = 1; i < sections.length; i += 2) {
      const sectionTitle = sections[i].trim();
      const items = sections[i + 1] ? sections[i + 1]
        .split('\n')
        .filter(line => line.trim().startsWith('-'))
        .map(line => line.replace(/^-\s*/, '').trim()) : [];
      criteria[sectionTitle] = items;
    }
    
    return criteria;
  }

  extractValidationRequirements(content) {
    const match = content.match(/### Validation Requirements\n(.*?)(?=\n###|\n##|\n$)/s);
    if (!match) return [];
    
    return match[1]
      .split('\n')
      .filter(line => line.trim().startsWith('- ['))
      .map(line => ({
        text: line.replace(/^-\s*\[\s*\]\s*/, '').trim(),
        completed: false
      }));
  }

  async executeTaskSteps(task, context = {}) {
    console.log(chalk.bold.blue(`\n🎯 ${task.title}`));
    console.log(chalk.dim(task.overview));
    console.log('');

    // Check prerequisites
    if (task.prerequisites.length > 0) {
      console.log(chalk.bold('📋 Prerequisites:'));
      task.prerequisites.forEach(prereq => {
        console.log(`  • ${prereq}`);
      });
      
      const proceedPrereq = await inquirer.prompt([{
        type: 'confirm',
        name: 'continue',
        message: 'All prerequisites met?',
        default: true
      }]);
      
      if (!proceedPrereq.continue) {
        return { success: false, reason: 'Prerequisites not met' };
      }
      console.log('');
    }

    // Execute steps
    const results = [];
    for (let i = 0; i < task.steps.length; i++) {
      const step = task.steps[i];
      const result = await this.executeStep(step, i + 1, task.steps.length, context);
      results.push(result);
      
      if (!result.success) {
        return { 
          success: false, 
          error: `Step ${step.number} failed`, 
          results 
        };
      }
    }

    // Validate outputs
    const validationResult = await this.validateTaskCompletion(task, results, context);
    
    if (validationResult.success) {
      console.log(chalk.green.bold('\n🎉 Task completed successfully!'));
      
      if (task.outputs && Object.keys(task.outputs).length > 0) {
        console.log(chalk.bold('\n📦 Outputs:'));
        Object.entries(task.outputs).forEach(([key, value]) => {
          console.log(`  ${key}: ${value.split('\n')[0]}...`);
        });
      }
    }

    return {
      success: validationResult.success,
      results,
      validation: validationResult,
      outputs: task.outputs
    };
  }

  async executeStep(step, current, total, context = {}) {
    console.log(chalk.bold(`\n📍 Step ${current}/${total}: ${step.title}`));
    console.log(chalk.dim(step.content.split('\n')[0]));
    
    // Show detailed content if requested
    const showDetails = await inquirer.prompt([{
      type: 'confirm',
      name: 'details',
      message: 'Show detailed instructions?',
      default: false
    }]);
    
    if (showDetails.details) {
      console.log(chalk.dim('\nDetailed Instructions:'));
      console.log(chalk.dim(step.content));
      console.log('');
    }

    // Execute step
    const executeChoice = await inquirer.prompt([{
      type: 'list',
      name: 'action',
      message: 'How would you like to proceed?',
      choices: [
        { name: '✅ Execute step and continue', value: 'execute' },
        { name: '⏭️ Skip this step', value: 'skip' },
        { name: '🔄 Get help with this step', value: 'help' },
        { name: '⏸️ Pause execution', value: 'pause' }
      ]
    }]);

    switch (executeChoice.action) {
      case 'execute':
        return await this.processStepExecution(step, context);
      case 'skip':
        console.log(chalk.yellow('⏭️ Step skipped'));
        return { success: true, skipped: true };
      case 'help':
        await this.showStepHelp(step);
        return await this.executeStep(step, current, total, context);
      case 'pause':
        return { success: false, paused: true };
      default:
        return { success: false, error: 'Unknown action' };
    }
  }

  async processStepExecution(step, context = {}) {
    console.log(chalk.blue('🔄 Processing step...'));
    
    // Simulate step execution
    await new Promise(resolve => setTimeout(resolve, 1000));
    
    // Check for validation
    if (step.validation) {
      console.log(chalk.blue(`🔍 Validation: ${step.validation}`));
      
      const validationPassed = await inquirer.prompt([{
        type: 'confirm',
        name: 'passed',
        message: 'Did validation pass?',
        default: true
      }]);
      
      if (!validationPassed.passed) {
        console.log(chalk.red('❌ Validation failed'));
        return { success: false, error: 'Validation failed' };
      }
    }

    // Check for quality check
    if (step.qualityCheck) {
      console.log(chalk.blue(`✅ Quality Check: ${step.qualityCheck}`));
      
      const qualityPassed = await inquirer.prompt([{
        type: 'confirm',
        name: 'passed',
        message: 'Did quality check pass?',
        default: true
      }]);
      
      if (!qualityPassed.passed) {
        console.log(chalk.red('❌ Quality check failed'));
        return { success: false, error: 'Quality check failed' };
      }
    }

    console.log(chalk.green('✅ Step completed'));
    return { success: true };
  }

  async showStepHelp(step) {
    console.log(chalk.bold.blue(`\n📚 Help for: ${step.title}`));
    console.log(chalk.dim('Full Instructions:'));
    console.log(step.content);
    
    if (step.validation) {
      console.log(chalk.bold('\nValidation Criteria:'));
      console.log(step.validation);
    }
    
    if (step.qualityCheck) {
      console.log(chalk.bold('\nQuality Check:'));
      console.log(step.qualityCheck);
    }
    
    await inquirer.prompt([{
      type: 'input',
      name: 'continue',
      message: 'Press Enter to continue...'
    }]);
  }

  async validateTaskCompletion(task, results, context = {}) {
    console.log(chalk.blue('\n🔍 Validating task completion...'));
    
    // Check validation requirements
    if (task.validationRequirements && task.validationRequirements.length > 0) {
      console.log(chalk.bold('Validation Requirements:'));
      
      for (const req of task.validationRequirements) {
        const passed = await inquirer.prompt([{
          type: 'confirm',
          name: 'met',
          message: `✓ ${req.text}`,
          default: true
        }]);
        
        req.completed = passed.met;
        
        if (!passed.met) {
          console.log(chalk.red(`❌ Requirement not met: ${req.text}`));
          return { success: false, error: 'Validation requirements not met' };
        }
      }
    }

    // Check success criteria
    if (task.successCriteria && Object.keys(task.successCriteria).length > 0) {
      console.log(chalk.bold('\nSuccess Criteria:'));
      
      for (const [category, criteria] of Object.entries(task.successCriteria)) {
        console.log(chalk.bold(`  ${category}:`));
        
        for (const criterion of criteria) {
          const met = await inquirer.prompt([{
            type: 'confirm',
            name: 'satisfied',
            message: `    • ${criterion}`,
            default: true
          }]);
          
          if (!met.satisfied) {
            console.log(chalk.red(`❌ Success criterion not met: ${criterion}`));
            return { success: false, error: 'Success criteria not met' };
          }
        }
      }
    }

    console.log(chalk.green('✅ All validations passed'));
    return { success: true };
  }

  async listTasks() {
    const taskDir = path.join(this.dataCore, 'tasks');
    
    if (!await fs.pathExists(taskDir)) {
      return [];
    }

    const files = await fs.readdir(taskDir);
    const tasks = [];

    for (const file of files) {
      if (file.endsWith('.md')) {
        const taskPath = path.join(taskDir, file);
        const content = await fs.readFile(taskPath, 'utf8');
        
        tasks.push({
          name: file.replace('.md', ''),
          title: this.extractTitle(content),
          description: this.extractOverview(content).split('.')[0] + '.',
          file
        });
      }
    }

    return tasks;
  }

  getExecutionState(taskName) {
    return this.executionState.get(taskName);
  }

  setExecutionState(taskName, state) {
    this.executionState.set(taskName, state);
  }

  clearExecutionState(taskName) {
    this.executionState.delete(taskName);
  }
}

module.exports = TaskOrchestrator;