agentic-data-stack-community/tools/cli.js

AI Agentic Data Stack Framework - Community Edition. Open source data engineering framework with 4 core agents, essential templates, and 3-dimensional quality validation.

#!/usr/bin/env node

/**
 * AI Agentic Data Stack Framework - Community Edition CLI
 * 
 * Simplified command-line interface for the community version
 * providing essential data engineering and analytics capabilities.
 */

const { Command } = require('commander');
const chalk = require('chalk');
const inquirer = require('inquirer');
const fs = require('fs-extra');
const path = require('path');
const yaml = require('yaml');
const CommandHandler = require('./lib/command-handler');
const AgentOrchestrator = require('./lib/agent-orchestrator');
const TaskOrchestrator = require('./lib/task-orchestrator');
const TemplateEngine = require('./lib/template-engine');

const program = new Command();

// CLI Version and Info
const packageJson = require('../package.json');

// Initialize orchestration system
let commandHandler;
let agentOrchestrator;
let taskOrchestrator;
let templateEngine;

async function initializeSystem() {
  try {
    agentOrchestrator = new AgentOrchestrator({ rootDir: __dirname + '/../' });
    taskOrchestrator = new TaskOrchestrator({ rootDir: __dirname + '/../' });
    templateEngine = new TemplateEngine({ rootDir: __dirname + '/../' });
    
    // Set up dependencies
    agentOrchestrator.setDependencies(taskOrchestrator, templateEngine);
    
    commandHandler = new CommandHandler({ 
      rootDir: __dirname + '/../',
      agentOrchestrator,
      taskOrchestrator,
      templateEngine
    });
    
    await agentOrchestrator.initialize();
    
    return true;
  } catch (error) {
    console.error(chalk.red('❌ Failed to initialize system:'), error.message);
    return false;
  }
}

program
  .name('agentic-data')
  .description('AI Agentic Data Stack Framework - Community Edition')
  .version(packageJson.version);

// =====================================================================
// HELP AND INFO COMMANDS
// =====================================================================

program
  .command('info')
  .description('Display framework information and capabilities')
  .action(() => {
    console.log('');
    console.log(chalk.bold.blue('🚀 AI Agentic Data Stack Framework - Community Edition'));
    console.log('');
    
    console.log(chalk.bold('📊 Included Features:'));
    console.log('  ✅ 4 Core AI Agents (Engineer, Analyst, PM, Quality)');
    console.log('  ✅ 20 Essential Templates for common use cases');
    console.log('  ✅ 3-Dimensional Quality Framework (Completeness, Accuracy, Consistency)');
    console.log('  ✅ RFM Customer Segmentation Analysis');
    console.log('  ✅ Basic Data Contracts and Validation');
    console.log('  ✅ Community Examples and Tutorials');
    console.log('  ✅ CLI Tools for project management');
    console.log('');
    
    console.log(chalk.bold('🏢 Enterprise Edition Adds:'));
    console.log('  • 8 Specialized AI Agents with advanced personas');
    console.log('  • 88 Interactive Templates with progressive disclosure');
    console.log('  • 7-Dimensional Quality Framework with ML enhancement');
    console.log('  • Real-time collaboration and approval workflows');
    console.log('  • Advanced compliance automation (HIPAA, GDPR, SOX)');
    console.log('  • Predictive analytics and business impact measurement');
    console.log('  • Professional support and training');
    console.log('');
    
    console.log(chalk.bold('📞 Contact:'));
    console.log('  Community: https://github.com/barnyp/agentic-data-stack-framework-community');
    console.log('  Enterprise: enterprise@agenticdata.com');
    console.log('');
    
    console.log(chalk.dim('───────────────────────────────────────────────────'));
    console.log(chalk.dim(`Version: ${packageJson.version} | License: MIT | Framework: Community Edition`));
  });

// =====================================================================
// PROJECT MANAGEMENT COMMANDS
// =====================================================================

program
  .command('init')
  .description('Initialize a new data project with framework structure')
  .option('-n, --name <name>', 'Project name')
  .option('-t, --template <template>', 'Template to use (ecommerce, basic, custom)')
  .action(async (options) => {
    console.log('');
    console.log(chalk.bold.blue('🎯 Initializing new AI Agentic Data Stack project...'));
    console.log('');
    
    // Get project details
    const answers = await inquirer.prompt([
      {
        type: 'input',
        name: 'projectName',
        message: 'Project name:',
        default: options.name || 'my-data-project',
        validate: (input) => input.length > 0 || 'Project name is required'
      },
      {
        type: 'list',
        name: 'template',
        message: 'Choose a template:',
        choices: [
          { name: '🛒 E-commerce Analytics (Customer Segmentation)', value: 'ecommerce' },
          { name: '📊 Basic Data Analysis (Generic)', value: 'basic' },
          { name: '🔧 Custom Setup (Empty Structure)', value: 'custom' }
        ],
        default: options.template || 'ecommerce'
      },
      {
        type: 'confirm',
        name: 'includeExample',
        message: 'Include sample data and examples?',
        default: true
      }
    ]);
    
    const projectPath = path.join(process.cwd(), answers.projectName);
    
    // Create project structure
    try {
      await createProjectStructure(projectPath, answers);
      
      console.log('');
      console.log(chalk.green('✅ Project created successfully!'));
      console.log('');
      console.log(`📁 Project location: ${projectPath}`);
      console.log('');
      console.log('🚀 Next steps:');
      console.log(`   cd ${answers.projectName}`);
      console.log('   agentic-data agents list    # View available agents');
      console.log('   agentic-data templates list # View available templates');
      
      if (answers.includeExample) {
        console.log('   # Follow README.md for example implementation');
      }
      
    } catch (error) {
      console.error(chalk.red('❌ Error creating project:'), error.message);
      process.exit(1);
    }
  });

// =====================================================================
// INTERACTIVE AGENT COMMANDS
// =====================================================================

// Interactive agent activation
program
  .command('agent <agentName>')
  .alias('@')
  .description('Activate an interactive AI agent')
  .option('-c, --context <context>', 'Additional context for agent')
  .action(async (agentName, options) => {
    if (!await initializeSystem()) return;
    
    try {
      const context = options.context ? JSON.parse(options.context) : {};
      const result = await commandHandler.activateAgent(agentName, context);
      
      if (result.success) {
        // Enter interactive mode
        await enterInteractiveMode(result.agent);
      }
    } catch (error) {
      console.error(chalk.red('❌ Error:'), error.message);
    }
  });

// Interactive shell command
program
  .command('interactive')
  .alias('shell')
  .description('Enter interactive command mode')
  .action(async () => {
    if (!await initializeSystem()) return;
    await enterInteractiveShell();
  });

// Workflow execution
program
  .command('workflow <workflowName>')
  .description('Execute a workflow')
  .option('-t, --type <type>', 'Workflow type (brownfield/greenfield)', 'greenfield')
  .action(async (workflowName, options) => {
    if (!await initializeSystem()) return;
    
    try {
      const context = { type: options.type };
      await commandHandler.executeWorkflow(workflowName, context);
    } catch (error) {
      console.error(chalk.red('❌ Error:'), error.message);
    }
  });

// Task execution
program
  .command('task <taskName>')
  .description('Execute a specific task')
  .option('-c, --context <context>', 'Task context as JSON')
  .action(async (taskName, options) => {
    if (!await initializeSystem()) return;
    
    try {
      const context = options.context ? JSON.parse(options.context) : {};
      await commandHandler.executeTask(taskName, context);
    } catch (error) {
      console.error(chalk.red('❌ Error:'), error.message);
    }
  });

// =====================================================================
// AGENT COMMANDS
// =====================================================================

// Create agents subcommand
const agentsCmd = program
  .command('agents')
  .description('Manage AI agents');

agentsCmd
  .command('list')
  .alias('ls')
  .description('List available AI agents')
  .action(async () => {
    if (!await initializeSystem()) return;
    
    try {
      const agents = await agentOrchestrator.listAgents();
      
      console.log('');
      console.log(chalk.bold.blue('🤖 Available AI Agents - Community Edition'));
      console.log('');
      
      agents.forEach((agent, index) => {
        console.log(`${index + 1}. ${chalk.bold(agent.name)} ${agent.icon || ''}`);
        console.log(`   📋 ID: ${agent.id}`);
        console.log(`   💡 Role: ${agent.description}`);
        console.log(`   🚀 Activate: ${chalk.cyan(`agentic-data agent ${agent.id}`)}`);
        console.log('');
      });
      
      console.log(chalk.dim('💼 Enterprise Edition includes 4 additional specialized agents:'));
      console.log(chalk.dim('   • Data Scientist (ML models, advanced analytics)'));
      console.log(chalk.dim('   • Data Governance Officer (compliance, policy enforcement)'));
      console.log(chalk.dim('   • Data Experience Designer (visualization, user experience)'));
      console.log(chalk.dim('   • Data Architect (system design, scalability planning)'));
    } catch (error) {
      console.error(chalk.red('❌ Error listing agents:'), error.message);
    }
  });

agentsCmd
  .command('show <agent>')
  .description('Display detailed information about an agent')
  .action(async (agentName) => {
    try {
      const agentPath = findAgentFile(agentName);
      const agentData = yaml.parse(await fs.readFile(agentPath, 'utf8'));
      
      console.log('');
      console.log(chalk.bold.blue(`🤖 ${agentData.agent_metadata.name}`));
      console.log('');
      console.log(`📋 Role: ${agentData.agent_metadata.role}`);
      console.log(`🎯 Specialty: ${agentData.agent_metadata.specialty}`);
      console.log(`📖 Description: ${agentData.agent_metadata.description}`);
      console.log('');
      
      console.log(chalk.bold('🛠️ Core Capabilities:'));
      agentData.core_capabilities.forEach(cap => {
        console.log(`  • ${cap}`);
      });
      
      console.log('');
      console.log(chalk.bold('📊 Key Deliverables:'));
      agentData.deliverables.forEach(deliverable => {
        console.log(`  • ${deliverable}`);
      });
      
    } catch (error) {
      console.error(chalk.red('❌ Error:'), error.message);
      process.exit(1);
    }
  });

// =====================================================================
// TEMPLATE COMMANDS  
// =====================================================================

// Create templates subcommand
const templatesCmd = program
  .command('templates')
  .description('Manage project templates');

templatesCmd
  .command('list')
  .alias('ls')
  .description('List available templates')
  .action(async () => {
    console.log('');
    console.log(chalk.bold.blue('📋 Available Templates - Community Edition'));
    console.log('');
    
    try {
      const templatesPath = path.join(__dirname, '../templates');
      const categories = await fs.readdir(templatesPath);
      
      for (const category of categories) {
        const categoryPath = path.join(templatesPath, category);
        const stat = await fs.stat(categoryPath);
        
        if (stat.isDirectory()) {
          console.log(chalk.bold(`📁 ${category.toUpperCase()}`));
          
          const templates = await fs.readdir(categoryPath);
          templates.forEach((template, index) => {
            if (template.endsWith('.yaml') || template.endsWith('.yml')) {
              const name = template.replace(/\\.(yaml|yml)$/, '');
              console.log(`   ${index + 1}. ${name}`);
            }
          });
          console.log('');
        }
      }
      
      console.log(chalk.dim('💼 Enterprise Edition includes 68 additional templates:'));
      console.log(chalk.dim('   • Advanced ML pipeline templates'));
      console.log(chalk.dim('   • Real-time streaming architectures'));
      console.log(chalk.dim('   • Enterprise compliance frameworks'));
      console.log(chalk.dim('   • Industry-specific solutions (healthcare, finance, retail)'));
      
    } catch (error) {
      console.error(chalk.red('❌ Error listing templates:'), error.message);
    }
  });

templatesCmd
  .command('show <template>')
  .description('Display template details')
  .action(async (templateName) => {
    try {
      const templatePath = findTemplateFile(templateName);
      const templateData = yaml.parse(await fs.readFile(templatePath, 'utf8'));
      
      console.log('');
      console.log(chalk.bold.blue(`📋 ${templateData.template_metadata.name}`));
      console.log('');
      console.log(`📖 Description: ${templateData.template_metadata.description}`);
      console.log(`🏷️ Category: ${templateData.template_metadata.category}`);
      console.log(`⏱️ Estimated Time: ${templateData.template_metadata.estimated_time}`);
      console.log(`📊 Complexity: ${templateData.template_metadata.complexity}`);
      console.log('');
      
      if (templateData.deliverables) {
        console.log(chalk.bold('📦 Deliverables:'));
        templateData.deliverables.forEach(deliverable => {
          console.log(`  • ${deliverable}`);
        });
        console.log('');
      }
      
      if (templateData.use_cases) {
        console.log(chalk.bold('🎯 Use Cases:'));
        templateData.use_cases.forEach(useCase => {
          console.log(`  • ${useCase}`);
        });
      }
      
    } catch (error) {
      console.error(chalk.red('❌ Error:'), error.message);
      process.exit(1);
    }
  });

// =====================================================================
// QUALITY VALIDATION COMMANDS
// =====================================================================

program
  .command('validate')
  .description('Run quality validation on project data')
  .option('-p, --path <path>', 'Path to data files', '.')
  .option('-f, --format <format>', 'Output format (console, json, html)', 'console')
  .action(async (options) => {
    console.log('');
    console.log(chalk.bold.blue('🔍 Running 3-Dimensional Quality Validation...'));
    console.log('');
    
    try {
      // Look for validation scripts
      const validationScript = path.join(__dirname, '../validation_scripts/community_quality_check.py');
      
      if (await fs.pathExists(validationScript)) {
        console.log('📊 Executing Python quality validation...');
        
        const { spawn } = require('child_process');
        const python = spawn('python3', [validationScript, options.path], {
          stdio: 'inherit'
        });
        
        python.on('error', (error) => {
          console.error(chalk.red('❌ Python validation failed:'), error.message);
          console.log(chalk.yellow('💡 Ensure Python 3 and required packages are installed'));
        });
        
      } else {
        console.log(chalk.yellow('⚠️ Python validation script not found, running basic checks...'));
        await runBasicValidation(options.path);
      }
      
    } catch (error) {
      console.error(chalk.red('❌ Validation error:'), error.message);
    }
  });

// =====================================================================
// EXAMPLE COMMANDS
// =====================================================================

// Create examples subcommand
const examplesCmd = program
  .command('examples')
  .description('Manage and run examples');

examplesCmd
  .command('list')
  .description('List available examples')
  .action(async () => {
    console.log('');
    console.log(chalk.bold.blue('📚 Available Examples - Community Edition'));
    console.log('');
    
    try {
      const examplesPath = path.join(__dirname, '../examples');
      const examples = await fs.readdir(examplesPath);
      
      for (const example of examples) {
        const examplePath = path.join(examplesPath, example);
        const stat = await fs.stat(examplePath);
        
        if (stat.isDirectory()) {
          const readmePath = path.join(examplePath, 'README.md');
          
          console.log(chalk.bold(`📁 ${example}`));
          
          if (await fs.pathExists(readmePath)) {
            // Extract description from README
            const readme = await fs.readFile(readmePath, 'utf8');
            const firstLine = readme.split('\\n').find(line => line.startsWith('#'));
            if (firstLine) {
              console.log(`   ${firstLine.replace('#', '').trim()}`);
            }
          }
          
          console.log(`   📂 Location: examples/${example}`);
          console.log('');
        }
      }
      
    } catch (error) {
      console.error(chalk.red('❌ Error listing examples:'), error.message);
    }
  });

examplesCmd
  .command('install <example>')
  .description('Install example into current directory')
  .action(async (exampleName) => {
    console.log('');
    console.log(chalk.bold.blue(`📦 Installing example: ${exampleName}`));
    console.log('');
    
    try {
      const sourcePath = path.join(__dirname, '../examples', exampleName);
      const targetPath = path.join(process.cwd(), exampleName);
      
      if (!await fs.pathExists(sourcePath)) {
        throw new Error(`Example '${exampleName}' not found`);
      }
      
      await fs.copy(sourcePath, targetPath);
      
      console.log(chalk.green('✅ Example installed successfully!'));
      console.log('');
      console.log(`📁 Location: ${targetPath}`);
      console.log('');
      console.log('🚀 Next steps:');
      console.log(`   cd ${exampleName}`);
      console.log('   # Follow the README.md for implementation steps');
      
    } catch (error) {
      console.error(chalk.red('❌ Error installing example:'), error.message);
    }
  });

// =====================================================================
// UTILITY FUNCTIONS
// =====================================================================

function findAgentFile(agentName) {
  const agentsPath = path.join(__dirname, '../agents');
  const possibleNames = [
    `${agentName}.yaml`,
    `${agentName}.yml`,
    `${agentName.toLowerCase().replace(/\\s+/g, '-')}.yaml`,
    `${agentName.toLowerCase().replace(/\\s+/g, '-')}.yml`
  ];
  
  for (const name of possibleNames) {
    const filePath = path.join(agentsPath, name);
    if (fs.pathExistsSync(filePath)) {
      return filePath;
    }
  }
  
  throw new Error(`Agent '${agentName}' not found. Use 'agentic-data agents list' to see available agents.`);
}

function findTemplateFile(templateName) {
  const templatesPath = path.join(__dirname, '../templates');
  
  // Search through all category directories
  const categories = fs.readdirSync(templatesPath);
  
  for (const category of categories) {
    const categoryPath = path.join(templatesPath, category);
    if (fs.statSync(categoryPath).isDirectory()) {
      const possibleNames = [
        `${templateName}.yaml`,
        `${templateName}.yml`,
        `${templateName.toLowerCase().replace(/\\s+/g, '-')}.yaml`,
        `${templateName.toLowerCase().replace(/\\s+/g, '-')}.yml`
      ];
      
      for (const name of possibleNames) {
        const filePath = path.join(categoryPath, name);
        if (fs.pathExistsSync(filePath)) {
          return filePath;
        }
      }
    }
  }
  
  throw new Error(`Template '${templateName}' not found. Use 'agentic-data templates list' to see available templates.`);
}

async function createProjectStructure(projectPath, options) {
  // Create basic directory structure
  const directories = [
    'data-contracts',
    'implementation',
    'documentation',
    'validation',
    'sample-data'
  ];
  
  for (const dir of directories) {
    await fs.ensureDir(path.join(projectPath, dir));
  }
  
  // Copy template files based on selection
  if (options.template === 'ecommerce' && options.includeExample) {
    const examplePath = path.join(__dirname, '../examples/simple-ecommerce-analytics');
    await fs.copy(examplePath, projectPath);
  }
  
  // Create basic README
  const readme = generateProjectReadme(options);
  await fs.writeFile(path.join(projectPath, 'README.md'), readme);
  
  // Create basic package.json for project
  const projectPackage = {
    name: options.projectName,
    version: '1.0.0',
    description: `Data project created with AI Agentic Data Stack Framework - Community Edition`,
    scripts: {
      validate: 'agentic-data validate',
      'quality-check': 'agentic-data validate --format json'
    },
    dependencies: {
      'agentic-data-stack-community': `^${packageJson.version}`
    }
  };
  
  await fs.writeJSON(path.join(projectPath, 'package.json'), projectPackage, { spaces: 2 });
}

function generateProjectReadme(options) {
  return `# ${options.projectName}

Created with AI Agentic Data Stack Framework - Community Edition

## 🚀 Quick Start

\`\`\`bash
# Install dependencies
npm install

# Run quality validation
npm run validate

# List available agents
agentic-data agents list

# List available templates  
agentic-data templates list
\`\`\`

## 📊 Framework Features

- **4 Core AI Agents**: Data Engineer, Data Analyst, Data Product Manager, Data Quality Engineer
- **20 Essential Templates**: Common use cases and patterns
- **3-Dimensional Quality**: Completeness, Accuracy, Consistency validation
- **RFM Segmentation**: Customer behavior analysis
- **Community Support**: Open source with community documentation

## 📁 Project Structure

\`\`\`
${options.projectName}/
├── data-contracts/     # Data specifications and contracts
├── implementation/     # SQL scripts and analysis code
├── documentation/      # Project documentation
├── validation/         # Quality validation scripts
└── sample-data/        # Sample data for testing
\`\`\`

## 🛠️ Available Commands

\`\`\`bash
agentic-data info                    # Framework information
agentic-data agents list             # List AI agents
agentic-data templates list          # List templates
agentic-data validate               # Run quality checks
agentic-data examples list          # List examples
\`\`\`

## 📈 Next Steps

1. Review the data contracts in \`data-contracts/\`
2. Implement data pipelines using the provided templates
3. Run quality validation to ensure data integrity
4. Create dashboards and reports from your analysis

## 🏢 Upgrade to Enterprise

For advanced features including:
- 8 Specialized AI Agents
- 88 Interactive Templates  
- 7-Dimensional Quality Framework
- Real-time Collaboration
- Enterprise Compliance

Contact: enterprise@agenticdata.com

---

**Framework**: AI Agentic Data Stack - Community Edition  
**License**: MIT  
**Support**: Community-driven via GitHub
`;
}

async function runBasicValidation(dataPath) {
  console.log(chalk.yellow('🔍 Running basic file validation...'));
  
  try {
    const files = await fs.readdir(dataPath);
    const dataFiles = files.filter(file => 
      file.endsWith('.csv') || file.endsWith('.json') || file.endsWith('.yaml')
    );
    
    console.log(`📄 Found ${dataFiles.length} data files:`);
    dataFiles.forEach(file => {
      console.log(`  • ${file}`);
    });
    
    console.log('');
    console.log(chalk.green('✅ Basic validation complete'));
    console.log(chalk.dim('💡 Install Python dependencies for advanced validation'));
    
  } catch (error) {
    console.error(chalk.red('❌ Validation failed:'), error.message);
  }
}

// Error handling
program.on('command:*', () => {
  console.error(chalk.red('Invalid command: %s'), program.args.join(' '));
  console.log('');
  console.log('See --help for a list of available commands.');
  process.exit(1);
});

// =====================================================================
// INTERACTIVE MODE FUNCTIONS
// =====================================================================

async function enterInteractiveMode(agent) {
  console.log(chalk.green(`\n🎯 Entering interactive mode with ${agent.name}`));
  console.log(chalk.dim('Type commands, *help for help, or *exit to quit\n'));
  
  while (true) {
    try {
      const input = await inquirer.prompt([{
        type: 'input',
        name: 'command',
        message: `${agent.icon} ${agent.name}:`,
        prefix: ''
      }]);
      
      if (input.command.trim() === '*exit' || input.command.trim() === 'exit') {
        console.log(chalk.green(`👋 Goodbye from ${agent.name}!`));
        break;
      }
      
      if (input.command.trim() === '') {
        continue;
      }
      
      await commandHandler.parseCommand(input.command, { agent });
      console.log('');
      
    } catch (error) {
      if (error.name === 'ExitPromptError') {
        console.log(chalk.green(`\n👋 Goodbye from ${agent.name}!`));
        break;
      }
      console.error(chalk.red('❌ Error:'), error.message);
    }
  }
}

async function enterInteractiveShell() {
  console.log(chalk.green('\n🐚 Entering interactive shell mode'));
  console.log(chalk.dim('Type commands, @agent-name to activate agents, or exit to quit\n'));
  
  while (true) {
    try {
      const input = await inquirer.prompt([{
        type: 'input',
        name: 'command',
        message: '🚀 agentic-data:',
        prefix: ''
      }]);
      
      if (input.command.trim() === 'exit' || input.command.trim() === 'quit') {
        console.log(chalk.green('👋 Goodbye!'));
        break;
      }
      
      if (input.command.trim() === '') {
        continue;
      }
      
      await commandHandler.parseCommand(input.command);
      console.log('');
      
    } catch (error) {
      if (error.name === 'ExitPromptError') {
        console.log(chalk.green('\n👋 Goodbye!'));
        break;
      }
      console.error(chalk.red('❌ Error:'), error.message);
    }
  }
}

// =====================================================================
// MAIN EXECUTION
// =====================================================================

// Parse command line arguments
program.parse(process.argv);

// Show help if no command provided
if (!process.argv.slice(2).length) {
  console.log(chalk.bold.blue('🚀 AI Agentic Data Stack Framework - Community Edition'));
  console.log('');
  console.log('Welcome to the enhanced data engineering and analytics framework!');
  console.log('');
  program.outputHelp();
  
  console.log('');
  console.log(chalk.dim('💡 Quick start:'));
  console.log(chalk.dim('   agentic-data info          # Framework overview'));
  console.log(chalk.dim('   agentic-data init          # Create new project'));
  console.log(chalk.dim('   agentic-data agent data-engineer # Activate agent'));
  console.log(chalk.dim('   agentic-data interactive   # Interactive shell'));
  console.log(chalk.dim('   agentic-data workflow analytics # Run workflow'));
}