csvlod-ai-mcp-server/tests/test-runner.ts

CSVLOD-AI MCP Server v3.0 with Quantum Context Intelligence - Revolutionary Context Intelligence Engine and Multimodal Processor for sovereign AI development

#!/usr/bin/env ts-node

import { execSync } from 'child_process';
import fs from 'fs';
import path from 'path';

interface TestSuite {
  name: string;
  path: string;
  description: string;
  critical: boolean;
}

interface TestResult {
  suite: string;
  passed: number;
  failed: number;
  duration: number;
  coverage?: number;
  success: boolean;
}

class CSVLODTestRunner {
  private testSuites: TestSuite[] = [
    {
      name: 'Core Components',
      path: 'mcp-server.test.ts',
      description: 'Basic functionality of CSVLODContext, Validator, and Generator',
      critical: true
    },
    {
      name: 'Individual Tools',
      path: 'tools/individual-tools.test.ts',
      description: 'Comprehensive testing of all 12 MCP tools',
      critical: true
    },
    {
      name: 'Integration Tests',
      path: 'integration/end-to-end.test.ts',
      description: 'End-to-end workflows and real-world scenarios',
      critical: true
    },
    {
      name: 'Performance Benchmarks',
      path: 'performance/benchmarks.test.ts',
      description: 'Performance validation and benchmark testing',
      critical: false
    }
  ];

  private results: TestResult[] = [];

  public async runAllTests(): Promise<void> {
    console.log('🚀 CSVLOD-AI MCP Server Test Suite Runner');
    console.log('==========================================\n');

    const startTime = Date.now();

    // Verify test environment
    await this.verifyTestEnvironment();

    // Run each test suite
    for (const suite of this.testSuites) {
      await this.runTestSuite(suite);
    }

    // Generate comprehensive report
    const totalDuration = Date.now() - startTime;
    await this.generateReport(totalDuration);

    // Exit with appropriate code
    const allCriticalPassed = this.results
      .filter(r => this.testSuites.find(s => s.name === r.suite)?.critical)
      .every(r => r.success);

    if (!allCriticalPassed) {
      console.log('❌ Critical tests failed. Framework not ready for production.');
      process.exit(1);
    } else {
      console.log('✅ All critical tests passed. Framework validated for production use.');
      process.exit(0);
    }
  }

  private async verifyTestEnvironment(): Promise<void> {
    console.log('🔍 Verifying test environment...');

    // Check Node.js version
    const nodeVersion = process.version;
    console.log(`   Node.js version: ${nodeVersion}`);

    // Check TypeScript compilation
    try {
      execSync('npm run build', { cwd: 'mcp-server', stdio: 'pipe' });
      console.log('   ✅ TypeScript compilation successful');
    } catch (error) {
      console.log('   ❌ TypeScript compilation failed');
      throw new Error('TypeScript compilation failed');
    }

    // Check test dependencies
    const packageJson = JSON.parse(fs.readFileSync('mcp-server/package.json', 'utf-8'));
    const testDeps = ['jest', 'ts-jest', '@types/jest'];
    
    for (const dep of testDeps) {
      if (packageJson.devDependencies?.[dep]) {
        console.log(`   ✅ ${dep} installed`);
      } else {
        console.log(`   ❌ ${dep} missing`);
        throw new Error(`Missing test dependency: ${dep}`);
      }
    }

    console.log('   ✅ Test environment verified\n');
  }

  private async runTestSuite(suite: TestSuite): Promise<void> {
    console.log(`📋 Running ${suite.name}...`);
    console.log(`   ${suite.description}`);

    const startTime = Date.now();
    let result: TestResult;

    try {
      // Run Jest for specific test file
      const output = execSync(
        `npm test -- --testPathPattern="${suite.path}" --verbose --json`,
        { 
          cwd: 'mcp-server',
          stdio: 'pipe',
          encoding: 'utf-8'
        }
      );

      const jestResult = JSON.parse(output);
      const testResult = jestResult.testResults[0];

      result = {
        suite: suite.name,
        passed: testResult?.numPassingTests || 0,
        failed: testResult?.numFailingTests || 0,
        duration: Date.now() - startTime,
        success: testResult?.numFailingTests === 0,
        coverage: this.calculateCoverage(jestResult)
      };

      if (result.success) {
        console.log(`   ✅ Passed (${result.passed} tests, ${result.duration}ms)`);
      } else {
        console.log(`   ❌ Failed (${result.failed}/${result.passed + result.failed} tests failed)`);
      }

    } catch (error) {
      result = {
        suite: suite.name,
        passed: 0,
        failed: 1,
        duration: Date.now() - startTime,
        success: false
      };

      console.log(`   ❌ Test suite execution failed: ${error}`);
    }

    this.results.push(result);
    console.log('');
  }

  private calculateCoverage(jestResult: any): number | undefined {
    if (jestResult.coverageMap) {
      const coverage = Object.values(jestResult.coverageMap) as any[];
      const totalStatements = coverage.reduce((sum, file) => sum + (file.s?.total || 0), 0);
      const coveredStatements = coverage.reduce((sum, file) => sum + (file.s?.covered || 0), 0);
      return totalStatements > 0 ? (coveredStatements / totalStatements) * 100 : undefined;
    }
    return undefined;
  }

  private async generateReport(totalDuration: number): Promise<void> {
    console.log('📊 Test Results Summary');
    console.log('=======================\n');

    const totalPassed = this.results.reduce((sum, r) => sum + r.passed, 0);
    const totalFailed = this.results.reduce((sum, r) => sum + r.failed, 0);
    const totalTests = totalPassed + totalFailed;
    const successRate = totalTests > 0 ? (totalPassed / totalTests) * 100 : 0;

    // Suite-by-suite results
    this.results.forEach(result => {
      const status = result.success ? '✅' : '❌';
      const critical = this.testSuites.find(s => s.name === result.suite)?.critical ? '🔴' : '🟡';
      const coverage = result.coverage ? ` (${result.coverage.toFixed(1)}% coverage)` : '';
      
      console.log(`${status} ${critical} ${result.suite}: ${result.passed} passed, ${result.failed} failed (${result.duration}ms)${coverage}`);
    });

    console.log('');

    // Overall statistics
    console.log(`📈 Overall Statistics:`);
    console.log(`   Total Tests: ${totalTests}`);
    console.log(`   Passed: ${totalPassed} (${successRate.toFixed(1)}%)`);
    console.log(`   Failed: ${totalFailed}`);
    console.log(`   Total Duration: ${totalDuration}ms`);

    // Coverage analysis
    const avgCoverage = this.results
      .filter(r => r.coverage !== undefined)
      .reduce((sum, r) => sum + (r.coverage || 0), 0) / 
      this.results.filter(r => r.coverage !== undefined).length;

    if (avgCoverage) {
      console.log(`   Average Coverage: ${avgCoverage.toFixed(1)}%`);
    }

    // Framework validation status
    console.log('\n🎯 Framework Validation Status:');
    
    const criticalTests = this.results.filter(r => 
      this.testSuites.find(s => s.name === r.suite)?.critical
    );
    
    const allCriticalPassed = criticalTests.every(r => r.success);
    const mcpToolsValidated = this.results.find(r => r.suite === 'Individual Tools')?.success;
    const integrationValidated = this.results.find(r => r.suite === 'Integration Tests')?.success;
    const performanceMet = this.results.find(r => r.suite === 'Performance Benchmarks')?.success;

    console.log(`   ✅ Critical Tests: ${allCriticalPassed ? 'PASSED' : 'FAILED'}`);
    console.log(`   ✅ 12 MCP Tools: ${mcpToolsValidated ? 'VALIDATED' : 'NEEDS WORK'}`);
    console.log(`   ✅ Integration: ${integrationValidated ? 'VALIDATED' : 'NEEDS WORK'}`);
    console.log(`   ${performanceMet ? '✅' : '⚠️'} Performance: ${performanceMet ? 'MEETS TARGETS' : 'NEEDS OPTIMIZATION'}`);

    // Generate detailed JSON report
    const report = {
      timestamp: new Date().toISOString(),
      totalDuration,
      totalTests,
      totalPassed,
      totalFailed,
      successRate,
      averageCoverage: avgCoverage || null,
      suiteResults: this.results,
      frameworkValidation: {
        criticalTestsPassed: allCriticalPassed,
        mcpToolsValidated,
        integrationValidated,
        performanceMet,
        productionReady: allCriticalPassed && mcpToolsValidated && integrationValidated
      }
    };

    fs.writeFileSync('mcp-server/test-results.json', JSON.stringify(report, null, 2));
    console.log('\n📄 Detailed report saved to: mcp-server/test-results.json');

    // Framework readiness assessment
    if (report.frameworkValidation.productionReady) {
      console.log('\n🎉 FRAMEWORK STATUS: PRODUCTION READY');
      console.log('   All 12 MCP tools validated');
      console.log('   Integration workflows tested');
      console.log('   Claims validation: Ready for community launch');
    } else {
      console.log('\n⚠️  FRAMEWORK STATUS: NEEDS ATTENTION');
      console.log('   Some critical tests failed');
      console.log('   Address issues before production deployment');
    }
  }
}

// CLI interface
if (import.meta.url === `file://${process.argv[1]}`) {
  const runner = new CSVLODTestRunner();
  
  runner.runAllTests().catch(error => {
    console.error('❌ Test runner failed:', error);
    process.exit(1);
  });
}

export { CSVLODTestRunner };