csvlod-ai-mcp-server/tests/performance/benchmarks.test.ts

CSVLOD-AI MCP Server v3.0 with Quantum Context Intelligence - Revolutionary Context Intelligence Engine and Multimodal Processor for sovereign AI development

import { jest } from '@jest/globals';
import { CSVLODContext } from '../../src/context.js';
import { CSVLODValidator } from '../../src/validator.js';
import { CSVLODGenerator } from '../../src/generator.js';
import { LocalMCPRegistry } from '../../src/registry.js';
import { createSwarmCoordinator } from '../../src/swarm.js';
import fs from 'fs/promises';
import path from 'path';
import os from 'os';

interface BenchmarkResult {
  operation: string;
  duration: number;
  success: boolean;
  throughput?: number;
  memoryUsage?: number;
}

describe('Performance Benchmark Tests', () => {
  let tempDir: string;
  const benchmarkResults: BenchmarkResult[] = [];
  
  beforeEach(async () => {
    tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'csvlod-bench-'));
  });

  afterEach(async () => {
    try {
      await fs.rm(tempDir, { recursive: true, force: true });
    } catch (error) {
      // Ignore cleanup errors
    }
  });

  afterAll(() => {
    // Output benchmark summary
    console.log('\n=== CSVLOD-AI MCP Server Performance Benchmarks ===');
    benchmarkResults.forEach(result => {
      const status = result.success ? '✅' : '❌';
      const throughput = result.throughput ? ` (${result.throughput} ops/sec)` : '';
      console.log(`${status} ${result.operation}: ${result.duration}ms${throughput}`);
    });
    
    const avgDuration = benchmarkResults.reduce((sum, r) => sum + r.duration, 0) / benchmarkResults.length;
    const successRate = (benchmarkResults.filter(r => r.success).length / benchmarkResults.length) * 100;
    console.log(`\nAverage Duration: ${avgDuration.toFixed(2)}ms`);
    console.log(`Success Rate: ${successRate.toFixed(1)}%`);
    console.log('================================================\n');
  });

  async function benchmark(operation: string, fn: () => Promise<any>): Promise<BenchmarkResult> {
    const memBefore = process.memoryUsage().heapUsed;
    const start = Date.now();
    let success = false;
    
    try {
      await fn();
      success = true;
    } catch (error) {
      console.error(`Benchmark ${operation} failed:`, error);
    }
    
    const duration = Date.now() - start;
    const memAfter = process.memoryUsage().heapUsed;
    const memoryUsage = memAfter - memBefore;
    
    const result: BenchmarkResult = {
      operation,
      duration,
      success,
      memoryUsage
    };
    
    benchmarkResults.push(result);
    return result;
  }

  describe('Core Tool Performance Benchmarks', () => {
    test('csvlod_init performance across project types', async () => {
      const context = new CSVLODContext();
      
      // Benchmark minimal initialization
      await benchmark('csvlod_init (minimal)', async () => {
        const minimalDir = path.join(tempDir, 'minimal');
        await fs.mkdir(minimalDir);
        await context.initialize(minimalDir, 'minimal');
      });
      
      // Benchmark basic initialization
      await benchmark('csvlod_init (basic)', async () => {
        const basicDir = path.join(tempDir, 'basic');
        await fs.mkdir(basicDir);
        await context.initialize(basicDir, 'basic');
      });
      
      // Benchmark enterprise initialization
      await benchmark('csvlod_init (enterprise)', async () => {
        const enterpriseDir = path.join(tempDir, 'enterprise');
        await fs.mkdir(enterpriseDir);
        await context.initialize(enterpriseDir, 'enterprise');
      });
      
      // Verify performance targets
      const initResults = benchmarkResults.filter(r => r.operation.includes('csvlod_init'));
      initResults.forEach(result => {
        expect(result.duration).toBeLessThan(5000); // 5 second target
        expect(result.success).toBe(true);
      });
    });

    test('csvlod_validate performance scaling', async () => {
      const validator = new CSVLODValidator();
      
      // Setup test projects of different sizes
      const context = new CSVLODContext();
      const smallProject = path.join(tempDir, 'small');
      const largeProject = path.join(tempDir, 'large');
      
      await fs.mkdir(smallProject);
      await fs.mkdir(largeProject);
      
      await context.initialize(smallProject, 'minimal');
      await context.initialize(largeProject, 'enterprise');
      
      // Benchmark validation performance
      await benchmark('csvlod_validate (small project)', async () => {
        await validator.validate(smallProject, { strict: false });
      });
      
      await benchmark('csvlod_validate (large project)', async () => {
        await validator.validate(largeProject, { strict: false });
      });
      
      await benchmark('csvlod_validate (strict mode)', async () => {
        await validator.validate(largeProject, { strict: true });
      });
      
      // Verify performance targets
      const validateResults = benchmarkResults.filter(r => r.operation.includes('csvlod_validate'));
      validateResults.forEach(result => {
        expect(result.duration).toBeLessThan(3000); // 3 second target
        expect(result.success).toBe(true);
      });
    });

    test('csvlod_generate performance across components', async () => {
      const generator = new CSVLODGenerator();
      const components = ['context', 'manifest', 'prompt', 'structure'];
      
      for (const component of components) {
        await benchmark(`csvlod_generate (${component})`, async () => {
          await generator.generate(component, tempDir);
        });
      }
      
      // Verify generation performance
      const generateResults = benchmarkResults.filter(r => r.operation.includes('csvlod_generate'));
      generateResults.forEach(result => {
        expect(result.duration).toBeLessThan(2000); // 2 second target
        expect(result.success).toBe(true);
      });
    });

    test('csvlod_analyze performance with metrics', async () => {
      const context = new CSVLODContext();
      await context.initialize(tempDir, 'enterprise');
      
      await benchmark('csvlod_analyze (basic)', async () => {
        await context.analyze(tempDir, { includeMetrics: false });
      });
      
      await benchmark('csvlod_analyze (with metrics)', async () => {
        await context.analyze(tempDir, { includeMetrics: true });
      });
      
      // Verify analysis performance
      const analyzeResults = benchmarkResults.filter(r => r.operation.includes('csvlod_analyze'));
      analyzeResults.forEach(result => {
        expect(result.duration).toBeLessThan(5000); // 5 second target
        expect(result.success).toBe(true);
      });
    });
  });

  describe('MCP Registry Performance Benchmarks', () => {
    test('mcp_registry operations performance', async () => {
      const registry = new LocalMCPRegistry();
      
      await benchmark('mcp_registry (sync)', async () => {
        await registry.execute('sync', {});
      });
      
      await benchmark('mcp_registry (search)', async () => {
        await registry.execute('search', { query: 'test' });
      });
      
      await benchmark('mcp_registry (list)', async () => {
        await registry.execute('list', {});
      });
      
      await benchmark('mcp_orchestrate', async () => {
        await registry.orchestrate('Test orchestration task', { preferLocal: true });
      });
      
      // Verify registry performance
      const registryResults = benchmarkResults.filter(r => 
        r.operation.includes('mcp_registry') || r.operation.includes('mcp_orchestrate')
      );
      registryResults.forEach(result => {
        expect(result.duration).toBeLessThan(3000); // 3 second target
        expect(result.success).toBe(true);
      });
    });
  });

  describe('Swarm Coordination Performance Benchmarks', () => {
    test('swarm operations performance scaling', async () => {
      const context = new CSVLODContext();
      await context.initialize(tempDir, 'enterprise');
      
      // Benchmark swarm initialization
      let coordinator: any;
      await benchmark('swarm_init', async () => {
        coordinator = await createSwarmCoordinator(tempDir);
      });
      
      // Benchmark task decomposition with different complexities
      await benchmark('swarm_decompose (simple)', async () => {
        await coordinator.decomposeTask('Simple task', tempDir, ['standards']);
      });
      
      await benchmark('swarm_decompose (complex)', async () => {
        await coordinator.decomposeTask(
          'Complex multi-phase project with security, testing, and deployment requirements',
          tempDir,
          ['principles', 'standards', 'designs']
        );
      });
      
      // Benchmark status operations
      await benchmark('swarm_status', async () => {
        coordinator.getSwarmStatus();
      });
      
      // Benchmark task management
      const tasks = await coordinator.decomposeTask('Benchmark task', tempDir, ['standards']);
      if (tasks.length > 0) {
        await benchmark('swarm_assign', async () => {
          await coordinator.assignTask(tasks[0].id);
        });
        
        await benchmark('swarm_complete', async () => {
          await coordinator.completeTask(tasks[0].id, {
            status: 'success',
            output: 'Benchmark completed'
          });
        });
        
        await benchmark('swarm_tasks', async () => {
          coordinator.getTaskDetails(tasks[0].id);
        });
      }
      
      // Verify swarm performance
      const swarmResults = benchmarkResults.filter(r => r.operation.includes('swarm_'));
      swarmResults.forEach(result => {
        expect(result.duration).toBeLessThan(4000); // 4 second target
        expect(result.success).toBe(true);
      });
    });
  });

  describe('Concurrent Operations Performance', () => {
    test('parallel tool execution performance', async () => {
      const context = new CSVLODContext();
      await context.initialize(tempDir, 'basic');
      
      await benchmark('concurrent_validation_analysis', async () => {
        const validator = new CSVLODValidator();
        const operations = [
          validator.validate(tempDir, { strict: false }),
          context.analyze(tempDir, { includeMetrics: false }),
          validator.validate(tempDir, { strict: true }),
          context.analyze(tempDir, { includeMetrics: true })
        ];
        
        await Promise.all(operations);
      });
      
      // Verify concurrent performance is better than sequential
      const concurrentResult = benchmarkResults.find(r => r.operation === 'concurrent_validation_analysis');
      expect(concurrentResult?.duration).toBeLessThan(8000); // Should be faster than 4 sequential operations
    });

    test('throughput benchmarks', async () => {
      const context = new CSVLODContext();
      const iterations = 5;
      
      // Throughput test for lightweight operations
      const start = Date.now();
      const promises = [];
      
      for (let i = 0; i < iterations; i++) {
        const projectDir = path.join(tempDir, `throughput-${i}`);
        await fs.mkdir(projectDir);
        promises.push(context.initialize(projectDir, 'minimal'));
      }
      
      await Promise.all(promises);
      const duration = Date.now() - start;
      const throughput = (iterations / duration) * 1000; // operations per second
      
      benchmarkResults.push({
        operation: 'throughput_init_minimal',
        duration,
        success: true,
        throughput
      });
      
      expect(throughput).toBeGreaterThan(0.5); // At least 0.5 ops/sec
    });
  });

  describe('Memory Performance Benchmarks', () => {
    test('memory usage under load', async () => {
      const initialMemory = process.memoryUsage().heapUsed;
      
      // Create multiple projects to test memory scaling
      const context = new CSVLODContext();
      const projects = [];
      
      for (let i = 0; i < 10; i++) {
        const projectDir = path.join(tempDir, `memory-test-${i}`);
        await fs.mkdir(projectDir);
        projects.push(projectDir);
      }
      
      await benchmark('memory_stress_test', async () => {
        const operations = projects.map(dir => 
          context.initialize(dir, 'basic')
        );
        await Promise.all(operations);
      });
      
      const finalMemory = process.memoryUsage().heapUsed;
      const memoryIncrease = finalMemory - initialMemory;
      
      // Memory increase should be reasonable (less than 100MB for 10 projects)
      expect(memoryIncrease).toBeLessThan(100 * 1024 * 1024);
    });
  });

  describe('Framework Performance Targets Validation', () => {
    test('should meet all stated performance targets', async () => {
      // Test framework claims against actual performance
      const targets = {
        'Context Load Time': 5000, // <5 seconds
        'Framework Setup': 30000,  // <30 seconds for complete setup
        'Validation': 3000,        // <3 seconds
        'AI Agent Success Rate': 0.90 // >90% (simulated)
      };
      
      // Context load time test
      const context = new CSVLODContext();
      await benchmark('target_context_load', async () => {
        await context.initialize(tempDir, 'basic');
        await context.analyze(tempDir, { includeMetrics: true });
      });
      
      // Framework setup test (complete workflow)
      await benchmark('target_framework_setup', async () => {
        const setupDir = path.join(tempDir, 'setup-test');
        await fs.mkdir(setupDir);
        
        // Complete framework setup
        await context.initialize(setupDir, 'enterprise');
        
        const validator = new CSVLODValidator();
        await validator.validate(setupDir, { strict: true });
        
        const generator = new CSVLODGenerator();
        await generator.generate('structure', setupDir);
        
        const coordinator = await createSwarmCoordinator(setupDir);
        await coordinator.decomposeTask('Setup validation task', setupDir, ['standards']);
      });
      
      // Validation speed test
      const validator = new CSVLODValidator();
      await benchmark('target_validation_speed', async () => {
        await validator.validate(tempDir, { strict: true });
      });
      
      // Verify all targets are met
      const contextLoadResult = benchmarkResults.find(r => r.operation === 'target_context_load');
      const setupResult = benchmarkResults.find(r => r.operation === 'target_framework_setup');
      const validationResult = benchmarkResults.find(r => r.operation === 'target_validation_speed');
      
      expect(contextLoadResult?.duration).toBeLessThan(targets['Context Load Time']);
      expect(setupResult?.duration).toBeLessThan(targets['Framework Setup']);
      expect(validationResult?.duration).toBeLessThan(targets['Validation']);
      
      // All operations should succeed (simulating >90% success rate)
      const allTargetResults = [contextLoadResult, setupResult, validationResult];
      const successRate = allTargetResults.filter(r => r?.success).length / allTargetResults.length;
      expect(successRate).toBeGreaterThanOrEqual(targets['AI Agent Success Rate']);
    });
  });
});