ctrlshiftleft/src/core/testRunner.ts

AI-powered toolkit for embedding QA and security testing into development workflows

import { chromium, firefox, webkit, Browser, BrowserContext, Page } from 'playwright';
import { execSync } from 'child_process';
import fs from 'fs/promises';
import path from 'path';
import { glob } from 'glob';
import chalk from 'chalk';
import { performance } from 'perf_hooks';
import util from 'util';
import { EventEmitter } from 'events';

export interface TestRunnerOptions {
  browser: string;
  headless: boolean;
  timeout: number;
  reporter: string;
  workers?: number;
  filter?: string;
  retries?: number;
  verbose?: boolean;
  updateSnapshots?: boolean;
}

export interface TestError {
  message: string;
  stack?: string;
  testPath: string;
  testName?: string;
}

export interface TestFileResult {
  path: string;
  passed: number;
  failed: number;
  skipped: number;
  duration: number;
  errors: TestError[];
  tests: TestResult[];
}

export interface TestResult {
  name: string;
  status: 'passed' | 'failed' | 'skipped';
  duration: number;
  error?: TestError;
}

export interface TestResults {
  total: number;
  passed: number;
  failed: number;
  skipped: number;
  duration: number;
  fileResults: TestFileResult[];
  errors: TestError[];
}

export class TestRunner extends EventEmitter {
  private options: TestRunnerOptions;
  
  constructor(options: TestRunnerOptions) {
    super();
    this.options = {
      browser: options.browser || 'chromium',
      headless: options.headless !== false,
      timeout: options.timeout || 30,
      reporter: options.reporter || 'list',
      workers: options.workers || 1,
      filter: options.filter || '',
      retries: options.retries || 0,
      verbose: options.verbose || false,
      updateSnapshots: options.updateSnapshots || false
    };
  }
  
  /**
   * Run tests from a file or directory
   * @param testPath Path to test file or directory
   * @returns Test results
   */
  async runTests(testPath: string): Promise<TestResults> {
    const startTime = performance.now();
    const testFiles = await this.getTestFiles(testPath);
    
    const results: TestResults = {
      total: 0,
      passed: 0,
      failed: 0,
      skipped: 0,
      duration: 0,
      fileResults: [],
      errors: []
    };
    
    this.emit('run:start', { testFiles });
    
    // If parallel execution is enabled
    if (this.options.workers && this.options.workers > 1) {
      // Run tests in parallel
      const chunks = this.chunkArray(testFiles, this.options.workers);
      const chunkResults = await Promise.all(chunks.map(chunk => this.runTestsChunk(chunk)));
      
      // Merge results
      for (const chunkResult of chunkResults) {
        results.total += chunkResult.total;
        results.passed += chunkResult.passed;
        results.failed += chunkResult.failed;
        results.skipped += chunkResult.skipped;
        results.fileResults.push(...chunkResult.fileResults);
        results.errors.push(...chunkResult.errors);
      }
    } else {
      // Run tests sequentially
      const fileResults = await Promise.all(testFiles.map(file => this.runTestFile(file)));
      
      for (const fileResult of fileResults) {
        results.total += fileResult.passed + fileResult.failed + fileResult.skipped;
        results.passed += fileResult.passed;
        results.failed += fileResult.failed;
        results.skipped += fileResult.skipped;
        results.fileResults.push(fileResult);
        results.errors.push(...fileResult.errors);
      }
    }
    
    results.duration = performance.now() - startTime;
    this.emit('run:end', results);
    
    return results;
  }
  
  /**
   * Run a chunk of test files in parallel
   */
  private async runTestsChunk(testFiles: string[]): Promise<TestResults> {
    const results: TestResults = {
      total: 0,
      passed: 0,
      failed: 0,
      skipped: 0,
      duration: 0,
      fileResults: [],
      errors: []
    };
    
    // Initialize browser
    const browser = await this.launchBrowser();
    
    try {
      for (const testFile of testFiles) {
        try {
          const fileResult = await this.runTestFile(testFile, browser);
          
          results.total += fileResult.passed + fileResult.failed + fileResult.skipped;
          results.passed += fileResult.passed;
          results.failed += fileResult.failed;
          results.skipped += fileResult.skipped;
          results.fileResults.push(fileResult);
          results.errors.push(...fileResult.errors);
          
          this.emit('file:end', fileResult);
        } catch (error) {
          const testError: TestError = {
            message: (error as Error).message,
            stack: (error as Error).stack,
            testPath: testFile
          };
          
          results.failed += 1;
          results.total += 1;
          results.errors.push(testError);
          results.fileResults.push({
            path: testFile,
            passed: 0,
            failed: 1,
            skipped: 0,
            duration: 0,
            errors: [testError],
            tests: [{
              name: path.basename(testFile),
              status: 'failed',
              duration: 0,
              error: testError
            }]
          });
          
          this.emit('error', testError);
        }
      }
    } finally {
      await browser.close();
    }
    
    return results;
  }
  
  /**
   * Run tests from a single file
   */
  private async runTestFile(testFile: string, browser?: Browser): Promise<TestFileResult> {
    const startTime = performance.now();
    const relativePath = path.relative(process.cwd(), testFile);
    
    this.emit('file:start', { path: testFile });
    
    const fileResult: TestFileResult = {
      path: testFile,
      passed: 0,
      failed: 0,
      skipped: 0,
      duration: 0,
      errors: [],
      tests: []
    };
    
    // Create a browser if not provided
    let localBrowser: Browser | null = null;
    const actualBrowser = browser || (localBrowser = await this.launchBrowser());
    
    try {
      // Create a new browser context for the test file
      const context = await actualBrowser.newContext();
      const page = await context.newPage();
      
      try {
        // Run in a try-catch to handle module loading errors
        try {
          // Import test file dynamically
          const testModule = require(testFile);
          const testSuite = testModule.default || testModule;
          
          if (typeof testSuite.runTests === 'function') {
            // Run the test suite
            this.emit('suite:start', { path: testFile });
            
            // Create a test context with hooks for individual test reporting
            const testResults: TestResult[] = [];
            const testContext = {
              beforeEach: (fn: Function) => fn(),
              afterEach: (fn: Function) => fn(),
              onTestStart: (name: string) => {
                this.emit('test:start', { path: testFile, name });
              },
              onTestComplete: (result: TestResult) => {
                testResults.push(result);
                if (result.status === 'passed') {
                  fileResult.passed++;
                } else if (result.status === 'failed') {
                  fileResult.failed++;
                  if (result.error) {
                    fileResult.errors.push(result.error);
                  }
                } else if (result.status === 'skipped') {
                  fileResult.skipped++;
                }
                this.emit('test:end', { path: testFile, ...result });
              }
            };
            
            // Run the test suite with our context
            const suiteResults = await testSuite.runTests(page, {
              timeout: this.options.timeout * 1000,
              context: testContext,
              filter: this.options.filter,
              updateSnapshots: this.options.updateSnapshots
            });
            
            // If the test suite used our hooks, we already have detailed results
            if (testResults.length === 0) {
              // Otherwise, use the legacy format
              fileResult.passed += suiteResults.passed;
              fileResult.failed += suiteResults.failed;
              fileResult.skipped += suiteResults.skipped;
            } else {
              fileResult.tests = testResults;
            }
            
            this.emit('suite:end', { 
              path: testFile, 
              passed: fileResult.passed,
              failed: fileResult.failed,
              skipped: fileResult.skipped
            });
          } else if (Object.keys(testModule).some(key => key.startsWith('test'))) {
            // This appears to be a standard Playwright test file
            // We need to execute it through Playwright's test runner
            this.emit('suite:start', { path: testFile });
            
            // Try to run with Playwright's test runner directly
            let projectConfig = {
              testDir: path.dirname(testFile),
              reporter: [['list']],
              timeout: this.options.timeout * 1000,
              retries: this.options.retries || 0
            };
            
            // Collect results via custom reporter
            const testResults: TestResult[] = [];
            
            try {
              // Use the Playwright test runner via CLI
              const command = `npx playwright test ${path.basename(testFile)} --reporter=list ${this.options.headless ? '--headed' : ''} ${this.options.workers && this.options.workers > 1 ? `--workers=${this.options.workers}` : ''}`;
              
              if (this.options.verbose) {
                this.emit('log', { message: `Running command: ${command}`, level: 'debug' });
              }
              
              try {
                execSync(command, { 
                  cwd: path.dirname(testFile),
                  stdio: this.options.verbose ? 'inherit' : 'pipe',
                  encoding: 'utf-8',
                  timeout: this.options.timeout * 2000 // Double the timeout for the process
                });
                
                // If we get here, the command succeeded
                fileResult.passed = 1;
              } catch (execError) {
                // The command failed with non-zero exit code
                fileResult.failed = 1;
                fileResult.errors.push({
                  message: `Playwright test run failed: ${(execError as Error).message}`,
                  stack: (execError as Error).stack,
                  testPath: testFile
                });
              }
            } catch (error) {
              // If the programmatic test runner fails, fall back to manual execution
              // This is a fallback to ensure our runner still works 
              fileResult.failed = 1;
              fileResult.errors.push({
                message: `Failed to run Playwright test: ${(error as Error).message}`,
                stack: (error as Error).stack,
                testPath: testFile
              });
            }
          } else {
            const error = `Warning: Test file does not export a runTests function: ${testFile}`;
            this.emit('warning', { path: testFile, message: error });
            
            if (this.options.verbose) {
              console.warn(chalk.yellow(error));
            }
          }
        } catch (error) {
          const testError: TestError = {
            message: `Error importing test file: ${(error as Error).message}`,
            stack: (error as Error).stack,
            testPath: testFile
          };
          
          fileResult.failed += 1;
          fileResult.errors.push(testError);
          
          this.emit('error', testError);
          
          if (this.options.verbose) {
            console.error(chalk.red(`Error importing test file ${relativePath}:`), error);
          }
        }
      } finally {
        await context.close();
      }
    } catch (error) {
      const testError: TestError = {
        message: `Browser context error: ${(error as Error).message}`,
        stack: (error as Error).stack,
        testPath: testFile
      };
      
      fileResult.failed += 1;
      fileResult.errors.push(testError);
      
      this.emit('error', testError);
      
      if (this.options.verbose) {
        console.error(chalk.red(`Browser context error in ${relativePath}:`), error);
      }
    } finally {
      // Close the local browser if we created it
      if (localBrowser) {
        await localBrowser.close();
      }
    }
    
    fileResult.duration = performance.now() - startTime;
    return fileResult;
  }
  
  /**
   * Split an array into chunks for parallel processing
   */
  private chunkArray<T>(array: T[], chunkSize: number): T[][] {
    if (chunkSize <= 0) {
      return [array];
    }
    
    const chunks: T[][] = [];
    const totalChunks = Math.min(chunkSize, array.length);
    const itemsPerChunk = Math.ceil(array.length / totalChunks);
    
    for (let i = 0; i < totalChunks; i++) {
      const startIndex = i * itemsPerChunk;
      const endIndex = Math.min(startIndex + itemsPerChunk, array.length);
      chunks.push(array.slice(startIndex, endIndex));
    }
    
    return chunks;
  }
  
  /**
   * Launch browser based on options
   */
  private async launchBrowser(): Promise<Browser> {
    switch (this.options.browser) {
      case 'firefox':
        return firefox.launch({ headless: this.options.headless });
      case 'webkit':
        return webkit.launch({ headless: this.options.headless });
      case 'chromium':
      default:
        return chromium.launch({ headless: this.options.headless });
    }
  }
  
  /**
   * Get test files from a path (file or directory)
   */
  private async getTestFiles(testPath: string): Promise<string[]> {
    const stats = await fs.stat(testPath);
    
    if (stats.isFile()) {
      return [testPath];
    }
    
    if (stats.isDirectory()) {
      // Find all test files
      return glob(`${testPath}/**/*.test.{js,ts}`);
    }
    
    return [];
  }
}