@stryker-mutator/tap-runner/src/tap-test-runner.ts

A plugin to use the TAP (test anything protocol) test runner in Stryker, the JavaScript mutation testing framework

import childProcess from 'child_process';
import path from 'path';
import { fileURLToPath } from 'url';
import fs from 'fs/promises';

import { commonTokens, Injector, PluginContext, tokens } from '@stryker-mutator/api/plugin';
import {
  BaseTestResult,
  determineHitLimitReached,
  DryRunOptions,
  DryRunResult,
  DryRunStatus,
  MutantRunOptions,
  MutantRunResult,
  TestResult,
  TestRunner,
  TestRunnerCapabilities,
  TestStatus,
  TimeoutDryRunResult,
  toMutantRunResult,
} from '@stryker-mutator/api/test-runner';
import { InstrumenterContext, INSTRUMENTER_CONSTANTS, MutantCoverage, StrykerOptions } from '@stryker-mutator/api/core';
import { Logger } from '@stryker-mutator/api/logging';
import { normalizeFileName } from '@stryker-mutator/util';

import * as pluginTokens from './plugin-tokens.js';
import { findTestyLookingFiles, captureTapResult, TapResult, buildArguments } from './tap-helper.js';
import { TapRunnerOptionsWithStrykerOptions } from './tap-runner-options-with-stryker-options.js';
import { strykerHitLimit, strykerNamespace, strykerDryRun, tempTapOutputFileName } from './setup/env.cjs';

export function createTapTestRunnerFactory(namespace: typeof INSTRUMENTER_CONSTANTS.NAMESPACE | '__stryker2__' = INSTRUMENTER_CONSTANTS.NAMESPACE): {
  (injector: Injector<PluginContext>): TapTestRunner;
  inject: ['$injector'];
} {
  createTapTestRunner.inject = tokens(commonTokens.injector);
  function createTapTestRunner(injector: Injector<PluginContext>) {
    return injector.provideValue(pluginTokens.globalNamespace, namespace).injectClass(TapTestRunner);
  }
  return createTapTestRunner;
}

export const createTapTestRunner = createTapTestRunnerFactory();

class HitLimitError extends Error {
  constructor(public readonly result: TimeoutDryRunResult) {
    super(result.reason);
  }
}

interface TapRunOptions {
  disableBail: boolean;
  activeMutant?: string;
  hitLimit?: number;
  dryRun?: boolean;
}

export class TapTestRunner implements TestRunner {
  public static inject = tokens(commonTokens.options, commonTokens.logger, pluginTokens.globalNamespace);
  private testFiles: string[] = [];
  private static readonly hookFile = normalizeFileName(path.resolve(path.dirname(fileURLToPath(import.meta.url)), 'setup', 'hook.cjs'));
  private readonly options: TapRunnerOptionsWithStrykerOptions;

  constructor(
    options: StrykerOptions,
    private readonly log: Logger,
    private readonly globalNamespace: typeof INSTRUMENTER_CONSTANTS.NAMESPACE | '__stryker2__',
  ) {
    this.options = options as TapRunnerOptionsWithStrykerOptions;
  }

  public capabilities(): Promise<TestRunnerCapabilities> | TestRunnerCapabilities {
    return { reloadEnvironment: true };
  }

  public async init(): Promise<void> {
    this.testFiles = await findTestyLookingFiles(this.options.tap.testFiles);
  }

  public async dryRun(options: DryRunOptions): Promise<DryRunResult> {
    return this.run({ disableBail: options.disableBail, dryRun: true });
  }

  public async mutantRun(options: MutantRunOptions): Promise<MutantRunResult> {
    return toMutantRunResult(
      await this.run({ disableBail: options.disableBail, activeMutant: options.activeMutant.id, hitLimit: options.hitLimit }, options.testFilter),
    );
  }

  private async run(testOptions: TapRunOptions, testFilter?: string[]): Promise<DryRunResult> {
    const testFiles = testFilter ?? this.testFiles;

    const runs: TestResult[] = [];
    const totalCoverage: MutantCoverage = {
      static: {},
      perTest: {},
    };

    for (const testFile of testFiles) {
      try {
        const { testResult, coverage } = await this.runFile(testFile, testOptions);
        runs.push(testResult);
        totalCoverage.perTest[testFile] = coverage?.static ?? {};

        if (testResult.status !== TestStatus.Success && !testOptions.disableBail) {
          break;
        }
      } catch (err) {
        if (err instanceof HitLimitError) {
          return err.result;
        } else if (err instanceof Error) {
          return {
            status: DryRunStatus.Error,
            errorMessage: `Error running file "${testFile}". ${err.message}`,
          };
        }
        throw err;
      }
    }
    return {
      status: DryRunStatus.Complete,
      tests: runs,
      mutantCoverage: totalCoverage,
    };
  }

  private async runFile(testFile: string, testOptions: TapRunOptions): Promise<{ testResult: TestResult; coverage: MutantCoverage | undefined }> {
    const env: NodeJS.ProcessEnv = {
      ...process.env,
      [strykerHitLimit]: testOptions.hitLimit?.toString(),
      [strykerNamespace]: this.globalNamespace,
      [INSTRUMENTER_CONSTANTS.ACTIVE_MUTANT_ENV_VARIABLE]: testOptions.activeMutant,
      [strykerDryRun]: testOptions.dryRun?.toString(),
    };
    const args = buildArguments(this.options.tap.nodeArgs, TapTestRunner.hookFile, testFile);

    if (this.log.isDebugEnabled()) {
      this.log.debug(`Running: \`node ${args.map((arg) => `"${arg}"`).join(' ')}\` in ${process.cwd()}`);
    }
    const now = () => new Date().getTime();
    const before = now();
    const tapProcess = childProcess.spawn('node', args, { env });
    const result = await captureTapResult(tapProcess, !testOptions.disableBail && this.options.tap.forceBail);
    const timeSpentMs = now() - before;
    const fileName = tempTapOutputFileName(tapProcess.pid);
    const fileContent = await fs.readFile(fileName, 'utf-8');
    await fs.rm(fileName);
    const file = JSON.parse(fileContent) as InstrumenterContext;
    const hitLimitReached = determineHitLimitReached(file.hitCount, testOptions.hitLimit);
    if (hitLimitReached) {
      throw new HitLimitError(hitLimitReached);
    }
    return { testResult: this.tapResultToTestResult(testFile, result, timeSpentMs), coverage: file.mutantCoverage };
  }

  private tapResultToTestResult(fileName: string, { result, failedTests }: TapResult, timeSpentMs: number): TestResult {
    const generic: BaseTestResult = {
      id: fileName,
      name: fileName,
      timeSpentMs: result.time ?? timeSpentMs,
      fileName: fileName,
      startPosition: undefined,
    };

    if (result.ok) {
      return {
        ...generic,
        status: TestStatus.Success,
      };
    } else {
      return {
        ...generic,
        status: TestStatus.Failed,
        failureMessage: failedTests.map((f) => `${f.fullname}: ${f.name}`).join(', ') ?? 'Unknown issue',
      };
    }
  }
}