peter/src/drivers/simple.js

Peter Test Framework

/**
 *  @file       src/drivers/simple.js
 *  @author     Ryan Rossiter, ryan@kingsds.network
 *  @date       July 2020
 *
 *  This test driver executes .simple tests and .syntax tests.
 *
 *  .simple tests are node programs with top-level async plumbed in via a function wrapper.
 *  If the program passes basic syntax checks and exits with exit code 0, the test passes.
 *
 *  .syntax tests are exactly the same, except syntax checks are skipped. This feature
 *  primarily exists to allow Peter to test its own syntax checking, particularly around
 *  ES5 strict mode.
 *
 *  A test which starts with ¬ are negative tests: these fail when the exit code is 0.
 *  A negative .simple test with a syntax error still fails.
 */

const { spawn } = require('child_process');
const path = require('path');
const fs = require('fs');

const { registerDriver } = require('.');
const logStream = require('../utils/log-stream');
const { debugLog } = require('../utils/debug');

const debugSimple = debugLog.extend('simple');
const debug = require('debug');
const { setTestTimeout } = require('../utils/test-timer');

/**
 * Forward current environment variables to the `.simple` testProcess.
 *
 * Helpful when trying to pass debug variables to .simple tests ran through
 * PETER.
 *
 * e.g. `DCP_DEBUG=tests npx peter -v tests`
 */
const { env } = process;

async function runSimple(testPath, options)
{
  let testContents = await fs.promises.readFile(testPath, { encoding: 'utf-8' });
  var cwd = path.dirname(testPath)
  var args = ['--unhandled-rejections=strict', path.join(__dirname, 'simple-context-runner.js'), ...options.childArgv];
  var testProcess, testExitCode;

  /**
   * By default, the child's stdin, stdout, and stderr are redirected to
   * corresponding `subprocess.stdin`, `subprocess.stdout`, and
   * `subprocess.stderr` streams on the ChildProcess object. This is equivalent
   * to setting the options.stdio equal to ['pipe', 'pipe', 'pipe'].
   *
   * Ignore all standard streams by default so that tests which flood the
   * standard streams don't hang. (e.g. console.log 10 000 times).
   *
   * @type {import('child_process').StdioOptions}
   */
  const stdio = ['ignore', 'pipe', 'pipe'];

  if (options.invert)
    args.push('--invert');
  if (options.syntax)
    args.push('--syntax');
  if (options.debug) {
    /**
     * Deciding not to pipe stderr for the debug statements in the context
     * runner since logStream will overwrite the lines.
     */
    args.push('--debug');
  }
  if (options.verbose) {
    // Turn on piping stdout and stderr so that logStream works properly.
    [stdio[1], stdio[2]] = ['pipe', 'pipe'];
  }

  debug('peter')('testProcess args:', args);
  args.push(testPath);
  const timeoutREs = [ /* 1st parens capture # of seconds */
    /^\s*(?:\*|\/\*|\/\/)\s*peter-timeout: (\d+)\s*/m,
  ];

  let reMatch = false;
  for (const timeoutRE of timeoutREs)
  {
    if (timeoutRE.test(testContents))
    {
      options.timeout = testContents.match(timeoutRE)[1];
      reMatch = true;
    }
    options.timeout = 0
      || Number(process.env.PETER_SIMPLE_TIMEOUT)
      || Number(process.env.PETER_TIMEOUT)
      || options.timeout;
  }

  if (!reMatch)
  {
    const match = testContents.match(/(?:^|\n)(.*peter-timeout:.*)\n/im);
    if (match)
    {
      const lhs = testContents.slice(0, match.index);
      const lineNum = Array.from(lhs).filter(char => char === '\n').length;
      console.warn(`warning: syntax error on line ${lineNum + 2} for peter-timeout: '${match[1]}'`);
    }
  }

  testProcess = spawn(process.execPath, args, {
    stdio,
    env: {
      PETER_TEST_NUM: String(options.testNum),
      ...env,
    },
    cwd,
  });

  const streams = logStream.captureTest(testProcess, testPath, options);
  let stderr = '';
  testProcess.stderr.on('readable', () => {
    let chunk;
    while ((chunk = testProcess.stderr.read()) !== null)
      stderr += chunk.toString('utf8');
  })

  let timeoutExceeded = null;
  const testTimeoutStart = Date.now();
  setTestTimeout(testProcess, testPath, options.timeout, testTimeoutStart).then((time) => { timeoutExceeded = time });

  /* Resolve the pTest promise with the exitCode when the process exits */
  await new Promise((resolve, reject) => {
    var done = false;
    testProcess.on('error', (error) => {
      if (done)
        return;
      done = true;
      testExitCode = 1;
      resolve(error || 1);
    });

    testProcess.on('close', (exitCode) => {
      if (done)
        return;
      done = true;
      if (exitCode === null && timeoutExceeded)
        exitCode = options.invert ? 0 : 98;
      testExitCode = exitCode;
      resolve(exitCode);
    });
  });

  const ansiCsRe = /\u001b\[[\u0030-\u003f]*[\u0020-\u002f]*[\u0040-u\007e]/g; /* Matches ANSI Control Sequences */
  const failures = [];
  if (timeoutExceeded)
    failures.push({
      name: 'Simple test failed',
      stderr: stderr.length ? stderr.replace(ansiCsRe, '').split(/\r?\n/) : undefined,
      diag: {
        at: 'timeout',
        operator: 'lt',
        expected: `${options.timeout}s`,
        actual: timeoutExceeded + 's',
      },
    });
  else if (testExitCode)
    failures.push({
      name: 'Simple test failed' + (options.failing ? '(expected)' : ''),
      stderr: stderr.length ? stderr.replace(ansiCsRe, '').split(/\r?\n/) : undefined,
      diag: {
        at: 'exit code',
        operator: 'eq',
        expected: 0,
        actual: testExitCode,
      },
    });

  const result = {
    type: 'simple',
    count: 1,
    pass: testExitCode === 0 ? 1 : 0,
    fail: testExitCode === 0 ? 0 : 1,
    skip: 0,
    ok: testExitCode === 0,
    failures,
  };

  return { result, streams, failureTypes: { timeoutExceeded, testExitCode } };
}

function runSyntax(testPath, options)
{
  return runSimple(testPath, Object.assign({ syntax: true }, options));
}

registerDriver('simple', { run: runSimple });
registerDriver('syntax', { run: runSyntax });