test/lib/internal/test_runner/test.js

Node.js 18's node:test, as an npm package

// https://github.com/nodejs/node/blob/9eb363a3e00dbba572756c7ed314273f17ea8e2e/lib/internal/test_runner/test.js

'use strict'

const {
  ArrayPrototypeMap,
  ArrayPrototypePush,
  ArrayPrototypeReduce,
  ArrayPrototypeShift,
  ArrayPrototypeSlice,
  ArrayPrototypeSome,
  ArrayPrototypeUnshift,
  FunctionPrototype,
  MathMax,
  Number,
  ObjectSeal,
  PromisePrototypeThen,
  PromiseResolve,
  ReflectApply,
  RegExpPrototypeExec,
  SafeMap,
  SafeSet,
  SafePromiseAll,
  SafePromiseRace,
  Symbol
} = require('#internal/per_context/primordials')
const { AsyncResource } = require('async_hooks')
const { once } = require('events')
const { AbortController } = require('#internal/abort_controller')
const {
  codes: {
    ERR_INVALID_ARG_TYPE,
    ERR_TEST_FAILURE
  },
  AbortError
} = require('#internal/errors')
const { getOptionValue } = require('#internal/options')
const { MockTracker } = require('#internal/test_runner/mock')
const { TestsStream } = require('#internal/test_runner/tests_stream')
const {
  convertStringToRegExp,
  createDeferredCallback,
  isTestFailureError
} = require('#internal/test_runner/utils')
const {
  createDeferredPromise,
  kEmptyObject,
  once: runOnce
} = require('#internal/util')
const { isPromise } = require('#internal/util/types')
const {
  validateAbortSignal,
  validateNumber,
  validateOneOf,
  validateUint32
} = require('#internal/validators')
const { setTimeout } = require('#timers/promises')
const { TIMEOUT_MAX } = require('#internal/timers')
const { cpus, availableParallelism } = require('os')
const { bigint: hrtime } = process.hrtime
const kCallbackAndPromisePresent = 'callbackAndPromisePresent'
const kCancelledByParent = 'cancelledByParent'
const kParentAlreadyFinished = 'parentAlreadyFinished'
const kSubtestsFailed = 'subtestsFailed'
const kTestCodeFailure = 'testCodeFailure'
const kTestTimeoutFailure = 'testTimeoutFailure'
const kHookFailure = 'hookFailed'
const kDefaultTimeout = null
const noop = FunctionPrototype
const isTestRunner = getOptionValue('--test')
const testOnlyFlag = !isTestRunner && getOptionValue('--test-only')
const testNamePatternFlag = isTestRunner
  ? null
  : getOptionValue('--test-name-pattern')
const testNamePatterns = testNamePatternFlag?.length > 0
  ? ArrayPrototypeMap(
    testNamePatternFlag,
    (re) => convertStringToRegExp(re, '--test-name-pattern')
  )
  : null
const kShouldAbort = Symbol('kShouldAbort')
const kFilename = process.argv?.[1]
const kHookNames = ObjectSeal(['before', 'after', 'beforeEach', 'afterEach'])
const kUnwrapErrors = new SafeSet()
  .add(kTestCodeFailure).add(kHookFailure)
  .add('uncaughtException').add('unhandledRejection')

function stopTest (timeout, signal) {
  if (timeout === kDefaultTimeout) {
    return once(signal, 'abort')
  }
  return PromisePrototypeThen(setTimeout(timeout, null, { ref: false, signal }), () => {
    throw new ERR_TEST_FAILURE(
      `test timed out after ${timeout}ms`,
      kTestTimeoutFailure
    )
  })
}

class TestContext {
  #test

  constructor (test) {
    this.#test = test
  }

  get signal () {
    return this.#test.signal
  }

  get name () {
    return this.#test.name
  }

  diagnostic (message) {
    this.#test.diagnostic(message)
  }

  get mock () {
    this.#test.mock = this.#test.mock ?? new MockTracker()
    return this.#test.mock
  }

  runOnly (value) {
    this.#test.runOnlySubtests = !!value
  }

  skip (message) {
    this.#test.skip(message)
  }

  todo (message) {
    this.#test.todo(message)
  }

  test (name, options, fn) {
    // eslint-disable-next-line no-use-before-define
    const subtest = this.#test.createSubtest(Test, name, options, fn)

    return subtest.start()
  }

  after (fn, options) {
    this.#test.createHook('after', fn, options)
  }

  beforeEach (fn, options) {
    this.#test.createHook('beforeEach', fn, options)
  }

  afterEach (fn, options) {
    this.#test.createHook('afterEach', fn, options)
  }
}

class Test extends AsyncResource {
  #abortController
  #outerSignal
  #reportedSubtest

  constructor (options) {
    super('Test')

    let { fn, name, parent, skip } = options
    const { concurrency, only, timeout, todo, signal } = options

    if (typeof fn !== 'function') {
      fn = noop
    }

    if (typeof name !== 'string' || name === '') {
      name = fn.name || '<anonymous>'
    }

    if (!(parent instanceof Test)) {
      parent = null
    }

    if (parent === null) {
      this.concurrency = 1
      this.nesting = 0
      this.only = testOnlyFlag
      this.reporter = new TestsStream()
      this.runOnlySubtests = this.only
      this.testNumber = 0
      this.timeout = kDefaultTimeout
    } else {
      const nesting = parent.parent === null
        ? parent.nesting
        : parent.nesting + 1

      this.concurrency = parent.concurrency
      this.nesting = nesting
      this.only = only ?? !parent.runOnlySubtests
      this.reporter = parent.reporter
      this.runOnlySubtests = !this.only
      this.testNumber = parent.subtests.length + 1
      this.timeout = parent.timeout
    }

    switch (typeof concurrency) {
      case 'number':
        validateUint32(concurrency, 'options.concurrency', 1)
        this.concurrency = concurrency
        break

      case 'boolean':
        if (concurrency) {
          this.concurrency = parent === null
            ? MathMax((typeof availableParallelism === 'undefined' ? cpus().length : availableParallelism()) - 1, 1)
            : Infinity
        } else {
          this.concurrency = 1
        }
        break

      default:
        if (concurrency != null) throw new ERR_INVALID_ARG_TYPE('options.concurrency', 'a number or boolean', concurrency)
    }

    if (timeout != null && timeout !== Infinity) {
      validateNumber(timeout, 'options.timeout', 0, TIMEOUT_MAX)
      this.timeout = timeout
    }

    if (testNamePatterns !== null) {
      // eslint-disable-next-line no-use-before-define
      const match = this instanceof TestHook || ArrayPrototypeSome(
        testNamePatterns,
        (re) => RegExpPrototypeExec(re, name) !== null
      )

      if (!match) {
        skip = 'test name does not match pattern'
      }
    }

    if (testOnlyFlag && !this.only) {
      skip = '\'only\' option not set'
    }

    if (skip) {
      fn = noop
    }

    this.#abortController = new AbortController()
    this.#outerSignal = signal
    this.signal = this.#abortController.signal

    validateAbortSignal(signal, 'options.signal')
    this.#outerSignal?.addEventListener('abort', this.#abortHandler)

    this.fn = fn
    this.mock = null
    this.name = name
    this.parent = parent
    this.cancelled = false
    this.skipped = !!skip
    this.isTodo = !!todo
    this.startTime = null
    this.endTime = null
    this.passed = false
    this.error = null
    this.diagnostics = []
    this.message = typeof skip === 'string'
      ? skip
      : typeof todo === 'string' ? todo : null
    this.activeSubtests = 0
    this.pendingSubtests = []
    this.readySubtests = new SafeMap()
    this.subtests = []
    this.hooks = {
      before: [],
      after: [],
      beforeEach: [],
      afterEach: []
    }
    this.waitingOn = 0
    this.finished = false
  }

  hasConcurrency () {
    return this.concurrency > this.activeSubtests
  }

  addPendingSubtest (deferred) {
    ArrayPrototypePush(this.pendingSubtests, deferred)
  }

  async processPendingSubtests () {
    while (this.pendingSubtests.length > 0 && this.hasConcurrency()) {
      const deferred = ArrayPrototypeShift(this.pendingSubtests)
      await deferred.test.run()
      deferred.resolve()
    }
  }

  addReadySubtest (subtest) {
    this.readySubtests.set(subtest.testNumber, subtest)
  }

  processReadySubtestRange (canSend) {
    const start = this.waitingOn
    const end = start + this.readySubtests.size

    for (let i = start; i < end; i++) {
      const subtest = this.readySubtests.get(i)

      // Check if the specified subtest is in the map. If it is not, return
      // early to avoid trying to process any more tests since they would be
      // out of order.
      if (subtest === undefined) {
        return
      }

      // Call isClearToSend() in the loop so that it is:
      // - Only called if there are results to report in the correct order.
      // - Guaranteed to only be called a maximum of once per call to
      //   processReadySubtestRange().
      canSend = canSend || this.isClearToSend()

      if (!canSend) {
        return
      }

      if (i === 1 && this.parent !== null) {
        this.reportStarted()
      }

      // Report the subtest's results and remove it from the ready map.
      subtest.finalize()
      this.readySubtests.delete(i)
    }
  }

  createSubtest (Factory, name, options, fn, overrides) {
    if (typeof name === 'function') {
      fn = name
    } else if (name !== null && typeof name === 'object') {
      fn = options
      options = name
    } else if (typeof options === 'function') {
      fn = options
    }

    if (options === null || typeof options !== 'object') {
      options = kEmptyObject
    }

    let parent = this

    // If this test has already ended, attach this test to the root test so
    // that the error can be properly reported.
    const preventAddingSubtests = this.finished || this.buildPhaseFinished
    if (preventAddingSubtests) {
      while (parent.parent !== null) {
        parent = parent.parent
      }
    }

    const test = new Factory({ __proto__: null, fn, name, parent, ...options, ...overrides })

    if (parent.waitingOn === 0) {
      parent.waitingOn = test.testNumber
    }

    if (preventAddingSubtests) {
      test.startTime = test.startTime || hrtime()
      test.fail(
        new ERR_TEST_FAILURE(
          'test could not be started because its parent finished',
          kParentAlreadyFinished
        )
      )
    }

    ArrayPrototypePush(parent.subtests, test)
    return test
  }

  #abortHandler = () => {
    this.cancel(this.#outerSignal?.reason || new AbortError('The test was aborted'))
  }

  cancel (error) {
    if (this.endTime !== null) {
      return
    }

    this.fail(error ||
      new ERR_TEST_FAILURE(
        'test did not finish before its parent and was cancelled',
        kCancelledByParent
      )
    )
    this.startTime = this.startTime || this.endTime // If a test was canceled before it was started, e.g inside a hook
    this.cancelled = true
    this.#abortController.abort()
  }

  createHook (name, fn, options) {
    validateOneOf(name, 'hook name', kHookNames)
    // eslint-disable-next-line no-use-before-define
    const hook = new TestHook(fn, options)
    ArrayPrototypePush(this.hooks[name], hook)
    return hook
  }

  fail (err) {
    if (this.error !== null) {
      return
    }

    this.endTime = hrtime()
    this.passed = false
    this.error = err
  }

  pass () {
    if (this.endTime !== null) {
      return
    }

    this.endTime = hrtime()
    this.passed = true
  }

  skip (message) {
    this.skipped = true
    this.message = message
  }

  todo (message) {
    this.isTodo = true
    this.message = message
  }

  diagnostic (message) {
    ArrayPrototypePush(this.diagnostics, message)
  }

  start () {
    // If there is enough available concurrency to run the test now, then do
    // it. Otherwise, return a Promise to the caller and mark the test as
    // pending for later execution.
    if (!this.parent.hasConcurrency()) {
      const deferred = createDeferredPromise()

      deferred.test = this
      this.parent.addPendingSubtest(deferred)
      return deferred.promise
    }

    return this.run()
  }

  [kShouldAbort] () {
    if (this.signal.aborted) {
      return true
    }
    if (this.#outerSignal?.aborted) {
      this.cancel(this.#outerSignal.reason || new AbortError('The test was aborted'))
      return true
    }
  }

  getRunArgs () {
    const ctx = new TestContext(this)
    return { ctx, args: [ctx] }
  }

  async runHook (hook, args) {
    validateOneOf(hook, 'hook name', kHookNames)
    try {
      await ArrayPrototypeReduce(this.hooks[hook], async (prev, hook) => {
        await prev
        await hook.run(args)
        if (hook.error) {
          throw hook.error
        }
      }, PromiseResolve())
    } catch (err) {
      const error = new ERR_TEST_FAILURE(`failed running ${hook} hook`, kHookFailure)
      error.cause = isTestFailureError(err) ? err.cause : err
      throw error
    }
  }

  async run () {
    if (this.parent !== null) {
      this.parent.activeSubtests++
    }
    this.startTime = hrtime()

    if (this[kShouldAbort]()) {
      this.postRun()
      return
    }

    const { args, ctx } = this.getRunArgs()
    const after = runOnce(async () => {
      if (this.hooks.after.length > 0) {
        await this.runHook('after', { args, ctx })
      }
    })
    const afterEach = runOnce(async () => {
      if (this.parent?.hooks.afterEach.length > 0) {
        await this.parent.runHook('afterEach', { args, ctx })
      }
    })

    try {
      if (this.parent?.hooks.beforeEach.length > 0) {
        await this.parent.runHook('beforeEach', { args, ctx })
      }
      const stopPromise = stopTest(this.timeout, this.signal)
      const runArgs = ArrayPrototypeSlice(args)
      ArrayPrototypeUnshift(runArgs, this.fn, ctx)

      if (this.fn.length === runArgs.length - 1) {
        // This test is using legacy Node.js error first callbacks.
        const { promise, cb } = createDeferredCallback()

        ArrayPrototypePush(runArgs, cb)
        const ret = ReflectApply(this.runInAsyncScope, this, runArgs)

        if (isPromise(ret)) {
          this.fail(new ERR_TEST_FAILURE(
            'passed a callback but also returned a Promise',
            kCallbackAndPromisePresent
          ))
          await SafePromiseRace([ret, stopPromise])
        } else {
          await SafePromiseRace([PromiseResolve(promise), stopPromise])
        }
      } else {
        // This test is synchronous or using Promises.
        const promise = ReflectApply(this.runInAsyncScope, this, runArgs)
        await SafePromiseRace([PromiseResolve(promise), stopPromise])
      }

      if (this[kShouldAbort]()) {
        this.postRun()
        return
      }

      await after()
      await afterEach()
      this.pass()
    } catch (err) {
      try { await after() } catch { /* Ignore error. */ }
      try { await afterEach() } catch { /* test is already failing, let's the error */ }
      if (isTestFailureError(err)) {
        if (err.failureType === kTestTimeoutFailure) {
          this.cancel(err)
        } else {
          this.fail(err)
        }
      } else {
        this.fail(new ERR_TEST_FAILURE(err, kTestCodeFailure))
      }
    }

    // Clean up the test. Then, try to report the results and execute any
    // tests that were pending due to available concurrency.
    this.postRun()
  }

  postRun (pendingSubtestsError) {
    const counters = { __proto__: null, failed: 0, passed: 0, cancelled: 0, skipped: 0, todo: 0, totalFailed: 0 }

    // If the test was failed before it even started, then the end time will
    // be earlier than the start time. Correct that here.
    if (this.endTime < this.startTime) {
      this.endTime = hrtime()
    }
    if (this.startTime == null) this.startTime = this.endTime

    // The test has run, so recursively cancel any outstanding subtests and
    // mark this test as failed if any subtests failed.
    this.pendingSubtests = []
    for (let i = 0; i < this.subtests.length; i++) {
      const subtest = this.subtests[i]

      if (!subtest.finished) {
        subtest.cancel(pendingSubtestsError)
        subtest.postRun(pendingSubtestsError)
      }

      // Check SKIP and TODO tests first, as those should not be counted as
      // failures.
      if (subtest.skipped) {
        counters.skipped++
      } else if (subtest.isTodo) {
        counters.todo++
      } else if (subtest.cancelled) {
        counters.cancelled++
      } else if (!subtest.passed) {
        counters.failed++
      } else {
        counters.passed++
      }

      if (!subtest.passed) {
        counters.totalFailed++
      }
    }

    if ((this.passed || this.parent === null) && counters.totalFailed > 0) {
      const subtestString = `subtest${counters.totalFailed > 1 ? 's' : ''}`
      const msg = `${counters.totalFailed} ${subtestString} failed`

      this.fail(new ERR_TEST_FAILURE(msg, kSubtestsFailed))
    }

    this.#outerSignal?.removeEventListener('abort', this.#abortHandler)
    this.mock?.reset()

    if (this.parent !== null) {
      this.parent.activeSubtests--
      this.parent.addReadySubtest(this)
      this.parent.processReadySubtestRange(false)
      this.parent.processPendingSubtests()
    } else if (!this.reported) {
      this.reported = true
      this.reporter.plan(this.nesting, kFilename, this.subtests.length)

      for (let i = 0; i < this.diagnostics.length; i++) {
        this.reporter.diagnostic(this.nesting, kFilename, this.diagnostics[i])
      }

      this.reporter.diagnostic(this.nesting, kFilename, `tests ${this.subtests.length}`)
      this.reporter.diagnostic(this.nesting, kFilename, `pass ${counters.passed}`)
      this.reporter.diagnostic(this.nesting, kFilename, `fail ${counters.failed}`)
      this.reporter.diagnostic(this.nesting, kFilename, `cancelled ${counters.cancelled}`)
      this.reporter.diagnostic(this.nesting, kFilename, `skipped ${counters.skipped}`)
      this.reporter.diagnostic(this.nesting, kFilename, `todo ${counters.todo}`)
      this.reporter.diagnostic(this.nesting, kFilename, `duration_ms ${this.#duration()}`)
      this.reporter.push(null)
    }
  }

  isClearToSend () {
    return this.parent === null ||
      (
        this.parent.waitingOn === this.testNumber && this.parent.isClearToSend()
      )
  }

  finalize () {
    // By the time this function is called, the following can be relied on:
    // - The current test has completed or been cancelled.
    // - All of this test's subtests have completed or been cancelled.
    // - It is the current test's turn to report its results.

    // Report any subtests that have not been reported yet. Since all of the
    // subtests have finished, it's safe to pass true to
    // processReadySubtestRange(), which will finalize all remaining subtests.
    this.processReadySubtestRange(true)

    // Output this test's results and update the parent's waiting counter.
    this.report()
    this.parent.waitingOn++
    this.finished = true
  }

  #duration () {
    // Duration is recorded in BigInt nanoseconds. Convert to milliseconds.
    return Number(this.endTime - this.startTime) / 1_000_000
  }

  report () {
    if (this.subtests.length > 0) {
      this.reporter.plan(this.subtests[0].nesting, kFilename, this.subtests.length)
    } else {
      this.reportStarted()
    }
    let directive
    const details = { __proto__: null, duration_ms: this.#duration() }

    if (this.skipped) {
      directive = this.reporter.getSkip(this.message)
    } else if (this.isTodo) {
      directive = this.reporter.getTodo(this.message)
    }

    if (this.passed) {
      this.reporter.ok(this.nesting, kFilename, this.testNumber, this.name, details, directive)
    } else {
      details.error = this.error
      this.reporter.fail(this.nesting, kFilename, this.testNumber, this.name, details, directive)
    }

    for (let i = 0; i < this.diagnostics.length; i++) {
      this.reporter.diagnostic(this.nesting, kFilename, this.diagnostics[i])
    }
  }

  reportStarted () {
    if (this.#reportedSubtest || this.parent === null) {
      return
    }
    this.#reportedSubtest = true
    this.parent.reportStarted()
    this.reporter.start(this.nesting, kFilename, this.name)
  }
}

class TestHook extends Test {
  #args
  constructor (fn, options) {
    if (options === null || typeof options !== 'object') {
      options = kEmptyObject
    }
    const { timeout, signal } = options
    super({ __proto__: null, fn, timeout, signal })
  }

  run (args) {
    this.#args = args
    return super.run()
  }

  getRunArgs () {
    return this.#args
  }

  postRun () {
  }
}

class ItTest extends Test {
  constructor (opt) { super(opt) } // eslint-disable-line no-useless-constructor
  getRunArgs () {
    return { ctx: { signal: this.signal, name: this.name }, args: [] }
  }
}

class Suite extends Test {
  constructor (options) {
    super(options)

    try {
      const { ctx, args } = this.getRunArgs()
      this.buildSuite = PromisePrototypeThen(
        PromiseResolve(this.runInAsyncScope(this.fn, ctx, args)),
        undefined,
        (err) => {
          this.fail(new ERR_TEST_FAILURE(err, kTestCodeFailure))
        })
    } catch (err) {
      this.fail(new ERR_TEST_FAILURE(err, kTestCodeFailure))
    }
    this.fn = () => {}
    this.buildPhaseFinished = true
  }

  getRunArgs () {
    return { ctx: { signal: this.signal, name: this.name }, args: [] }
  }

  async run () {
    const hookArgs = this.getRunArgs()
    const afterEach = runOnce(async () => {
      if (this.parent?.hooks.afterEach.length > 0) {
        await this.parent.runHook('afterEach', hookArgs)
      }
    })

    try {
      this.parent.activeSubtests++
      await this.buildSuite
      this.startTime = hrtime()

      if (this[kShouldAbort]()) {
        this.subtests = []
        this.postRun()
        return
      }

      if (this.parent?.hooks.beforeEach.length > 0) {
        await this.parent.runHook('beforeEach', hookArgs)
      }

      await this.runHook('before', hookArgs)

      const stopPromise = stopTest(this.timeout, this.signal)
      const subtests = this.skipped || this.error ? [] : this.subtests
      const promise = SafePromiseAll(subtests, (subtests) => subtests.start())

      await SafePromiseRace([promise, stopPromise])
      await this.runHook('after', hookArgs)
      await afterEach()

      this.pass()
    } catch (err) {
      try { await afterEach() } catch { /* test is already failing, let's the error */ }
      if (isTestFailureError(err)) {
        this.fail(err)
      } else {
        this.fail(new ERR_TEST_FAILURE(err, kTestCodeFailure))
      }
    }

    this.postRun()
  }
}

module.exports = {
  ItTest,
  kCancelledByParent,
  kSubtestsFailed,
  kTestCodeFailure,
  kUnwrapErrors,
  Suite,
  Test
}