dd-trace
Version:
Datadog APM tracing client for JavaScript
1,501 lines (1,330 loc) • 50.5 kB
JavaScript
'use strict'
// Capture real timers at module load time, before any test can install fake timers.
const realSetTimeout = setTimeout
const path = require('node:path')
const { performance } = require('node:perf_hooks')
const shimmer = require('../../datadog-shimmer')
const log = require('../../dd-trace/src/log')
const {
VITEST_WORKER_TRACE_PAYLOAD_CODE,
VITEST_WORKER_LOGS_PAYLOAD_CODE,
DYNAMIC_NAME_RE,
getTestSuitePath,
getEfdRetryCount,
getMaxEfdRetryCount,
recordAttemptToFixExecution,
collectTestOptimizationSummariesFromTraces,
logAttemptToFixTestExecution,
logTestOptimizationSummary,
getTestOptimizationRequestResults,
} = require('../../dd-trace/src/plugins/util/test')
const { addHook, channel } = require('./helpers/instrument')
// test hooks
const testStartCh = channel('ci:vitest:test:start')
const testFinishTimeCh = channel('ci:vitest:test:finish-time')
const testPassCh = channel('ci:vitest:test:pass')
const testErrorCh = channel('ci:vitest:test:error')
const testSkipCh = channel('ci:vitest:test:skip')
const isNewTestCh = channel('ci:vitest:test:is-new')
const isAttemptToFixCh = channel('ci:vitest:test:is-attempt-to-fix')
const isDisabledCh = channel('ci:vitest:test:is-disabled')
const isQuarantinedCh = channel('ci:vitest:test:is-quarantined')
const isModifiedCh = channel('ci:vitest:test:is-modified')
const testFnCh = channel('ci:vitest:test:fn')
// test suite hooks
const testSuiteStartCh = channel('ci:vitest:test-suite:start')
const testSuiteFinishCh = channel('ci:vitest:test-suite:finish')
const testSuiteErrorCh = channel('ci:vitest:test-suite:error')
// test session hooks
const testSessionStartCh = channel('ci:vitest:session:start')
const testSessionFinishCh = channel('ci:vitest:session:finish')
const testSessionConfigurationCh = channel('ci:vitest:session:configuration')
const libraryConfigurationCh = channel('ci:vitest:library-configuration')
const knownTestsCh = channel('ci:vitest:known-tests')
const isEarlyFlakeDetectionFaultyCh = channel('ci:vitest:is-early-flake-detection-faulty')
const testManagementTestsCh = channel('ci:vitest:test-management-tests')
const modifiedFilesCh = channel('ci:vitest:modified-files')
const workerReportTraceCh = channel('ci:vitest:worker-report:trace')
const workerReportLogsCh = channel('ci:vitest:worker-report:logs')
const codeCoverageReportCh = channel('ci:vitest:coverage-report')
const taskToCtx = new WeakMap()
const taskToStatuses = new WeakMap()
const attemptToFixTaskToStatuses = new WeakMap()
const originalHookFns = new WeakMap()
const newTasks = new WeakSet()
const dynamicNameTasks = new WeakSet()
const newTestsWithDynamicNames = new Set()
const disabledTasks = new WeakSet()
const quarantinedTasks = new WeakSet()
const attemptToFixTasks = new WeakSet()
const modifiedTasks = new WeakSet()
const efdDeterminedRetries = new WeakMap()
const efdSlowAbortedTasks = new WeakSet()
const efdExecutionStartByTask = new WeakMap()
const efdSkippedRetryResults = new WeakMap()
const attemptToFixExecutions = new Map()
const loggedAttemptToFixTests = new Set()
let isRetryReasonEfd = false
let isRetryReasonAttemptToFix = false
const switchedStatuses = new WeakSet()
const workerProcesses = new WeakSet()
const mainProcessSetupPromises = new WeakMap()
const coverageWrappedProviders = new WeakSet()
const finishWrappedContexts = new WeakSet()
let isFlakyTestRetriesEnabled = false
let flakyTestRetriesCount = 0
let isEarlyFlakeDetectionEnabled = false
let earlyFlakeDetectionNumRetries = 0
let earlyFlakeDetectionSlowTestRetries = {}
let isEarlyFlakeDetectionFaulty = false
let isKnownTestsEnabled = false
let isTestManagementTestsEnabled = false
let isImpactedTestsEnabled = false
let vitestGetFn = null
let vitestSetFn = null
let vitestGetHooks = null
let testManagementAttemptToFixRetries = 0
let isDiEnabled = false
let testCodeCoverageLinesTotal
let coverageRootDir
let isSessionStarted = false
let vitestPool = null
const BREAKPOINT_HIT_GRACE_PERIOD_MS = 400
function getConfiguredEfdRetryCount (slowTestRetries, fallbackRetryCount) {
if (!slowTestRetries || !Object.keys(slowTestRetries).length) {
return fallbackRetryCount
}
return getMaxEfdRetryCount(slowTestRetries)
}
function getTestCommand () {
return `vitest ${process.argv.slice(2).join(' ')}`
}
function waitForHitProbe () {
return new Promise(resolve => {
realSetTimeout(() => {
resolve()
}, BREAKPOINT_HIT_GRACE_PERIOD_MS)
})
}
function isValidKnownTests (receivedKnownTests) {
return !!receivedKnownTests.vitest
}
function getProvidedContext () {
try {
const {
_ddIsEarlyFlakeDetectionEnabled,
_ddIsDiEnabled,
_ddKnownTests: knownTests,
_ddEarlyFlakeDetectionNumRetries: numRepeats,
_ddEarlyFlakeDetectionSlowTestRetries: slowTestRetries,
_ddIsKnownTestsEnabled: isKnownTestsEnabled,
_ddIsTestManagementTestsEnabled: isTestManagementTestsEnabled,
_ddTestManagementAttemptToFixRetries: testManagementAttemptToFixRetries,
_ddTestManagementTests: testManagementTests,
_ddIsFlakyTestRetriesEnabled: isFlakyTestRetriesEnabled,
_ddFlakyTestRetriesCount: flakyTestRetriesCount,
_ddIsImpactedTestsEnabled: isImpactedTestsEnabled,
_ddModifiedFiles: modifiedFiles,
_ddTestSessionId: testSessionId,
_ddTestModuleId: testModuleId,
_ddTestCommand: testCommand,
} = globalThis.__vitest_worker__.providedContext
return {
isDiEnabled: _ddIsDiEnabled,
isEarlyFlakeDetectionEnabled: _ddIsEarlyFlakeDetectionEnabled,
knownTests,
numRepeats,
slowTestRetries: slowTestRetries ?? {},
isKnownTestsEnabled,
isTestManagementTestsEnabled,
testManagementAttemptToFixRetries,
testManagementTests,
isFlakyTestRetriesEnabled,
flakyTestRetriesCount: flakyTestRetriesCount ?? 0,
isImpactedTestsEnabled,
modifiedFiles,
testSessionId,
testModuleId,
testCommand,
}
} catch {
log.error('Vitest workers could not parse provided context, so some features will not work.')
return {
isDiEnabled: false,
isEarlyFlakeDetectionEnabled: false,
knownTests: {},
numRepeats: 0,
slowTestRetries: {},
isKnownTestsEnabled: false,
isTestManagementTestsEnabled: false,
testManagementAttemptToFixRetries: 0,
testManagementTests: {},
isFlakyTestRetriesEnabled: false,
flakyTestRetriesCount: 0,
isImpactedTestsEnabled: false,
modifiedFiles: {},
testSessionId: undefined,
testModuleId: undefined,
testCommand: undefined,
}
}
}
function isReporterPackage (vitestPackage) {
return vitestPackage.B?.name === 'BaseSequencer'
}
// from 2.0.0
function isReporterPackageNew (vitestPackage) {
return vitestPackage.e?.name === 'BaseSequencer'
}
function isReporterPackageNewest (vitestPackage) {
return vitestPackage.h?.name === 'BaseSequencer'
}
/**
* Finds an export by its `.name` property in a minified vitest chunk.
* Minified export keys change across versions, so we search by function/class name.
* @param {object} pkg - The module exports object
* @param {string} name - The `.name` value to look for
* @returns {{ key: string, value: Function } | undefined}
*/
function findExportByName (pkg, name) {
for (const [key, value] of Object.entries(pkg)) {
if (value?.name === name) {
return { key, value }
}
}
}
function getBaseSequencerExport (vitestPackage) {
return findExportByName(vitestPackage, 'BaseSequencer')
}
function getChannelPromise (channelToPublishTo, frameworkVersion) {
return new Promise(resolve => {
channelToPublishTo.publish({ onDone: resolve, frameworkVersion })
})
}
function isCliApiPackage (vitestPackage) {
return !!findExportByName(vitestPackage, 'startVitest')
}
function getTestRunnerExport (testPackage) {
return findExportByName(testPackage, 'VitestTestRunner') || findExportByName(testPackage, 'TestRunner')
}
function getVitestExport (vitestPackage) {
return findExportByName(vitestPackage, 'Vitest')
}
function getForksPoolWorkerExport (vitestPackage) {
return findExportByName(vitestPackage, 'ForksPoolWorker')
}
function getThreadsPoolWorkerExport (vitestPackage) {
return findExportByName(vitestPackage, 'ThreadsPoolWorker')
}
function getSessionStatus (state) {
if (state.getCountOfFailedTests() > 0) {
return 'fail'
}
if (state.pathsSet.size === 0) {
return 'skip'
}
return 'pass'
}
// From https://github.com/vitest-dev/vitest/blob/51c04e2f44d91322b334f8ccbcdb368facc3f8ec/packages/runner/src/run.ts#L243-L250
function getVitestTestStatus (test, retryCount) {
if (test.result.state !== 'fail' && (!test.repeats || (test.retry ?? 0) === retryCount)) {
return 'pass'
}
return 'fail'
}
function getTypeTasks (fileTasks, type = 'test') {
const typeTasks = []
function getTasks (tasks) {
for (const task of tasks) {
if (task.type === type) {
typeTasks.push(task)
} else if (task.tasks) {
getTasks(task.tasks)
}
}
}
getTasks(fileTasks)
return typeTasks
}
function getTestName (task) {
let testName = task.name
let currentTask = task.suite
while (currentTask) {
if (currentTask.name) {
testName = `${currentTask.name} ${testName}`
}
currentTask = currentTask.suite
}
return testName
}
function getFinalAttemptToFixStatus (task, state, isSwitchedStatus, testCtx) {
if (isSwitchedStatus && attemptToFixTasks.has(task) && testCtx?.status) {
return testCtx.status
}
return state === 'fail' ? 'fail' : 'pass'
}
function recordFinalAttemptToFixExecution (task, status, providedContext) {
const statuses = attemptToFixTaskToStatuses.get(task)
if (statuses && statuses.length <= providedContext.testManagementAttemptToFixRetries) {
statuses.push(status)
}
recordAttemptToFixExecution(attemptToFixExecutions, {
testSuite: getTestSuitePath(task.file.filepath, process.cwd()),
testName: getTestName(task),
status,
isDisabled: disabledTasks.has(task),
isQuarantined: quarantinedTasks.has(task),
})
}
/**
* Wraps a function so it runs inside the current test span context.
* @param {object} task
* @param {Function} fn
* @returns {Function}
*/
function wrapTestScopedFn (task, fn) {
return shimmer.wrapFunction(fn, fn => function (...args) {
return testFnCh.runStores(taskToCtx.get(task), () => fn.apply(this, args))
})
}
/**
* Wraps a `beforeEach` cleanup callback so it inherits the test span context.
* Vitest allows `beforeEach` to return a cleanup function, including via a promise.
* @param {object} task
* @param {unknown} result
* @returns {unknown}
*/
function wrapBeforeEachCleanupResult (task, result) {
if (typeof result === 'function') {
return wrapTestScopedFn(task, result)
}
if (result && typeof result.then === 'function') {
return result.then(cleanupFn => wrapBeforeEachCleanupResult(task, cleanupFn))
}
return result
}
function getWorkspaceProject (ctx) {
return ctx.getCoreWorkspaceProject
? ctx.getCoreWorkspaceProject()
: ctx.getRootProject()
}
function setProvidedContext (ctx, values, warningMessage) {
try {
Object.assign(getWorkspaceProject(ctx)._provided, values)
} catch {
log.warn(warningMessage)
}
}
function getTestFilepathsFromSpecifications (testSpecifications) {
if (!Array.isArray(testSpecifications) || !testSpecifications.length) {
return
}
return testSpecifications.map(testSpecification => {
const testFile = Array.isArray(testSpecification) ? testSpecification[1] : testSpecification
return testFile?.moduleId || testFile?.filepath || testFile
})
}
function getTestFilepaths (ctx, testSpecifications) {
const testFilepaths = getTestFilepathsFromSpecifications(testSpecifications)
if (testFilepaths) {
return testFilepaths
}
const getFilePaths = ctx.getTestFilepaths || ctx._globTestFilepaths
return getFilePaths.call(ctx)
}
function wrapCoverageProvider (ctx) {
const { coverageProvider } = ctx
if (!coverageProvider?.generateCoverage || coverageWrappedProviders.has(coverageProvider)) {
return
}
coverageWrappedProviders.add(coverageProvider)
// Capture coverage root directory from config (default is 'coverage' in cwd)
try {
const coverageConfig = ctx.config?.coverage
const reportsDirectory = coverageConfig?.reportsDirectory || 'coverage'
const rootDir = ctx.config?.root || process.cwd()
coverageRootDir = path.isAbsolute(reportsDirectory) ? reportsDirectory : path.join(rootDir, reportsDirectory)
} catch {
// Fallback to cwd if we can't get config
coverageRootDir = process.cwd()
}
shimmer.wrap(coverageProvider, 'generateCoverage', generateCoverage => async function () {
const totalCodeCoverage = await generateCoverage.apply(this, arguments)
try {
testCodeCoverageLinesTotal = totalCodeCoverage.getCoverageSummary().lines.pct
} catch {
// ignore errors
}
return totalCodeCoverage
})
}
function wrapSessionFinish (ctx) {
if (finishWrappedContexts.has(ctx)) {
return
}
finishWrappedContexts.add(ctx)
shimmer.wrap(ctx, 'exit', getFinishWrapper)
shimmer.wrap(ctx, 'close', getFinishWrapper)
}
async function runMainProcessSetup (ctx, frameworkVersion, testSpecifications) {
if (!testSessionFinishCh.hasSubscribers) {
return
}
try {
const { err, libraryConfig } = await getChannelPromise(libraryConfigurationCh, frameworkVersion)
if (!err) {
isFlakyTestRetriesEnabled = libraryConfig.isFlakyTestRetriesEnabled
flakyTestRetriesCount = libraryConfig.flakyTestRetriesCount
isEarlyFlakeDetectionEnabled = libraryConfig.isEarlyFlakeDetectionEnabled
earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries
earlyFlakeDetectionSlowTestRetries = libraryConfig.earlyFlakeDetectionSlowTestRetries ?? {}
isDiEnabled = libraryConfig.isDiEnabled
isKnownTestsEnabled = libraryConfig.isKnownTestsEnabled
isTestManagementTestsEnabled = libraryConfig.isTestManagementEnabled
testManagementAttemptToFixRetries = libraryConfig.testManagementAttemptToFixRetries
isImpactedTestsEnabled = libraryConfig.isImpactedTestsEnabled
}
} catch {
isFlakyTestRetriesEnabled = false
isEarlyFlakeDetectionEnabled = false
isDiEnabled = false
isKnownTestsEnabled = false
isImpactedTestsEnabled = false
}
if (testSessionConfigurationCh.hasSubscribers) {
const { testSessionId, testModuleId, testCommand } = await getChannelPromise(
testSessionConfigurationCh,
frameworkVersion
)
setProvidedContext(ctx, {
_ddTestSessionId: testSessionId,
_ddTestModuleId: testModuleId,
_ddTestCommand: testCommand,
}, 'Could not send test session configuration to workers.')
}
const {
knownTestsResponse,
testManagementTestsResponse,
} = await getTestOptimizationRequestResults({
isKnownTestsEnabled,
isTestManagementTestsEnabled,
getKnownTests: () => getChannelPromise(knownTestsCh),
getTestManagementTests: () => getChannelPromise(testManagementTestsCh),
})
if (isFlakyTestRetriesEnabled && !ctx.config.retry && flakyTestRetriesCount > 0) {
ctx.config.retry = flakyTestRetriesCount
setProvidedContext(ctx, {
_ddIsFlakyTestRetriesEnabled: isFlakyTestRetriesEnabled,
_ddFlakyTestRetriesCount: flakyTestRetriesCount,
}, 'Could not send library configuration to workers.')
}
if (isKnownTestsEnabled) {
const currentKnownTestsResponse = knownTestsResponse || await getChannelPromise(knownTestsCh)
if (currentKnownTestsResponse.err) {
isEarlyFlakeDetectionEnabled = false
} else {
const knownTests = currentKnownTestsResponse.knownTests
const testFilepaths = await getTestFilepaths(ctx, testSpecifications)
if (isValidKnownTests(knownTests)) {
isEarlyFlakeDetectionFaultyCh.publish({
knownTests: knownTests.vitest,
testFilepaths,
onDone: (isFaulty) => {
isEarlyFlakeDetectionFaulty = isFaulty
},
})
if (isEarlyFlakeDetectionFaulty) {
isEarlyFlakeDetectionEnabled = false
log.warn('New test detection is disabled because the number of new tests is too high.')
} else {
setProvidedContext(ctx, {
_ddIsKnownTestsEnabled: isKnownTestsEnabled,
_ddKnownTests: knownTests,
_ddIsEarlyFlakeDetectionEnabled: isEarlyFlakeDetectionEnabled,
_ddEarlyFlakeDetectionNumRetries:
getConfiguredEfdRetryCount(earlyFlakeDetectionSlowTestRetries, earlyFlakeDetectionNumRetries),
_ddEarlyFlakeDetectionSlowTestRetries: earlyFlakeDetectionSlowTestRetries,
}, 'Could not send known tests to workers so Early Flake Detection will not work.')
}
} else {
isEarlyFlakeDetectionFaulty = true
isEarlyFlakeDetectionEnabled = false
}
}
}
if (isDiEnabled) {
setProvidedContext(ctx, {
_ddIsDiEnabled: isDiEnabled,
}, 'Could not send Dynamic Instrumentation configuration to workers.')
}
if (isTestManagementTestsEnabled) {
const { err, testManagementTests: receivedTestManagementTests } =
testManagementTestsResponse || await getChannelPromise(testManagementTestsCh)
if (err) {
isTestManagementTestsEnabled = false
log.error('Could not get test management tests.')
} else {
setProvidedContext(ctx, {
_ddIsTestManagementTestsEnabled: isTestManagementTestsEnabled,
_ddTestManagementAttemptToFixRetries: testManagementAttemptToFixRetries,
_ddTestManagementTests: receivedTestManagementTests,
}, 'Could not send test management tests to workers so Test Management will not work.')
}
}
if (isImpactedTestsEnabled) {
const { err, modifiedFiles } = await getChannelPromise(modifiedFilesCh)
if (err) {
log.error('Could not get modified tests.')
} else {
setProvidedContext(ctx, {
_ddIsImpactedTestsEnabled: isImpactedTestsEnabled,
_ddModifiedFiles: modifiedFiles,
}, 'Could not send modified tests to workers so Impacted Tests will not work.')
}
}
wrapCoverageProvider(ctx)
wrapSessionFinish(ctx)
}
function ensureMainProcessSetup (ctx, frameworkVersion, testSpecifications) {
let setupPromise = mainProcessSetupPromises.get(ctx)
if (!setupPromise) {
setupPromise = runMainProcessSetup(ctx, frameworkVersion, testSpecifications)
mainProcessSetupPromises.set(ctx, setupPromise)
}
return setupPromise
}
function getSortWrapper (sort, frameworkVersion) {
return async function () {
await ensureMainProcessSetup(this.ctx, frameworkVersion, arguments[0])
return sort.apply(this, arguments)
}
}
function getFinishWrapper (exitOrClose) {
let isClosed = false
return async function () {
if (isClosed) { // needed because exit calls close
return exitOrClose.apply(this, arguments)
}
isClosed = true
if (!testSessionFinishCh.hasSubscribers) {
return exitOrClose.apply(this, arguments)
}
let onFinish
const flushPromise = new Promise(resolve => {
onFinish = resolve
})
const failedSuites = this.state.getFailedFilepaths()
let error
if (failedSuites.length) {
error = new Error(`Test suites failed: ${failedSuites.length}.`)
}
testSessionFinishCh.publish({
status: getSessionStatus(this.state),
testCodeCoverageLinesTotal,
error,
isEarlyFlakeDetectionEnabled,
isEarlyFlakeDetectionFaulty,
isTestManagementTestsEnabled,
vitestPool,
onFinish,
})
logTestOptimizationSummary({ attemptToFixExecutions, newTestsWithDynamicNames })
loggedAttemptToFixTests.clear()
await flushPromise
// If coverage was generated, publish coverage report channel for upload
if (coverageRootDir && codeCoverageReportCh.hasSubscribers) {
await new Promise((resolve) => {
codeCoverageReportCh.publish({ rootDir: coverageRootDir, onDone: resolve })
})
}
return exitOrClose.apply(this, arguments)
}
}
function getCliOrStartVitestWrapper (frameworkVersion) {
return function (oldCliOrStartVitest) {
return function (...args) {
if (!testSessionStartCh.hasSubscribers || isSessionStarted) {
return oldCliOrStartVitest.apply(this, args)
}
isSessionStarted = true
testSessionStartCh.publish({ command: getTestCommand(), frameworkVersion })
return oldCliOrStartVitest.apply(this, args)
}
}
}
function wrapVitestRunFiles (Vitest, frameworkVersion) {
if (!Vitest?.prototype?.runFiles) {
return
}
shimmer.wrap(Vitest.prototype, 'runFiles', runFiles => async function (testSpecifications) {
await ensureMainProcessSetup(this, frameworkVersion, testSpecifications)
return runFiles.apply(this, arguments)
})
}
function getCreateCliWrapper (vitestPackage, frameworkVersion) {
const createCliExport = findExportByName(vitestPackage, 'createCLI')
if (!createCliExport) {
return vitestPackage
}
shimmer.wrap(vitestPackage, createCliExport.key, getCliOrStartVitestWrapper(frameworkVersion))
return vitestPackage
}
function threadHandler (thread) {
const { runtime } = thread
let workerProcess
if (runtime === 'child_process') {
vitestPool = 'child_process'
workerProcess = thread.process
} else if (runtime === 'worker_threads') {
vitestPool = 'worker_threads'
workerProcess = thread.thread
} else {
vitestPool = 'unknown'
}
if (!workerProcess) {
log.error('Vitest error: could not get process or thread from TinyPool#run')
return
}
if (workerProcesses.has(workerProcess)) {
return
}
workerProcesses.add(workerProcess)
workerProcess.on('message', (message) => {
if (message.__tinypool_worker_message__ && message.data) {
if (message.interprocessCode === VITEST_WORKER_TRACE_PAYLOAD_CODE) {
collectTestOptimizationSummariesFromTraces(message.data, {
newTestsWithDynamicNames,
attemptToFixExecutions,
})
workerReportTraceCh.publish(message.data)
} else if (message.interprocessCode === VITEST_WORKER_LOGS_PAYLOAD_CODE) {
workerReportLogsCh.publish(message.data)
}
}
})
}
function wrapTinyPoolRun (TinyPool) {
shimmer.wrap(TinyPool.prototype, 'run', run => async function () {
// We have to do this before and after because the threads list gets recycled, that is, the processes are re-created
// eslint-disable-next-line unicorn/no-array-for-each
this.threads.forEach(threadHandler)
const runResult = await run.apply(this, arguments)
// eslint-disable-next-line unicorn/no-array-for-each
this.threads.forEach(threadHandler)
return runResult
})
}
addHook({
name: 'tinypool',
// version from tinypool@0.8 was used in vitest@1.6.0
versions: ['>=0.8.0'],
}, (TinyPool) => {
wrapTinyPoolRun(TinyPool)
return TinyPool
})
function getWrappedOn (on) {
return function (event, callback) {
if (event !== 'message') {
return on.apply(this, arguments)
}
// `arguments[1]` is the callback function, which
// we modify to intercept our messages to not interfere
// with vitest's own messages
arguments[1] = shimmer.wrapFunction(callback, callback => function (message) {
if (message.type !== 'Buffer' && Array.isArray(message)) {
const [interprocessCode, data] = message
if (interprocessCode === VITEST_WORKER_TRACE_PAYLOAD_CODE) {
collectTestOptimizationSummariesFromTraces(data, {
newTestsWithDynamicNames,
attemptToFixExecutions,
})
workerReportTraceCh.publish(data)
} else if (interprocessCode === VITEST_WORKER_LOGS_PAYLOAD_CODE) {
workerReportLogsCh.publish(data)
}
// If we execute the callback vitest crashes, as the message is not supported
return
}
return callback.apply(this, arguments)
})
return on.apply(this, arguments)
}
}
function getStartVitestWrapper (cliApiPackage, frameworkVersion) {
if (!isCliApiPackage(cliApiPackage)) {
return cliApiPackage
}
const startVitestExport = findExportByName(cliApiPackage, 'startVitest')
shimmer.wrap(cliApiPackage, startVitestExport.key, getCliOrStartVitestWrapper(frameworkVersion))
const vitest = getVitestExport(cliApiPackage)
if (vitest) {
wrapVitestRunFiles(vitest.value, frameworkVersion)
}
const forksPoolWorker = getForksPoolWorkerExport(cliApiPackage)
if (forksPoolWorker) {
// function is async
shimmer.wrap(forksPoolWorker.value.prototype, 'start', start => function (...args) {
vitestPool = 'child_process'
this.env.DD_VITEST_WORKER = '1'
return start.apply(this, args)
})
shimmer.wrap(forksPoolWorker.value.prototype, 'on', getWrappedOn)
}
const threadsPoolWorker = getThreadsPoolWorkerExport(cliApiPackage)
if (threadsPoolWorker) {
// function is async
shimmer.wrap(threadsPoolWorker.value.prototype, 'start', start => function (...args) {
vitestPool = 'worker_threads'
this.env.DD_VITEST_WORKER = '1'
return start.apply(this, args)
})
shimmer.wrap(threadsPoolWorker.value.prototype, 'on', getWrappedOn)
}
return cliApiPackage
}
function wrapVitestTestRunner (VitestTestRunner) {
// `onBeforeRunTask` is run before any repetition or attempt is run
// `onBeforeRunTask` is an async function
shimmer.wrap(VitestTestRunner.prototype, 'onBeforeRunTask', onBeforeRunTask => function (task) {
const testName = getTestName(task)
const {
knownTests,
isEarlyFlakeDetectionEnabled,
isKnownTestsEnabled,
numRepeats,
isTestManagementTestsEnabled,
testManagementAttemptToFixRetries,
testManagementTests,
isImpactedTestsEnabled,
modifiedFiles,
} = getProvidedContext()
if (isTestManagementTestsEnabled) {
isAttemptToFixCh.publish({
testManagementTests,
testSuiteAbsolutePath: task.file.filepath,
testName,
onDone: (isAttemptToFix) => {
if (isAttemptToFix) {
isRetryReasonAttemptToFix = task.repeats !== testManagementAttemptToFixRetries
task.repeats = testManagementAttemptToFixRetries
attemptToFixTasks.add(task)
attemptToFixTaskToStatuses.set(task, [])
}
},
})
isDisabledCh.publish({
testManagementTests,
testSuiteAbsolutePath: task.file.filepath,
testName,
onDone: (isTestDisabled) => {
if (isTestDisabled) {
disabledTasks.add(task)
if (!attemptToFixTasks.has(task)) {
// we only actually skip if the test is not being attempted to be fixed
task.mode = 'skip'
}
}
},
})
}
if (isImpactedTestsEnabled) {
isModifiedCh.publish({
modifiedFiles,
testSuiteAbsolutePath: task.file.filepath,
onDone: (isImpacted) => {
if (isImpacted) {
if (isEarlyFlakeDetectionEnabled) {
isRetryReasonEfd = true
task.repeats = numRepeats
}
modifiedTasks.add(task)
taskToStatuses.set(task, [])
}
},
})
}
if (isKnownTestsEnabled) {
isNewTestCh.publish({
knownTests,
testSuiteAbsolutePath: task.file.filepath,
testName,
onDone: (isNew) => {
if (isNew && !attemptToFixTasks.has(task)) {
if (isEarlyFlakeDetectionEnabled && !modifiedTasks.has(task)) {
isRetryReasonEfd = true
task.repeats = numRepeats
}
newTasks.add(task)
taskToStatuses.set(task, [])
if (DYNAMIC_NAME_RE.test(testName)) {
dynamicNameTasks.add(task)
}
}
},
})
}
return onBeforeRunTask.apply(this, arguments)
})
// `onAfterRunTask` is run after all repetitions or attempts are run
// `onAfterRunTask` is an async function
shimmer.wrap(VitestTestRunner.prototype, 'onAfterRunTask', onAfterRunTask => function (task) {
const { isEarlyFlakeDetectionEnabled, isTestManagementTestsEnabled } = getProvidedContext()
if (isTestManagementTestsEnabled) {
const isAttemptingToFix = attemptToFixTasks.has(task)
const isQuarantined = quarantinedTasks.has(task)
if (isAttemptingToFix) {
const statuses = attemptToFixTaskToStatuses.get(task)
if (task.result.state === 'pass' && statuses?.includes('fail')) {
switchedStatuses.add(task)
task.result.state = 'fail'
}
}
if (!isAttemptingToFix && isQuarantined) {
if (task.result.state === 'fail') {
switchedStatuses.add(task)
}
task.result.state = 'pass'
}
}
if (isEarlyFlakeDetectionEnabled && taskToStatuses.has(task) && !attemptToFixTasks.has(task)) {
const statuses = taskToStatuses.get(task)
// If the test has passed at least once, we consider it passed
if (statuses.includes('pass')) {
if (task.result.state === 'fail') {
switchedStatuses.add(task)
}
task.result.state = 'pass'
}
}
return onAfterRunTask.apply(this, arguments)
})
// test start (only tests that are not marked as skip or todo)
// `onBeforeTryTask` is run for every repetition and attempt of the test
shimmer.wrap(VitestTestRunner.prototype, 'onBeforeTryTask', onBeforeTryTask => async function (task, retryInfo) {
if (!testPassCh.hasSubscribers && !testErrorCh.hasSubscribers && !testSkipCh.hasSubscribers) {
return onBeforeTryTask.apply(this, arguments)
}
const testName = getTestName(task)
let isNew = false
let isQuarantined = false
const {
isKnownTestsEnabled,
isEarlyFlakeDetectionEnabled,
isDiEnabled,
isTestManagementTestsEnabled,
testManagementTests,
isFlakyTestRetriesEnabled,
slowTestRetries,
} = getProvidedContext()
if (isKnownTestsEnabled) {
isNew = newTasks.has(task)
}
if (isTestManagementTestsEnabled) {
isQuarantinedCh.publish({
testManagementTests,
testSuiteAbsolutePath: task.file.filepath,
testName,
onDone: (isTestQuarantined) => {
isQuarantined = isTestQuarantined
if (isTestQuarantined) {
quarantinedTasks.add(task)
}
},
})
}
const { retry: numAttempt, repeats: numRepetition } = retryInfo
const isEfdManagedTask = isEarlyFlakeDetectionEnabled && taskToStatuses.has(task) && !attemptToFixTasks.has(task)
if (isEfdManagedTask && numRepetition > 0 && !efdDeterminedRetries.has(task)) {
const previousExecutionStart = efdExecutionStartByTask.get(task)
const duration = previousExecutionStart === undefined
? task.result?.duration ?? 0
: performance.now() - previousExecutionStart
const retryCount = getEfdRetryCount(duration, slowTestRetries)
efdDeterminedRetries.set(task, retryCount)
task.repeats = retryCount
if (retryCount === 0) {
efdSlowAbortedTasks.add(task)
}
}
const efdRetryCount = efdDeterminedRetries.get(task)
if (isEfdManagedTask && efdRetryCount !== undefined && numRepetition > efdRetryCount) {
if (task.result) {
efdSkippedRetryResults.set(task, {
...task.result,
errors: task.result.errors?.slice(),
})
}
if (vitestSetFn) {
const noop = function () {}
noop.__ddTraceWrapped = true
vitestSetFn(task, noop)
}
return onBeforeTryTask.apply(this, arguments)
}
if (isEfdManagedTask) {
efdExecutionStartByTask.set(task, performance.now())
}
// We finish the previous test here because we know it has failed already
if (numAttempt > 0) {
const shouldWaitForHitProbe = isDiEnabled && numAttempt > 1
if (shouldWaitForHitProbe) {
await waitForHitProbe()
}
const promises = {}
const shouldSetProbe = isDiEnabled && numAttempt === 1
const ctx = taskToCtx.get(task)
const testError = task.result?.errors?.[0]
if (ctx) {
testErrorCh.publish({
error: testError,
shouldSetProbe,
promises,
...ctx.currentStore,
})
// We wait for the probe to be set
if (promises.setProbePromise) {
await promises.setProbePromise
}
}
}
const lastExecutionStatus = task.result.state
const isAtf = attemptToFixTasks.has(task)
const shouldTrackStatuses = isEarlyFlakeDetectionEnabled || isAtf
const shouldFlipStatus = isEarlyFlakeDetectionEnabled || isAtf
const statuses = isAtf ? attemptToFixTaskToStatuses.get(task) : taskToStatuses.get(task)
// These clauses handle task.repeats, whether EFD is enabled or not
// The only thing that EFD does is to forcefully pass the test if it has passed at least once
if (numRepetition > 0 && numRepetition < task.repeats) { // it may or may have not failed
// Here we finish the earlier iteration,
// as long as it's not the _last_ iteration (which will be finished normally)
const ctx = taskToCtx.get(task)
if (ctx) {
if (lastExecutionStatus === 'fail') {
const testError = task.result?.errors?.[0]
testErrorCh.publish({ error: testError, ...ctx.currentStore })
} else {
testPassCh.publish({ task, ...ctx.currentStore })
}
if (shouldTrackStatuses && statuses) {
statuses.push(lastExecutionStatus)
}
if (shouldFlipStatus) {
// If we don't "reset" the result.state to "pass", once a repetition fails,
// vitest will always consider the test as failed, so we can't read the actual status
// This means that we change vitest's behavior:
// if the last attempt passes, vitest would consider the test as failed
// but after this change, it will consider the test as passed
task.result.state = 'pass'
}
}
} else if (numRepetition === task.repeats) {
if (shouldTrackStatuses && statuses) {
statuses.push(lastExecutionStatus)
}
const ctx = taskToCtx.get(task)
if (lastExecutionStatus === 'fail') {
const testError = task.result?.errors?.[0]
testErrorCh.publish({ error: testError, ...ctx.currentStore })
} else {
testPassCh.publish({ task, ...ctx.currentStore })
}
if (shouldFlipStatus) {
task.result.state = 'pass'
}
}
const isRetryReasonAtr = numAttempt > 0 &&
isFlakyTestRetriesEnabled &&
!isRetryReasonAttemptToFix &&
!isRetryReasonEfd
const ctx = {
testName,
testSuiteAbsolutePath: task.file.filepath,
isRetry: numAttempt > 0 || numRepetition > 0,
isRetryReasonEfd,
isRetryReasonAttemptToFix: isRetryReasonAttemptToFix && numRepetition > 0,
isNew,
hasDynamicName: dynamicNameTasks.has(task),
mightHitProbe: isDiEnabled && numAttempt > 0,
isAttemptToFix: attemptToFixTasks.has(task),
isDisabled: disabledTasks.has(task),
isQuarantined,
isRetryReasonAtr,
isModified: modifiedTasks.has(task),
}
taskToCtx.set(task, ctx)
if (attemptToFixTasks.has(task)) {
logAttemptToFixTestExecution(
getTestSuitePath(task.file.filepath, process.cwd()),
testName,
loggedAttemptToFixTests
)
}
testStartCh.runStores(ctx, () => {})
// Wrap the test function so it runs inside the test span context.
// Without this, HTTP requests during test execution become orphaned root spans.
if (vitestGetFn && vitestSetFn) {
const originalFn = vitestGetFn(task)
if (originalFn && !originalFn.__ddTraceWrapped) {
const wrappedFn = wrapTestScopedFn(task, originalFn)
wrappedFn.__ddTraceWrapped = true
vitestSetFn(task, wrappedFn)
}
}
// Wrap beforeEach/afterEach hooks so they also run inside the test span context.
// In vitest 4+, hooks are in a WeakMap accessed via getHooks(). In older versions, they're on suite.hooks.
let currentSuite = task.suite
while (currentSuite) {
const hooks = vitestGetHooks ? vitestGetHooks(currentSuite) : currentSuite.hooks
if (hooks) {
for (const hookType of ['beforeEach', 'afterEach']) {
const hookArray = hooks[hookType]
if (!hookArray) continue
for (let i = 0; i < hookArray.length; i++) {
const currentFn = hookArray[i]
const originalFn = originalHookFns.get(currentFn) || currentFn
const wrappedFn = shimmer.wrapFunction(originalFn, fn => function (...args) {
const result = testFnCh.runStores(taskToCtx.get(task), () => fn.apply(this, args))
if (hookType === 'beforeEach') {
return wrapBeforeEachCleanupResult(task, result)
}
return result
})
originalHookFns.set(wrappedFn, originalFn)
hookArray[i] = wrappedFn
}
}
}
currentSuite = currentSuite.suite
}
return onBeforeTryTask.apply(this, arguments)
})
// test finish (only passed tests)
shimmer.wrap(VitestTestRunner.prototype, 'onAfterTryTask', onAfterTryTask =>
async function (task, retryInfo) {
if (!testPassCh.hasSubscribers && !testErrorCh.hasSubscribers && !testSkipCh.hasSubscribers) {
return onAfterTryTask.apply(this, arguments)
}
const result = await onAfterTryTask.apply(this, arguments)
const {
isEarlyFlakeDetectionEnabled,
testManagementAttemptToFixRetries,
slowTestRetries,
} = getProvidedContext()
const status = getVitestTestStatus(task, retryInfo.retry)
const ctx = taskToCtx.get(task)
const { isDiEnabled } = getProvidedContext()
if (efdSkippedRetryResults.has(task)) {
task.result = efdSkippedRetryResults.get(task)
efdSkippedRetryResults.delete(task)
return result
}
if (isDiEnabled && retryInfo.retry > 1) {
await waitForHitProbe()
}
if (
isEarlyFlakeDetectionEnabled &&
(retryInfo.repeats ?? 0) === 0 &&
taskToStatuses.has(task) &&
!attemptToFixTasks.has(task) &&
!efdDeterminedRetries.has(task)
) {
const executionStart = efdExecutionStartByTask.get(task)
const duration = executionStart === undefined ? task.result?.duration ?? 0 : performance.now() - executionStart
const retryCount = getEfdRetryCount(duration, slowTestRetries)
efdDeterminedRetries.set(task, retryCount)
task.repeats = retryCount
if (retryCount === 0) {
efdSlowAbortedTasks.add(task)
}
}
let attemptToFixPassed = false
let attemptToFixFailed = false
if (attemptToFixTasks.has(task)) {
const statuses = attemptToFixTaskToStatuses.get(task)
if (statuses.length === testManagementAttemptToFixRetries) {
if (status === 'pass' && statuses.every(status => status === 'pass')) {
attemptToFixPassed = true
} else if (status === 'fail' || statuses.includes('fail')) {
attemptToFixFailed = true
}
}
}
if (ctx) {
// We don't finish here because the test might fail in a later hook (afterEach)
ctx.status = status
ctx.task = task
ctx.attemptToFixPassed = attemptToFixPassed
ctx.attemptToFixFailed = attemptToFixFailed
testFinishTimeCh.runStores(ctx, () => {})
}
return result
})
}
function captureRunnerFunctions (pkg) {
if (vitestGetFn) return
const getFnExport = findExportByName(pkg, 'getFn')
const setFnExport = findExportByName(pkg, 'setFn')
if (getFnExport && setFnExport) {
vitestGetFn = getFnExport.value
vitestSetFn = setFnExport.value
}
const getHooksExport = findExportByName(pkg, 'getHooks')
if (getHooksExport) {
vitestGetHooks = getHooksExport.value
}
}
addHook({
name: 'vitest',
versions: ['>=4.0.0'],
filePattern: 'dist/chunks/test.*',
}, (testPackage) => {
const testRunner = getTestRunnerExport(testPackage)
if (!testRunner) {
return testPackage
}
captureRunnerFunctions(testPackage)
wrapVitestTestRunner(testRunner.value)
return testPackage
})
addHook({
name: '@vitest/runner',
versions: ['>=1.6.0'],
}, (runnerModule) => {
if (!vitestGetFn && runnerModule.getFn && runnerModule.setFn) {
vitestGetFn = runnerModule.getFn
vitestSetFn = runnerModule.setFn
}
if (!vitestGetHooks && runnerModule.getHooks) {
vitestGetHooks = runnerModule.getHooks
}
return runnerModule
})
addHook({
name: 'vitest',
versions: ['>=1.6.0 <4.0.0'],
file: 'dist/runners.js',
}, (vitestPackage) => {
const { VitestTestRunner } = vitestPackage
wrapVitestTestRunner(VitestTestRunner)
return vitestPackage
})
// There are multiple index* files across different versions of vitest,
// so we check for the existence of BaseSequencer to determine if we are in the right file
addHook({
name: 'vitest',
versions: ['>=1.6.0 <2.0.0'],
filePattern: 'dist/vendor/index.*',
}, (vitestPackage) => {
if (isReporterPackage(vitestPackage)) {
shimmer.wrap(vitestPackage.B.prototype, 'sort', getSortWrapper)
}
return vitestPackage
})
addHook({
name: 'vitest',
versions: ['>=2.0.0 <2.0.5'],
filePattern: 'dist/vendor/index.*',
}, (vitestPackage) => {
if (isReporterPackageNew(vitestPackage)) {
shimmer.wrap(vitestPackage.e.prototype, 'sort', getSortWrapper)
}
return vitestPackage
})
addHook({
name: 'vitest',
versions: ['>=2.0.5 <2.1.0'],
filePattern: 'dist/chunks/index.*',
}, (vitestPackage) => {
if (isReporterPackageNewest(vitestPackage)) {
shimmer.wrap(vitestPackage.h.prototype, 'sort', getSortWrapper)
}
return vitestPackage
})
addHook({
name: 'vitest',
versions: ['>=2.1.0 <3.0.0'],
filePattern: 'dist/chunks/RandomSequencer.*',
}, (randomSequencerPackage) => {
shimmer.wrap(randomSequencerPackage.B.prototype, 'sort', getSortWrapper)
return randomSequencerPackage
})
addHook({
name: 'vitest',
versions: ['>=3.0.9'],
filePattern: 'dist/chunks/coverage.*',
}, (coveragePackage) => {
const baseSequencer = getBaseSequencerExport(coveragePackage)
if (baseSequencer) {
shimmer.wrap(baseSequencer.value.prototype, 'sort', getSortWrapper)
}
return coveragePackage
})
addHook({
name: 'vitest',
versions: ['>=3.0.0 <3.0.9'],
filePattern: 'dist/chunks/resolveConfig.*',
}, (resolveConfigPackage) => {
shimmer.wrap(resolveConfigPackage.B.prototype, 'sort', getSortWrapper)
return resolveConfigPackage
})
// Can't specify file because compiled vitest includes hashes in their files
// Following 3 wrappers are for test session start
addHook({
name: 'vitest',
versions: ['>=1.6.0 <2.0.5'],
filePattern: 'dist/vendor/cac.*',
}, getCreateCliWrapper)
addHook({
name: 'vitest',
versions: ['>=2.0.5'],
filePattern: 'dist/chunks/cac.*',
}, getCreateCliWrapper)
addHook({
name: 'vitest',
versions: ['>=1.6.0 <2.0.5'],
filePattern: 'dist/vendor/cli-api.*',
}, getStartVitestWrapper)
addHook({
name: 'vitest',
versions: ['>=2.0.5'],
filePattern: 'dist/chunks/cli-api.*',
}, getStartVitestWrapper)
// test suite start and finish
// only relevant for workers
addHook({
name: '@vitest/runner',
versions: ['>=1.6.0'],
}, (vitestPackage, frameworkVersion) => {
shimmer.wrap(vitestPackage, 'startTests', startTests => async function (testPaths) {
let testSuiteError = null
if (!testSuiteFinishCh.hasSubscribers) {
return startTests.apply(this, arguments)
}
// From >=3.0.1, the first arguments changes from a string to an object containing the filepath
const testSuiteAbsolutePath = testPaths[0]?.filepath || testPaths[0]
const providedContext = getProvidedContext()
const testSuiteCtx = {
testSuiteAbsolutePath,
frameworkVersion,
testSessionId: providedContext.testSessionId,
testModuleId: providedContext.testModuleId,
testCommand: providedContext.testCommand,
}
testSuiteStartCh.runStores(testSuiteCtx, () => {})
const startTestsResponse = await startTests.apply(this, arguments)
let onFinish = null
const onFinishPromise = new Promise(resolve => {
onFinish = resolve
})
const testTasks = getTypeTasks(startTestsResponse[0].tasks)
// Only one test task per test, even if there are retries
for (const task of testTasks) {
const testCtx = taskToCtx.get(task)
const { result } = task
// We have to trick vitest into thinking that the test has passed
// but we want to report it as failed if it did fail
const isSwitchedStatus = switchedStatuses.has(task)
if (result) {
const { state, duration, errors } = result
const testError = errors?.[0]
if (attemptToFixTasks.has(task)) {
const status = getFinalAttemptToFixStatus(task, state, isSwitchedStatus, testCtx)
recordFinalAttemptToFixExecution(task, status, providedContext)
}
if (state === 'skip') { // programmatic skip
testSkipCh.publish({
testName: getTestName(task),
testSuiteAbsolutePath: task.file.filepath,
isNew: newTasks.has(task),
isDisabled: disabledTasks.has(task),
})
} else if (state === 'pass' && !isSwitchedStatus) {
if (testCtx) {
const isSkippedByTestManagement =
!attemptToFixTasks.has(task) && (disabledTasks.has(task) || quarantinedTasks.has(task))
testPassCh.publish({
task,
finalStatus: isSkippedByTestManagement ? 'skip' : 'pass',
earlyFlakeAbortReason: efdSlowAbortedTasks.has(task) ? 'slow' : undefined,
...testCtx.currentStore,
})
}
} else if (state === 'fail' || isSwitchedStatus) {
let hasFailedAllRetries = false
let attemptToFixFailed = false
if (attemptToFixTasks.has(task)) {
const statuses = attemptToFixTaskToStatuses.get(task)
if (statuses.includes('fail')) {
attemptToFixFailed = true
}
if (statuses.every(status => status === 'fail')) {
hasFailedAllRetries = true
}
}
// Check if all EFD retries failed
const isEfdRetry =
providedContext.isEarlyFlakeDetectionEnabled && (newTasks.has(task) || modifiedTasks.has(task))
if (isEfdRetry) {
const statuses = taskToStatuses.get(task)
const efdRetryCount = efdDeterminedRetries.get(task) ?? providedContext.numRepeats
// statuses only includes repetitions (not the initial run), so we check against retry count (not +1)
if (efdRetryCount > 0 && statuses && statuses.length === efdRetryCount &&
statuses.every(status => status === 'fail')) {
hasFailedAllRetries = true
}
}
// ATR: set hasFailedAllRetries when all auto test retries were exhausted and every attempt failed
const isAtrRetry = providedContext.isFlakyTestRetriesEnabled && !attemptToFixTasks.has(task) &&
!newTasks.has(task) && !modifiedTasks.has(task)
if (isAtrRetry) {
const maxRetries = providedContext.flakyTestRetriesCount ?? 0
if (maxRetries > 0 && task.result?.retryCount === maxRetries) {
hasFailedAllRetries = true
}
}
if (testCtx) {
const isRetry = task.result?.retryCount > 0
// `duration` is the duration of all the retries, so it can't be used if there are retries
let finalStatus
if (isSwitchedStatus) {
if (!attemptToFixTasks.has(task) && (disabledTasks.has(task) || quarantinedTasks.has(task))) {
finalStatus = 'skip'
} else if (isAtrRetry || isEfdRetry) {
finalStatus = hasFailedAllRetries ? 'fail' : 'pass'
} else if (attemptToFixTasks.has(task)) {
finalStatus = attemptToFixFailed ? 'fail' : 'pass'
} else {
finalStatus = undefined
}
} else {
finalStatus = 'fail'
}
testErrorCh.publish({
duration: isRetry ? undefined : duration,
error: testError,
hasFailedAllRetries,
attemptToFixFailed,
finalStatus,
earlyFlakeAbortReason: efdSlowAbortedTasks.has(task) ? 'slow' : undefined,
...testCtx.currentStore,
})
}
if (errors?.length) {
testSuiteError = testError // we store the error to bubble it up to the suite
}
}
} else { // test.skip or test.todo
testSkipCh.publish({
testName: getTestName(task),
testSuiteAbsolutePath: task.file.filepath,
isNew: newTasks.has(task),
isDisabled: disabledTasks.has(task),
})
}
}
const testSuiteResult = startTestsResponse[0].result
if (testSuiteResult.errors?.length) { // Errors from root level hooks
testSuiteError = testSuiteResult.errors[0]
} else if (testSuiteResult.state === 'fail') { // Errors from `describe` level hooks
const suiteTasks = getTypeTasks(startTestsResponse[0].tasks, 'suite')
const failedSuites = suiteTasks.filter(task