dd-trace
Version:
Datadog APM tracing client for JavaScript
1,532 lines (1,356 loc) • 60.7 kB
JavaScript
'use strict'
// Capture real timers at module load time, before any test can install fake timers.
const realSetTimeout = setTimeout
const path = require('node:path')
const { performance } = require('node:perf_hooks')
const shimmer = require('../../datadog-shimmer')
const log = require('../../dd-trace/src/log')
const {
VITEST_WORKER_TRACE_PAYLOAD_CODE,
VITEST_WORKER_LOGS_PAYLOAD_CODE,
DYNAMIC_NAME_RE,
getTestSuitePath,
getEfdRetryCount,
getMaxEfdRetryCount,
recordAttemptToFixExecution,
collectTestOptimizationSummariesFromTraces,
logAttemptToFixTestExecution,
logTestOptimizationSummary,
getTestOptimizationRequestResults,
} = require('../../dd-trace/src/plugins/util/test')
const { addHook, channel } = require('./helpers/instrument')
// test hooks
const testStartCh = channel('ci:vitest:test:start')
const testFinishTimeCh = channel('ci:vitest:test:finish-time')
const testPassCh = channel('ci:vitest:test:pass')
const testErrorCh = channel('ci:vitest:test:error')
const testSkipCh = channel('ci:vitest:test:skip')
const isNewTestCh = channel('ci:vitest:test:is-new')
const isAttemptToFixCh = channel('ci:vitest:test:is-attempt-to-fix')
const isDisabledCh = channel('ci:vitest:test:is-disabled')
const isQuarantinedCh = channel('ci:vitest:test:is-quarantined')
const isModifiedCh = channel('ci:vitest:test:is-modified')
const testFnCh = channel('ci:vitest:test:fn')
// test suite hooks
const testSuiteStartCh = channel('ci:vitest:test-suite:start')
const testSuiteFinishCh = channel('ci:vitest:test-suite:finish')
const testSuiteErrorCh = channel('ci:vitest:test-suite:error')
// test session hooks
const testSessionStartCh = channel('ci:vitest:session:start')
const testSessionFinishCh = channel('ci:vitest:session:finish')
const testSessionConfigurationCh = channel('ci:vitest:session:configuration')
const libraryConfigurationCh = channel('ci:vitest:library-configuration')
const knownTestsCh = channel('ci:vitest:known-tests')
const isEarlyFlakeDetectionFaultyCh = channel('ci:vitest:is-early-flake-detection-faulty')
const testManagementTestsCh = channel('ci:vitest:test-management-tests')
const modifiedFilesCh = channel('ci:vitest:modified-files')
const workerReportTraceCh = channel('ci:vitest:worker-report:trace')
const workerReportLogsCh = channel('ci:vitest:worker-report:logs')
const codeCoverageReportCh = channel('ci:vitest:coverage-report')
const taskToCtx = new WeakMap()
const taskToStatuses = new WeakMap()
const taskToReportedErrorCount = new WeakMap()
const attemptToFixTaskToStatuses = new WeakMap()
const originalHookFns = new WeakMap()
const newTasks = new WeakSet()
const dynamicNameTasks = new WeakSet()
const newTestsWithDynamicNames = new Set()
const disabledTasks = new WeakSet()
const quarantinedTasks = new WeakSet()
const attemptToFixTasks = new WeakSet()
const modifiedTasks = new WeakSet()
const efdDeterminedRetries = new WeakMap()
const efdSlowAbortedTasks = new WeakSet()
const efdExecutionStartByTask = new WeakMap()
const efdSkippedRetryResults = new WeakMap()
const attemptToFixExecutions = new Map()
const loggedAttemptToFixTests = new Set()
let isRetryReasonEfd = false
let isRetryReasonAttemptToFix = false
const switchedStatuses = new WeakSet()
const workerProcesses = new WeakSet()
const mainProcessSetupPromises = new WeakMap()
const coverageWrappedProviders = new WeakSet()
const finishWrappedContexts = new WeakSet()
let isFlakyTestRetriesEnabled = false
let flakyTestRetriesCount = 0
let isEarlyFlakeDetectionEnabled = false
let earlyFlakeDetectionNumRetries = 0
let earlyFlakeDetectionSlowTestRetries = {}
let isEarlyFlakeDetectionFaulty = false
let isKnownTestsEnabled = false
let isTestManagementTestsEnabled = false
let isImpactedTestsEnabled = false
let vitestGetFn = null
let vitestSetFn = null
let vitestGetHooks = null
let testManagementAttemptToFixRetries = 0
let isDiEnabled = false
let testCodeCoverageLinesTotal
let coverageRootDir
let isSessionStarted = false
let vitestPool = null
const BREAKPOINT_HIT_GRACE_PERIOD_MS = 400
function getConfiguredEfdRetryCount (slowTestRetries, fallbackRetryCount) {
if (!slowTestRetries || !Object.keys(slowTestRetries).length) {
return fallbackRetryCount
}
return getMaxEfdRetryCount(slowTestRetries)
}
function getTestCommand () {
return `vitest ${process.argv.slice(2).join(' ')}`
}
function waitForHitProbe () {
return new Promise(resolve => {
realSetTimeout(() => {
resolve()
}, BREAKPOINT_HIT_GRACE_PERIOD_MS)
})
}
function isValidKnownTests (receivedKnownTests) {
return !!receivedKnownTests.vitest
}
function getProvidedContext () {
try {
const {
_ddIsEarlyFlakeDetectionEnabled,
_ddIsDiEnabled,
_ddKnownTests: knownTests,
_ddEarlyFlakeDetectionNumRetries: numRepeats,
_ddEarlyFlakeDetectionSlowTestRetries: slowTestRetries,
_ddIsKnownTestsEnabled: isKnownTestsEnabled,
_ddIsTestManagementTestsEnabled: isTestManagementTestsEnabled,
_ddTestManagementAttemptToFixRetries: testManagementAttemptToFixRetries,
_ddTestManagementTests: testManagementTests,
_ddIsFlakyTestRetriesEnabled: isFlakyTestRetriesEnabled,
_ddFlakyTestRetriesCount: flakyTestRetriesCount,
_ddFlakyTestRetriesIncludesUnnamedProject: flakyTestRetriesIncludesUnnamedProject,
_ddFlakyTestRetriesProjectNames: flakyTestRetriesProjectNames,
_ddIsImpactedTestsEnabled: isImpactedTestsEnabled,
_ddModifiedFiles: modifiedFiles,
_ddTestSessionId: testSessionId,
_ddTestModuleId: testModuleId,
_ddTestCommand: testCommand,
_ddRepositoryRoot: repositoryRoot,
_ddCodeOwnersEntries: codeOwnersEntries,
} = globalThis.__vitest_worker__.providedContext
return {
isDiEnabled: _ddIsDiEnabled,
isEarlyFlakeDetectionEnabled: _ddIsEarlyFlakeDetectionEnabled,
knownTests,
numRepeats,
slowTestRetries: slowTestRetries ?? {},
isKnownTestsEnabled,
isTestManagementTestsEnabled,
testManagementAttemptToFixRetries,
testManagementTests,
isFlakyTestRetriesEnabled,
flakyTestRetriesCount: flakyTestRetriesCount ?? 0,
flakyTestRetriesIncludesUnnamedProject,
flakyTestRetriesProjectNames,
isImpactedTestsEnabled,
modifiedFiles,
testSessionId,
testModuleId,
testCommand,
repositoryRoot,
codeOwnersEntries,
}
} catch {
log.error('Vitest workers could not parse provided context, so some features will not work.')
return {
isDiEnabled: false,
isEarlyFlakeDetectionEnabled: false,
knownTests: {},
numRepeats: 0,
slowTestRetries: {},
isKnownTestsEnabled: false,
isTestManagementTestsEnabled: false,
testManagementAttemptToFixRetries: 0,
testManagementTests: {},
isFlakyTestRetriesEnabled: false,
flakyTestRetriesCount: 0,
flakyTestRetriesIncludesUnnamedProject: false,
flakyTestRetriesProjectNames: undefined,
isImpactedTestsEnabled: false,
modifiedFiles: {},
testSessionId: undefined,
testModuleId: undefined,
testCommand: undefined,
repositoryRoot: undefined,
codeOwnersEntries: undefined,
}
}
}
function isReporterPackage (vitestPackage) {
return vitestPackage.B?.name === 'BaseSequencer'
}
// from 2.0.0
function isReporterPackageNew (vitestPackage) {
return vitestPackage.e?.name === 'BaseSequencer'
}
function isReporterPackageNewest (vitestPackage) {
return vitestPackage.h?.name === 'BaseSequencer'
}
/**
* Finds an export by its `.name` property in a minified vitest chunk.
* Minified export keys change across versions, so we search by function/class name.
* @param {object} pkg - The module exports object
* @param {string} name - The `.name` value to look for
* @returns {{ key: string, value: Function } | undefined}
*/
function findExportByName (pkg, name) {
for (const [key, value] of Object.entries(pkg)) {
if (value?.name === name) {
return { key, value }
}
}
}
function getBaseSequencerExport (vitestPackage) {
return findExportByName(vitestPackage, 'BaseSequencer')
}
function getChannelPromise (channelToPublishTo, frameworkVersion) {
return new Promise(resolve => {
channelToPublishTo.publish({ onDone: resolve, frameworkVersion })
})
}
function isCliApiPackage (vitestPackage) {
return !!findExportByName(vitestPackage, 'startVitest')
}
function getTestRunnerExport (testPackage) {
return findExportByName(testPackage, 'VitestTestRunner') || findExportByName(testPackage, 'TestRunner')
}
function getVitestExport (vitestPackage) {
return findExportByName(vitestPackage, 'Vitest')
}
function getForksPoolWorkerExport (vitestPackage) {
return findExportByName(vitestPackage, 'ForksPoolWorker')
}
function getThreadsPoolWorkerExport (vitestPackage) {
return findExportByName(vitestPackage, 'ThreadsPoolWorker')
}
function getSessionStatus (state) {
if (state.getCountOfFailedTests() > 0) {
return 'fail'
}
if (state.pathsSet.size === 0) {
return 'skip'
}
return 'pass'
}
// From https://github.com/vitest-dev/vitest/blob/51c04e2f44d91322b334f8ccbcdb368facc3f8ec/packages/runner/src/run.ts#L243-L250
function getVitestTestStatus (test, retryCount) {
if (test.result.state !== 'fail' && (!test.repeats || (test.retry ?? 0) === retryCount)) {
return 'pass'
}
return 'fail'
}
function getTypeTasks (fileTasks, type = 'test') {
const typeTasks = []
function getTasks (tasks) {
for (const task of tasks) {
if (task.type === type) {
typeTasks.push(task)
} else if (task.tasks) {
getTasks(task.tasks)
}
}
}
getTasks(fileTasks)
return typeTasks
}
function getTestName (task) {
let testName = task.name
let currentTask = task.suite
while (currentTask) {
if (currentTask.name) {
testName = `${currentTask.name} ${testName}`
}
currentTask = currentTask.suite
}
return testName
}
function getFinalAttemptToFixStatus (task, state, isSwitchedStatus, testCtx) {
if (isSwitchedStatus && attemptToFixTasks.has(task) && testCtx?.status) {
return testCtx.status
}
return state === 'fail' ? 'fail' : 'pass'
}
function recordFinalAttemptToFixExecution (task, status, providedContext) {
const statuses = attemptToFixTaskToStatuses.get(task)
if (statuses && statuses.length <= providedContext.testManagementAttemptToFixRetries) {
statuses.push(status)
}
recordAttemptToFixExecution(attemptToFixExecutions, {
testSuite: getTestSuitePath(task.file.filepath, process.cwd()),
testName: getTestName(task),
status,
isDisabled: disabledTasks.has(task),
isQuarantined: quarantinedTasks.has(task),
})
}
function disableFrameworkRetries (task) {
task.retry = 0
}
/**
* Vitest accumulates retry and repeat errors on one task result. The first error added since
* the last reported attempt is the primary error for the failed attempt currently being reported.
*
* @param {object} task
* @param {Array<object> | undefined} errors
* @returns {object | undefined}
*/
function getCurrentAttemptTestError (task, errors) {
if (!errors?.length) return
const previousErrorCount = taskToReportedErrorCount.get(task) ?? 0
const testError = errors[previousErrorCount] ?? errors[0]
taskToReportedErrorCount.set(task, errors.length)
return testError
}
/**
* Wraps a function so it runs inside the current test span context.
* @param {object} task
* @param {Function} fn
* @returns {Function}
*/
function wrapTestScopedFn (task, fn) {
return shimmer.wrapFunction(fn, fn => function (...args) {
return testFnCh.runStores(taskToCtx.get(task), () => fn.apply(this, args))
})
}
/**
* Wraps a `beforeEach` cleanup callback so it inherits the test span context.
* Vitest allows `beforeEach` to return a cleanup function, including via a promise.
* @param {object} task
* @param {unknown} result
* @returns {unknown}
*/
function wrapBeforeEachCleanupResult (task, result) {
if (typeof result === 'function') {
return wrapTestScopedFn(task, result)
}
if (result && typeof result.then === 'function') {
return result.then(cleanupFn => wrapBeforeEachCleanupResult(task, cleanupFn))
}
return result
}
function getWorkspaceProject (ctx) {
return ctx.getCoreWorkspaceProject
? ctx.getCoreWorkspaceProject()
: ctx.getRootProject()
}
function setProvidedContext (ctx, values, warningMessage) {
try {
Object.assign(getWorkspaceProject(ctx)._provided, values)
} catch {
log.warn(warningMessage)
}
}
function getTestFilepathsFromSpecifications (testSpecifications) {
if (!Array.isArray(testSpecifications) || !testSpecifications.length) {
return
}
return testSpecifications.map(testSpecification => {
const testFile = Array.isArray(testSpecification) ? testSpecification[1] : testSpecification
return testFile?.moduleId || testFile?.filepath || testFile
})
}
function getTestFilepaths (ctx, testSpecifications) {
const testFilepaths = getTestFilepathsFromSpecifications(testSpecifications)
if (testFilepaths) {
return testFilepaths
}
const getFilePaths = ctx.getTestFilepaths || ctx._globTestFilepaths
return getFilePaths.call(ctx)
}
function wrapCoverageProvider (ctx) {
const { coverageProvider } = ctx
if (!coverageProvider?.generateCoverage || coverageWrappedProviders.has(coverageProvider)) {
return
}
coverageWrappedProviders.add(coverageProvider)
// Capture coverage root directory from config (default is 'coverage' in cwd)
try {
const coverageConfig = ctx.config?.coverage
const reportsDirectory = coverageConfig?.reportsDirectory || 'coverage'
const rootDir = ctx.config?.root || process.cwd()
coverageRootDir = path.isAbsolute(reportsDirectory) ? reportsDirectory : path.join(rootDir, reportsDirectory)
} catch {
// Fallback to cwd if we can't get config
coverageRootDir = process.cwd()
}
shimmer.wrap(coverageProvider, 'generateCoverage', generateCoverage => async function () {
const totalCodeCoverage = await generateCoverage.apply(this, arguments)
try {
testCodeCoverageLinesTotal = totalCodeCoverage.getCoverageSummary().lines.pct
} catch {
// ignore errors
}
return totalCodeCoverage
})
}
function wrapSessionFinish (ctx) {
if (finishWrappedContexts.has(ctx)) {
return
}
finishWrappedContexts.add(ctx)
shimmer.wrap(ctx, 'exit', getFinishWrapper)
shimmer.wrap(ctx, 'close', getFinishWrapper)
}
async function runMainProcessSetup (ctx, frameworkVersion, testSpecifications) {
if (!testSessionFinishCh.hasSubscribers) {
return
}
try {
const { err, libraryConfig } = await getChannelPromise(libraryConfigurationCh, frameworkVersion)
if (!err) {
isFlakyTestRetriesEnabled = libraryConfig.isFlakyTestRetriesEnabled
flakyTestRetriesCount = libraryConfig.flakyTestRetriesCount
isEarlyFlakeDetectionEnabled = libraryConfig.isEarlyFlakeDetectionEnabled
earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries
earlyFlakeDetectionSlowTestRetries = libraryConfig.earlyFlakeDetectionSlowTestRetries ?? {}
isDiEnabled = libraryConfig.isDiEnabled
isKnownTestsEnabled = libraryConfig.isKnownTestsEnabled
isTestManagementTestsEnabled = libraryConfig.isTestManagementEnabled
testManagementAttemptToFixRetries = libraryConfig.testManagementAttemptToFixRetries
isImpactedTestsEnabled = libraryConfig.isImpactedTestsEnabled
}
} catch {
isFlakyTestRetriesEnabled = false
isEarlyFlakeDetectionEnabled = false
isDiEnabled = false
isKnownTestsEnabled = false
isImpactedTestsEnabled = false
}
if (testSessionConfigurationCh.hasSubscribers) {
const { testSessionId, testModuleId, testCommand, repositoryRoot, codeOwnersEntries } = await getChannelPromise(
testSessionConfigurationCh,
frameworkVersion
)
setProvidedContext(ctx, {
_ddTestSessionId: testSessionId,
_ddTestModuleId: testModuleId,
_ddTestCommand: testCommand,
_ddRepositoryRoot: repositoryRoot,
_ddCodeOwnersEntries: codeOwnersEntries,
}, 'Could not send test session configuration to workers.')
}
const {
knownTestsResponse,
testManagementTestsResponse,
} = await getTestOptimizationRequestResults({
isKnownTestsEnabled,
isTestManagementTestsEnabled,
getKnownTests: () => getChannelPromise(knownTestsCh),
getTestManagementTests: () => getChannelPromise(testManagementTestsCh),
})
const flakyTestRetriesConfiguration = configureFlakyTestRetries(ctx, testSpecifications)
if (flakyTestRetriesConfiguration) {
setProvidedContext(ctx, {
_ddIsFlakyTestRetriesEnabled: isFlakyTestRetriesEnabled,
_ddFlakyTestRetriesCount: flakyTestRetriesCount,
_ddFlakyTestRetriesIncludesUnnamedProject: flakyTestRetriesConfiguration.includesUnnamedProject,
_ddFlakyTestRetriesProjectNames: flakyTestRetriesConfiguration.projectNames,
}, 'Could not send library configuration to workers.')
}
if (isKnownTestsEnabled) {
const currentKnownTestsResponse = knownTestsResponse || await getChannelPromise(knownTestsCh)
if (currentKnownTestsResponse.err) {
isEarlyFlakeDetectionEnabled = false
} else {
const knownTests = currentKnownTestsResponse.knownTests
const testFilepaths = await getTestFilepaths(ctx, testSpecifications)
if (isValidKnownTests(knownTests)) {
isEarlyFlakeDetectionFaultyCh.publish({
knownTests: knownTests.vitest,
testFilepaths,
onDone: (isFaulty) => {
isEarlyFlakeDetectionFaulty = isFaulty
},
})
if (isEarlyFlakeDetectionFaulty) {
isEarlyFlakeDetectionEnabled = false
log.warn('New test detection is disabled because the number of new tests is too high.')
} else {
setProvidedContext(ctx, {
_ddIsKnownTestsEnabled: isKnownTestsEnabled,
_ddKnownTests: knownTests,
_ddIsEarlyFlakeDetectionEnabled: isEarlyFlakeDetectionEnabled,
_ddEarlyFlakeDetectionNumRetries:
getConfiguredEfdRetryCount(earlyFlakeDetectionSlowTestRetries, earlyFlakeDetectionNumRetries),
_ddEarlyFlakeDetectionSlowTestRetries: earlyFlakeDetectionSlowTestRetries,
}, 'Could not send known tests to workers so Early Flake Detection will not work.')
}
} else {
isEarlyFlakeDetectionFaulty = true
isEarlyFlakeDetectionEnabled = false
}
}
}
if (isDiEnabled) {
setProvidedContext(ctx, {
_ddIsDiEnabled: isDiEnabled,
}, 'Could not send Dynamic Instrumentation configuration to workers.')
}
if (isTestManagementTestsEnabled) {
const { err, testManagementTests: receivedTestManagementTests } =
testManagementTestsResponse || await getChannelPromise(testManagementTestsCh)
if (err) {
isTestManagementTestsEnabled = false
log.error('Could not get test management tests.')
} else {
setProvidedContext(ctx, {
_ddIsTestManagementTestsEnabled: isTestManagementTestsEnabled,
_ddTestManagementAttemptToFixRetries: testManagementAttemptToFixRetries,
_ddTestManagementTests: receivedTestManagementTests,
}, 'Could not send test management tests to workers so Test Management will not work.')
}
}
if (isImpactedTestsEnabled) {
const { err, modifiedFiles } = await getChannelPromise(modifiedFilesCh)
if (err) {
log.error('Could not get modified tests.')
} else {
setProvidedContext(ctx, {
_ddIsImpactedTestsEnabled: isImpactedTestsEnabled,
_ddModifiedFiles: modifiedFiles,
}, 'Could not send modified tests to workers so Impacted Tests will not work.')
}
}
wrapCoverageProvider(ctx)
wrapSessionFinish(ctx)
}
function ensureMainProcessSetup (ctx, frameworkVersion, testSpecifications) {
let setupPromise = mainProcessSetupPromises.get(ctx)
if (!setupPromise) {
setupPromise = runMainProcessSetup(ctx, frameworkVersion, testSpecifications)
mainProcessSetupPromises.set(ctx, setupPromise)
}
return setupPromise
}
/**
* Configure Vitest retries for the root project and resolved workspace projects.
*
* @param {object} ctx
* @param {unknown[]|undefined} testSpecifications
* @returns {{ projectNames: string[], includesUnnamedProject: boolean }|undefined}
*/
function configureFlakyTestRetries (ctx, testSpecifications) {
if (!isFlakyTestRetriesEnabled || flakyTestRetriesCount <= 0) return
let configured = false
let includesUnnamedProject = false
const projectNames = []
for (const { config, projectName } of getVitestProjectConfigs(ctx, testSpecifications)) {
if (!config.retry) {
config.retry = flakyTestRetriesCount
configured = true
if (projectName) {
projectNames.push(projectName)
} else {
includesUnnamedProject = true
}
}
}
if (!configured) return
return {
includesUnnamedProject,
projectNames,
}
}
/**
* Return unique Vitest configs that can be used to run tests.
*
* @param {object} ctx
* @param {unknown[]|undefined} testSpecifications
* @returns {{ config: object, projectName?: string }[]}
*/
function getVitestProjectConfigs (ctx, testSpecifications) {
const entries = []
addTestSpecificationConfigs(entries, testSpecifications)
if (entries.length > 0) {
return entries
}
const selectedProjectNames = getSelectedProjectNames()
addSelectedInlineProjectConfigs(entries, safeConfig(ctx), selectedProjectNames)
addSelectedRuntimeProjectConfigs(entries, ctx?.projects, selectedProjectNames)
if (entries.length > 0) {
return entries
}
if (Array.isArray(ctx?.projects)) {
for (const project of ctx.projects) {
addConfig(entries, safeConfig(project), getProjectName(project))
}
if (entries.length > 0) {
return entries
}
}
addConfig(entries, safeConfig(ctx))
addConfig(entries, safeConfig(safeWorkspaceProject(ctx)))
return entries
}
/**
* Add configs from runnable test specifications once.
*
* @param {{ config: object, projectName?: string }[]} entries
* @param {unknown[]|undefined} testSpecifications
*/
function addTestSpecificationConfigs (entries, testSpecifications) {
if (!Array.isArray(testSpecifications)) return
for (const testSpecification of testSpecifications) {
const project = getTestSpecificationProject(testSpecification)
addConfig(entries, safeConfig(project), getProjectName(project))
}
}
/**
* Add selected inline project configs from the root Vitest config once.
*
* @param {{ config: object, projectName?: string }[]} entries
* @param {object|undefined} rootConfig
* @param {string[]} selectedProjectNames
*/
function addSelectedInlineProjectConfigs (entries, rootConfig, selectedProjectNames) {
if (selectedProjectNames.length === 0 || !Array.isArray(rootConfig?.projects)) return
for (const project of rootConfig.projects) {
const config = getInlineProjectConfig(project)
const projectName = getProjectName(project)
if (selectedProjectNames.includes(projectName)) {
addConfig(entries, config, projectName)
}
}
}
/**
* Add selected resolved project configs once.
*
* @param {{ config: object, projectName?: string }[]} entries
* @param {unknown[]|undefined} projects
* @param {string[]} selectedProjectNames
*/
function addSelectedRuntimeProjectConfigs (entries, projects, selectedProjectNames) {
if (selectedProjectNames.length === 0 || !Array.isArray(projects)) return
for (const project of projects) {
const projectName = getProjectName(project)
if (selectedProjectNames.includes(projectName)) {
addConfig(entries, safeConfig(project), projectName)
}
}
}
/**
* Return selected project names from the Vitest CLI arguments.
*
* @returns {string[]}
*/
function getSelectedProjectNames () {
const names = []
for (let index = 0; index < process.argv.length; index++) {
const argument = process.argv[index]
if (argument === '--project' && process.argv[index + 1]) {
names.push(process.argv[index + 1])
index++
} else if (argument.startsWith('--project=')) {
names.push(argument.slice('--project='.length))
}
}
return names
}
/**
* Return the test config from an inline Vitest project entry.
*
* @param {unknown} project
* @returns {object|undefined}
*/
function getInlineProjectConfig (project) {
return project?.test || project
}
/**
* Return a Vitest project name from runtime or inline project objects.
*
* @param {unknown} project
* @returns {string|undefined}
*/
function getProjectName (project) {
return normalizeProjectName(project?.name || project?.config?.name || project?.test?.name)
}
/**
* Return a normalized Vitest project name.
*
* @param {unknown} name
* @returns {string|undefined}
*/
function normalizeProjectName (name) {
if (typeof name === 'string') return name
const label = name?.label
return typeof label === 'string' ? label : undefined
}
/**
* Add a config object once.
*
* @param {{ config: object, projectName?: string }[]} entries
* @param {object|undefined} config
* @param {string|undefined} projectName
*/
function addConfig (entries, config, projectName) {
if (config && !entries.some(entry => entry.config === config || (projectName && entry.projectName === projectName))) {
entries.push({ config, projectName })
}
}
/**
* Read a Vitest config object without assuming the project is initialized.
*
* @param {object|undefined} project
* @returns {object|undefined}
*/
function safeConfig (project) {
let config
try {
config = project?.config
} catch {}
return config
}
/**
* Read the workspace project without assuming the root server is initialized.
*
* @param {object} ctx
* @returns {object|undefined}
*/
function safeWorkspaceProject (ctx) {
let project
try {
project = getWorkspaceProject(ctx)
} catch {}
return project
}
/**
* Return whether Datadog configured ATR retries for a task.
*
* @param {object} providedContext
* @param {object} task
* @returns {boolean}
*/
function isFlakyTestRetriesEnabledForTask (providedContext, task) {
if (!providedContext.isFlakyTestRetriesEnabled) return false
const { flakyTestRetriesProjectNames } = providedContext
if (!Array.isArray(flakyTestRetriesProjectNames)) return true
const projectName = task.file?.projectName
if (!projectName) {
return providedContext.flakyTestRetriesIncludesUnnamedProject === true
}
return flakyTestRetriesProjectNames.includes(projectName)
}
function getSortWrapper (sort, frameworkVersion) {
return async function () {
await ensureMainProcessSetup(this.ctx, frameworkVersion, arguments[0])
return sort.apply(this, arguments)
}
}
function getFinishWrapper (exitOrClose) {
let isClosed = false
return async function () {
if (isClosed) { // needed because exit calls close
return exitOrClose.apply(this, arguments)
}
isClosed = true
if (!testSessionFinishCh.hasSubscribers) {
return exitOrClose.apply(this, arguments)
}
let onFinish
const flushPromise = new Promise(resolve => {
onFinish = resolve
})
const failedSuites = this.state.getFailedFilepaths()
let error
if (failedSuites.length) {
error = new Error(`Test suites failed: ${failedSuites.length}.`)
}
testSessionFinishCh.publish({
status: getSessionStatus(this.state),
testCodeCoverageLinesTotal,
error,
isEarlyFlakeDetectionEnabled,
isEarlyFlakeDetectionFaulty,
isTestManagementTestsEnabled,
vitestPool,
onFinish,
})
logTestOptimizationSummary({ attemptToFixExecutions, newTestsWithDynamicNames })
loggedAttemptToFixTests.clear()
await flushPromise
// If coverage was generated, publish coverage report channel for upload
if (coverageRootDir && codeCoverageReportCh.hasSubscribers) {
await new Promise((resolve) => {
codeCoverageReportCh.publish({ rootDir: coverageRootDir, onDone: resolve })
})
}
return exitOrClose.apply(this, arguments)
}
}
function getCliOrStartVitestWrapper (frameworkVersion) {
return function (oldCliOrStartVitest) {
return function (...args) {
if (!testSessionStartCh.hasSubscribers || isSessionStarted) {
return oldCliOrStartVitest.apply(this, args)
}
isSessionStarted = true
testSessionStartCh.publish({ command: getTestCommand(), frameworkVersion })
return oldCliOrStartVitest.apply(this, args)
}
}
}
function isForkPool (pool) {
return pool === 'forks' || pool === 'vmForks'
}
function isThreadPool (pool) {
return pool === 'threads' || pool === 'vmThreads'
}
/**
* Return the project object attached to a Vitest test specification.
*
* @param {unknown} testSpecification
* @returns {object|undefined}
*/
function getTestSpecificationProject (testSpecification) {
if (Array.isArray(testSpecification)) {
return testSpecification[0]
}
return testSpecification?.project
}
function getTestSpecificationPool (testSpecification) {
const project = getTestSpecificationProject(testSpecification)
return project?.config?.pool || project?.serializedConfig?.pool || project?.pool || testSpecification?.pool
}
function hasForkPoolTestSpecification (testSpecifications) {
if (!Array.isArray(testSpecifications)) {
return false
}
for (const testSpecification of testSpecifications) {
if (isForkPool(getTestSpecificationPool(testSpecification))) {
return true
}
}
return false
}
function shouldMarkVitestWorkerEnv (pool, testSpecifications) {
return isForkPool(pool) || hasForkPoolTestSpecification(testSpecifications) ||
(!testSpecifications && !isThreadPool(pool))
}
function markVitestWorkerEnv (ctx, testSpecifications) {
const config = ctx?.config
if (!config || !shouldMarkVitestWorkerEnv(config.pool, testSpecifications)) {
return
}
config.env = config.env || {}
config.env.DD_VITEST_WORKER = '1'
}
function wrapVitestRunFiles (Vitest, frameworkVersion) {
if (!Vitest?.prototype?.runFiles) {
return
}
shimmer.wrap(Vitest.prototype, 'runFiles', runFiles => async function (testSpecifications) {
markVitestWorkerEnv(this, testSpecifications)
await ensureMainProcessSetup(this, frameworkVersion, testSpecifications)
return runFiles.apply(this, arguments)
})
if (Vitest.prototype.collectTests) {
shimmer.wrap(Vitest.prototype, 'collectTests', collectTests => function () {
markVitestWorkerEnv(this)
return collectTests.apply(this, arguments)
})
}
}
function getCreateCliWrapper (vitestPackage, frameworkVersion) {
const createCliExport = findExportByName(vitestPackage, 'createCLI')
if (!createCliExport) {
return vitestPackage
}
shimmer.wrap(vitestPackage, createCliExport.key, getCliOrStartVitestWrapper(frameworkVersion))
return vitestPackage
}
function threadHandler (thread) {
const { runtime } = thread
let workerProcess
if (runtime === 'child_process') {
vitestPool = 'child_process'
workerProcess = thread.process
} else if (runtime === 'worker_threads') {
vitestPool = 'worker_threads'
workerProcess = thread.thread
} else {
vitestPool = 'unknown'
}
if (!workerProcess) {
log.error('Vitest error: could not get process or thread from TinyPool#run')
return
}
if (workerProcesses.has(workerProcess)) {
return
}
workerProcesses.add(workerProcess)
workerProcess.on('message', (message) => {
if (message.__tinypool_worker_message__ && message.data) {
if (message.interprocessCode === VITEST_WORKER_TRACE_PAYLOAD_CODE) {
collectTestOptimizationSummariesFromTraces(message.data, {
newTestsWithDynamicNames,
attemptToFixExecutions,
})
workerReportTraceCh.publish(message.data)
} else if (message.interprocessCode === VITEST_WORKER_LOGS_PAYLOAD_CODE) {
workerReportLogsCh.publish(message.data)
}
}
})
}
function wrapTinyPoolRun (TinyPool) {
shimmer.wrap(TinyPool.prototype, 'run', run => async function () {
// We have to do this before and after because the threads list gets recycled, that is, the processes are re-created
// eslint-disable-next-line unicorn/no-array-for-each
this.threads.forEach(threadHandler)
const runResult = await run.apply(this, arguments)
// eslint-disable-next-line unicorn/no-array-for-each
this.threads.forEach(threadHandler)
return runResult
})
}
addHook({
name: 'tinypool',
// version from tinypool@0.8 was used in vitest@1.6.0
versions: ['>=0.8.0'],
}, (TinyPool) => {
wrapTinyPoolRun(TinyPool)
return TinyPool
})
function getWrappedOn (on) {
return function (event, callback) {
if (event !== 'message') {
return on.apply(this, arguments)
}
// `arguments[1]` is the callback function, which
// we modify to intercept our messages to not interfere
// with vitest's own messages
arguments[1] = shimmer.wrapFunction(callback, callback => function (message) {
if (message.type !== 'Buffer' && Array.isArray(message)) {
const [interprocessCode, data] = message
if (interprocessCode === VITEST_WORKER_TRACE_PAYLOAD_CODE) {
collectTestOptimizationSummariesFromTraces(data, {
newTestsWithDynamicNames,
attemptToFixExecutions,
})
workerReportTraceCh.publish(data)
} else if (interprocessCode === VITEST_WORKER_LOGS_PAYLOAD_CODE) {
workerReportLogsCh.publish(data)
}
// If we execute the callback vitest crashes, as the message is not supported
return
}
return callback.apply(this, arguments)
})
return on.apply(this, arguments)
}
}
function getStartVitestWrapper (cliApiPackage, frameworkVersion) {
if (!isCliApiPackage(cliApiPackage)) {
return cliApiPackage
}
const startVitestExport = findExportByName(cliApiPackage, 'startVitest')
shimmer.wrap(cliApiPackage, startVitestExport.key, getCliOrStartVitestWrapper(frameworkVersion))
const vitest = getVitestExport(cliApiPackage)
if (vitest) {
wrapVitestRunFiles(vitest.value, frameworkVersion)
}
const forksPoolWorker = getForksPoolWorkerExport(cliApiPackage)
if (forksPoolWorker) {
// function is async
shimmer.wrap(forksPoolWorker.value.prototype, 'start', start => function (...args) {
vitestPool = 'child_process'
this.env.DD_VITEST_WORKER = '1'
return start.apply(this, args)
})
shimmer.wrap(forksPoolWorker.value.prototype, 'on', getWrappedOn)
}
const threadsPoolWorker = getThreadsPoolWorkerExport(cliApiPackage)
if (threadsPoolWorker) {
// function is async
shimmer.wrap(threadsPoolWorker.value.prototype, 'start', start => function (...args) {
vitestPool = 'worker_threads'
this.env.DD_VITEST_WORKER = '1'
return start.apply(this, args)
})
shimmer.wrap(threadsPoolWorker.value.prototype, 'on', getWrappedOn)
}
return cliApiPackage
}
function wrapVitestTestRunner (VitestTestRunner) {
// `onBeforeRunTask` is run before any repetition or attempt is run
// `onBeforeRunTask` is an async function
shimmer.wrap(VitestTestRunner.prototype, 'onBeforeRunTask', onBeforeRunTask => function (task) {
const testName = getTestName(task)
const {
knownTests,
isEarlyFlakeDetectionEnabled,
isKnownTestsEnabled,
numRepeats,
isTestManagementTestsEnabled,
testManagementAttemptToFixRetries,
testManagementTests,
isImpactedTestsEnabled,
modifiedFiles,
} = getProvidedContext()
if (isTestManagementTestsEnabled) {
isAttemptToFixCh.publish({
testManagementTests,
testSuiteAbsolutePath: task.file.filepath,
testName,
onDone: (isAttemptToFix) => {
if (isAttemptToFix) {
isRetryReasonAttemptToFix = task.repeats !== testManagementAttemptToFixRetries
disableFrameworkRetries(task)
task.repeats = testManagementAttemptToFixRetries
attemptToFixTasks.add(task)
attemptToFixTaskToStatuses.set(task, [])
}
},
})
isDisabledCh.publish({
testManagementTests,
testSuiteAbsolutePath: task.file.filepath,
testName,
onDone: (isTestDisabled) => {
if (isTestDisabled) {
disabledTasks.add(task)
if (!attemptToFixTasks.has(task)) {
// we only actually skip if the test is not being attempted to be fixed
task.mode = 'skip'
}
}
},
})
}
if (isImpactedTestsEnabled) {
isModifiedCh.publish({
modifiedFiles,
testSuiteAbsolutePath: task.file.filepath,
onDone: (isImpacted) => {
if (isImpacted) {
if (isEarlyFlakeDetectionEnabled) {
isRetryReasonEfd = true
disableFrameworkRetries(task)
task.repeats = numRepeats
}
modifiedTasks.add(task)
taskToStatuses.set(task, [])
}
},
})
}
if (isKnownTestsEnabled) {
isNewTestCh.publish({
knownTests,
testSuiteAbsolutePath: task.file.filepath,
testName,
onDone: (isNew) => {
if (isNew && !attemptToFixTasks.has(task)) {
if (isEarlyFlakeDetectionEnabled && !modifiedTasks.has(task)) {
isRetryReasonEfd = true
disableFrameworkRetries(task)
task.repeats = numRepeats
}
newTasks.add(task)
taskToStatuses.set(task, [])
if (DYNAMIC_NAME_RE.test(testName)) {
dynamicNameTasks.add(task)
}
}
},
})
}
return onBeforeRunTask.apply(this, arguments)
})
// `onAfterRunTask` is run after all repetitions or attempts are run
// `onAfterRunTask` is an async function
shimmer.wrap(VitestTestRunner.prototype, 'onAfterRunTask', onAfterRunTask => function (task) {
const { isEarlyFlakeDetectionEnabled, isTestManagementTestsEnabled } = getProvidedContext()
if (isTestManagementTestsEnabled) {
const isAttemptingToFix = attemptToFixTasks.has(task)
const isQuarantined = quarantinedTasks.has(task)
if (isAttemptingToFix) {
const statuses = attemptToFixTaskToStatuses.get(task)
if (task.result.state === 'pass' && statuses?.includes('fail')) {
switchedStatuses.add(task)
task.result.state = 'fail'
}
}
if (!isAttemptingToFix && isQuarantined) {
if (task.result.state === 'fail') {
switchedStatuses.add(task)
}
task.result.state = 'pass'
}
}
if (isEarlyFlakeDetectionEnabled && taskToStatuses.has(task) && !attemptToFixTasks.has(task)) {
const statuses = taskToStatuses.get(task)
// If the test has passed at least once, we consider it passed
if (statuses.includes('pass')) {
if (task.result.state === 'fail') {
switchedStatuses.add(task)
}
task.result.state = 'pass'
}
}
return onAfterRunTask.apply(this, arguments)
})
// test start (only tests that are not marked as skip or todo)
// `onBeforeTryTask` is run for every repetition and attempt of the test
shimmer.wrap(VitestTestRunner.prototype, 'onBeforeTryTask', onBeforeTryTask => async function (task, retryInfo) {
if (!testPassCh.hasSubscribers && !testErrorCh.hasSubscribers && !testSkipCh.hasSubscribers) {
return onBeforeTryTask.apply(this, arguments)
}
const testName = getTestName(task)
let isNew = false
let isQuarantined = false
const providedContext = getProvidedContext()
const {
isKnownTestsEnabled,
isEarlyFlakeDetectionEnabled,
isDiEnabled,
isTestManagementTestsEnabled,
testManagementTests,
slowTestRetries,
} = providedContext
if (isKnownTestsEnabled) {
isNew = newTasks.has(task)
}
if (isTestManagementTestsEnabled) {
isQuarantinedCh.publish({
testManagementTests,
testSuiteAbsolutePath: task.file.filepath,
testName,
onDone: (isTestQuarantined) => {
isQuarantined = isTestQuarantined
if (isTestQuarantined) {
quarantinedTasks.add(task)
}
},
})
}
const { retry: numAttempt, repeats: numRepetition } = retryInfo
const isEfdManagedTask = isEarlyFlakeDetectionEnabled && taskToStatuses.has(task) && !attemptToFixTasks.has(task)
if (isEfdManagedTask && numRepetition > 0 && !efdDeterminedRetries.has(task)) {
const previousExecutionStart = efdExecutionStartByTask.get(task)
const duration = previousExecutionStart === undefined
? task.result?.duration ?? 0
: performance.now() - previousExecutionStart
const retryCount = getEfdRetryCount(duration, slowTestRetries)
efdDeterminedRetries.set(task, retryCount)
task.repeats = retryCount
if (retryCount === 0) {
efdSlowAbortedTasks.add(task)
}
}
const efdRetryCount = efdDeterminedRetries.get(task)
if (isEfdManagedTask && efdRetryCount !== undefined && numRepetition > efdRetryCount) {
if (task.result) {
efdSkippedRetryResults.set(task, {
...task.result,
errors: task.result.errors?.slice(),
})
}
if (vitestSetFn) {
const noop = function () {}
noop.__ddTraceWrapped = true
vitestSetFn(task, noop)
}
return onBeforeTryTask.apply(this, arguments)
}
if (isEfdManagedTask) {
efdExecutionStartByTask.set(task, performance.now())
}
// We finish the previous test here because we know it has failed already
if (numAttempt > 0) {
const shouldWaitForHitProbe = isDiEnabled && numAttempt > 1
if (shouldWaitForHitProbe) {
await waitForHitProbe()
}
const promises = {}
const shouldSetProbe = isDiEnabled && numAttempt === 1
const ctx = taskToCtx.get(task)
const testError = getCurrentAttemptTestError(task, task.result?.errors)
if (ctx) {
testErrorCh.publish({
error: testError,
shouldSetProbe,
promises,
...ctx.currentStore,
})
// We wait for the probe to be set
if (promises.setProbePromise) {
await promises.setProbePromise
}
}
}
const lastExecutionStatus = task.result.state
const isAtf = attemptToFixTasks.has(task)
const shouldTrackStatuses = isEarlyFlakeDetectionEnabled || isAtf
const shouldFlipStatus = isEarlyFlakeDetectionEnabled || isAtf
const statuses = isAtf ? attemptToFixTaskToStatuses.get(task) : taskToStatuses.get(task)
// These clauses handle task.repeats, whether EFD is enabled or not
// The only thing that EFD does is to forcefully pass the test if it has passed at least once
if (numRepetition > 0 && numRepetition < task.repeats) { // it may or may have not failed
// Here we finish the earlier iteration,
// as long as it's not the _last_ iteration (which will be finished normally)
const ctx = taskToCtx.get(task)
if (ctx) {
if (lastExecutionStatus === 'fail') {
const testError = getCurrentAttemptTestError(task, task.result?.errors)
testErrorCh.publish({ error: testError, ...ctx.currentStore })
} else {
testPassCh.publish({ task, ...ctx.currentStore })
}
if (shouldTrackStatuses && statuses) {
statuses.push(lastExecutionStatus)
}
if (shouldFlipStatus) {
// If we don't "reset" the result.state to "pass", once a repetition fails,
// vitest will always consider the test as failed, so we can't read the actual status
// This means that we change vitest's behavior:
// if the last attempt passes, vitest would consider the test as failed
// but after this change, it will consider the test as passed
task.result.state = 'pass'
}
}
} else if (numRepetition === task.repeats) {
if (shouldTrackStatuses && statuses) {
statuses.push(lastExecutionStatus)
}
const ctx = taskToCtx.get(task)
if (lastExecutionStatus === 'fail') {
const testError = getCurrentAttemptTestError(task, task.result?.errors)
testErrorCh.publish({ error: testError, ...ctx.currentStore })
} else {
testPassCh.publish({ task, ...ctx.currentStore })
}
if (shouldFlipStatus) {
task.result.state = 'pass'
}
}
const isRetryReasonAtr = numAttempt > 0 &&
isFlakyTestRetriesEnabledForTask(providedContext, task) &&
!isRetryReasonAttemptToFix &&
!isRetryReasonEfd
const ctx = {
testName,
testSuiteAbsolutePath: task.file.filepath,
isRetry: numAttempt > 0 || numRepetition > 0,
isRetryReasonEfd,
isRetryReasonAttemptToFix: isRetryReasonAttemptToFix && numRepetition > 0,
isNew,
hasDynamicName: dynamicNameTasks.has(task),
mightHitProbe: isDiEnabled && numAttempt > 0,
isAttemptToFix: attemptToFixTasks.has(task),
isDisabled: disabledTasks.has(task),
isQuarantined,
isRetryReasonAtr,
isModified: modifiedTasks.has(task),
}
taskToCtx.set(task, ctx)
if (attemptToFixTasks.has(task)) {
logAttemptToFixTestExecution(
getTestSuitePath(task.file.filepath, process.cwd()),
testName,
loggedAttemptToFixTests
)
}
testStartCh.runStores(ctx, () => {})
// Wrap the test function so it runs inside the test span context.
// Without this, HTTP requests during test execution become orphaned root spans.
if (vitestGetFn && vitestSetFn) {
const originalFn = vitestGetFn(task)
if (originalFn && !originalFn.__ddTraceWrapped) {
const wrappedFn = wrapTestScopedFn(task, originalFn)
wrappedFn.__ddTraceWrapped = true
vitestSetFn(task, wrappedFn)
}
}
// Wrap beforeEach/afterEach hooks so they also run inside the test span context.
// In vitest 4+, hooks are in a WeakMap accessed via getHooks(). In older versions, they're on suite.hooks.
let currentSuite = task.suite
while (currentSuite) {
const hooks = vitestGetHooks ? vitestGetHooks(currentSuite) : currentSuite.hooks
if (hooks) {
for (const hookType of ['beforeEach', 'afterEach']) {
const hookArray = hooks[hookType]
if (!hookArray) continue
for (let i = 0; i < hookArray.length; i++) {
const currentFn = hookArray[i]
const originalFn = originalHookFns.get(currentFn) || currentFn
const wrappedFn = shimmer.wrapFunction(originalFn, fn => function (...args) {
const result = testFnCh.runStores(taskToCtx.get(task), () => fn.apply(this, args))
if (hookType === 'beforeEach') {
return wrapBeforeEachCleanupResult(task, result)
}
return result
})
originalHookFns.set(wrappedFn, originalFn)
hookArray[i] = wrappedFn
}
}
}
currentSuite = currentSuite.suite
}
return onBeforeTryTask.apply(this, arguments)
})
// test finish (only passed tests)
shimmer.wrap(VitestTestRunner.prototype, 'onAfterTryTask', onAfterTryTask =>
async function (task, retryInfo) {
if (!testPassCh.hasSubscribers && !testErrorCh.hasSubscribers && !testSkipCh.hasSubscribers) {
return onAfterTryTask.apply(this, arguments)
}
const result = await onAfterTryTask.apply(this, arguments)
const {
isEarlyFlakeDetectionEnabled,
testManagementAttemptToFixRetries,
slowTestRetries,
} = getProvidedContext()
const status = getVitestTestStatus(task, retryInfo.retry)
const ctx = taskToCtx.get(task)
const { isDiEnabled } = getProvidedContext()
if (efdSkippedRetryResults.has(task)) {
task.result = efdSkippedRetryResults.get(task)
efdSkippedRetryResults.delete(task)
return result
}
if (isDiEnabled && retryInfo.retry > 1) {
await waitForHitProbe()
}
if (
isEarlyFlakeDetectionEnabled &&
(retryInfo.repeats ?? 0) === 0 &&
taskToStatuses.has(task) &&
!attemptToFixTasks.has(task) &&
!efdDeterminedRetries.has(task)
) {
const executionStart = efdExecutionStartByTask.get(task)
const duration = executionStart === undefined ? task.result?.duration ?? 0 : performance.now() - executionStart
const retryCount = getEfdRetryCount(duration, slowTestRetries)
efdDeterminedRetries.set(task, retryCount)
task.repeats = retryCount
if (retryCount === 0) {
efdSlowAbortedTasks.add(task)
}
}
let attemptToFixPassed = false
let attemptToFixFailed = false
if (attemptToFixTasks.has(task)) {
const statuses = attemptToFixTaskToStatuses.get(task)
if (statuses.length === testManagementAttemptToFixRetries) {
if (status === 'pass' && statuses.every(status => status === 'pass')) {
attemptToFixPassed = true
} else if (status === 'fail' || statuses.includes('fail')) {
attemptToFixFailed = true
}
}
}
if (ctx) {
// We don't finish here because the test might fail in a later hook (afterEach)
ctx.status = status
ctx.task = task
ctx.attemptToFixPassed = attemptToFixPassed
ctx.attemptToFixFailed = attemptToFixFailed
testFinishTimeCh.runStores(ctx, () =