UNPKG

dd-trace

Version:

Datadog APM tracing client for JavaScript

1,532 lines (1,356 loc) 60.7 kB
'use strict' // Capture real timers at module load time, before any test can install fake timers. const realSetTimeout = setTimeout const path = require('node:path') const { performance } = require('node:perf_hooks') const shimmer = require('../../datadog-shimmer') const log = require('../../dd-trace/src/log') const { VITEST_WORKER_TRACE_PAYLOAD_CODE, VITEST_WORKER_LOGS_PAYLOAD_CODE, DYNAMIC_NAME_RE, getTestSuitePath, getEfdRetryCount, getMaxEfdRetryCount, recordAttemptToFixExecution, collectTestOptimizationSummariesFromTraces, logAttemptToFixTestExecution, logTestOptimizationSummary, getTestOptimizationRequestResults, } = require('../../dd-trace/src/plugins/util/test') const { addHook, channel } = require('./helpers/instrument') // test hooks const testStartCh = channel('ci:vitest:test:start') const testFinishTimeCh = channel('ci:vitest:test:finish-time') const testPassCh = channel('ci:vitest:test:pass') const testErrorCh = channel('ci:vitest:test:error') const testSkipCh = channel('ci:vitest:test:skip') const isNewTestCh = channel('ci:vitest:test:is-new') const isAttemptToFixCh = channel('ci:vitest:test:is-attempt-to-fix') const isDisabledCh = channel('ci:vitest:test:is-disabled') const isQuarantinedCh = channel('ci:vitest:test:is-quarantined') const isModifiedCh = channel('ci:vitest:test:is-modified') const testFnCh = channel('ci:vitest:test:fn') // test suite hooks const testSuiteStartCh = channel('ci:vitest:test-suite:start') const testSuiteFinishCh = channel('ci:vitest:test-suite:finish') const testSuiteErrorCh = channel('ci:vitest:test-suite:error') // test session hooks const testSessionStartCh = channel('ci:vitest:session:start') const testSessionFinishCh = channel('ci:vitest:session:finish') const testSessionConfigurationCh = channel('ci:vitest:session:configuration') const libraryConfigurationCh = channel('ci:vitest:library-configuration') const knownTestsCh = channel('ci:vitest:known-tests') const isEarlyFlakeDetectionFaultyCh = channel('ci:vitest:is-early-flake-detection-faulty') const testManagementTestsCh = channel('ci:vitest:test-management-tests') const modifiedFilesCh = channel('ci:vitest:modified-files') const workerReportTraceCh = channel('ci:vitest:worker-report:trace') const workerReportLogsCh = channel('ci:vitest:worker-report:logs') const codeCoverageReportCh = channel('ci:vitest:coverage-report') const taskToCtx = new WeakMap() const taskToStatuses = new WeakMap() const taskToReportedErrorCount = new WeakMap() const attemptToFixTaskToStatuses = new WeakMap() const originalHookFns = new WeakMap() const newTasks = new WeakSet() const dynamicNameTasks = new WeakSet() const newTestsWithDynamicNames = new Set() const disabledTasks = new WeakSet() const quarantinedTasks = new WeakSet() const attemptToFixTasks = new WeakSet() const modifiedTasks = new WeakSet() const efdDeterminedRetries = new WeakMap() const efdSlowAbortedTasks = new WeakSet() const efdExecutionStartByTask = new WeakMap() const efdSkippedRetryResults = new WeakMap() const attemptToFixExecutions = new Map() const loggedAttemptToFixTests = new Set() let isRetryReasonEfd = false let isRetryReasonAttemptToFix = false const switchedStatuses = new WeakSet() const workerProcesses = new WeakSet() const mainProcessSetupPromises = new WeakMap() const coverageWrappedProviders = new WeakSet() const finishWrappedContexts = new WeakSet() let isFlakyTestRetriesEnabled = false let flakyTestRetriesCount = 0 let isEarlyFlakeDetectionEnabled = false let earlyFlakeDetectionNumRetries = 0 let earlyFlakeDetectionSlowTestRetries = {} let isEarlyFlakeDetectionFaulty = false let isKnownTestsEnabled = false let isTestManagementTestsEnabled = false let isImpactedTestsEnabled = false let vitestGetFn = null let vitestSetFn = null let vitestGetHooks = null let testManagementAttemptToFixRetries = 0 let isDiEnabled = false let testCodeCoverageLinesTotal let coverageRootDir let isSessionStarted = false let vitestPool = null const BREAKPOINT_HIT_GRACE_PERIOD_MS = 400 function getConfiguredEfdRetryCount (slowTestRetries, fallbackRetryCount) { if (!slowTestRetries || !Object.keys(slowTestRetries).length) { return fallbackRetryCount } return getMaxEfdRetryCount(slowTestRetries) } function getTestCommand () { return `vitest ${process.argv.slice(2).join(' ')}` } function waitForHitProbe () { return new Promise(resolve => { realSetTimeout(() => { resolve() }, BREAKPOINT_HIT_GRACE_PERIOD_MS) }) } function isValidKnownTests (receivedKnownTests) { return !!receivedKnownTests.vitest } function getProvidedContext () { try { const { _ddIsEarlyFlakeDetectionEnabled, _ddIsDiEnabled, _ddKnownTests: knownTests, _ddEarlyFlakeDetectionNumRetries: numRepeats, _ddEarlyFlakeDetectionSlowTestRetries: slowTestRetries, _ddIsKnownTestsEnabled: isKnownTestsEnabled, _ddIsTestManagementTestsEnabled: isTestManagementTestsEnabled, _ddTestManagementAttemptToFixRetries: testManagementAttemptToFixRetries, _ddTestManagementTests: testManagementTests, _ddIsFlakyTestRetriesEnabled: isFlakyTestRetriesEnabled, _ddFlakyTestRetriesCount: flakyTestRetriesCount, _ddFlakyTestRetriesIncludesUnnamedProject: flakyTestRetriesIncludesUnnamedProject, _ddFlakyTestRetriesProjectNames: flakyTestRetriesProjectNames, _ddIsImpactedTestsEnabled: isImpactedTestsEnabled, _ddModifiedFiles: modifiedFiles, _ddTestSessionId: testSessionId, _ddTestModuleId: testModuleId, _ddTestCommand: testCommand, _ddRepositoryRoot: repositoryRoot, _ddCodeOwnersEntries: codeOwnersEntries, } = globalThis.__vitest_worker__.providedContext return { isDiEnabled: _ddIsDiEnabled, isEarlyFlakeDetectionEnabled: _ddIsEarlyFlakeDetectionEnabled, knownTests, numRepeats, slowTestRetries: slowTestRetries ?? {}, isKnownTestsEnabled, isTestManagementTestsEnabled, testManagementAttemptToFixRetries, testManagementTests, isFlakyTestRetriesEnabled, flakyTestRetriesCount: flakyTestRetriesCount ?? 0, flakyTestRetriesIncludesUnnamedProject, flakyTestRetriesProjectNames, isImpactedTestsEnabled, modifiedFiles, testSessionId, testModuleId, testCommand, repositoryRoot, codeOwnersEntries, } } catch { log.error('Vitest workers could not parse provided context, so some features will not work.') return { isDiEnabled: false, isEarlyFlakeDetectionEnabled: false, knownTests: {}, numRepeats: 0, slowTestRetries: {}, isKnownTestsEnabled: false, isTestManagementTestsEnabled: false, testManagementAttemptToFixRetries: 0, testManagementTests: {}, isFlakyTestRetriesEnabled: false, flakyTestRetriesCount: 0, flakyTestRetriesIncludesUnnamedProject: false, flakyTestRetriesProjectNames: undefined, isImpactedTestsEnabled: false, modifiedFiles: {}, testSessionId: undefined, testModuleId: undefined, testCommand: undefined, repositoryRoot: undefined, codeOwnersEntries: undefined, } } } function isReporterPackage (vitestPackage) { return vitestPackage.B?.name === 'BaseSequencer' } // from 2.0.0 function isReporterPackageNew (vitestPackage) { return vitestPackage.e?.name === 'BaseSequencer' } function isReporterPackageNewest (vitestPackage) { return vitestPackage.h?.name === 'BaseSequencer' } /** * Finds an export by its `.name` property in a minified vitest chunk. * Minified export keys change across versions, so we search by function/class name. * @param {object} pkg - The module exports object * @param {string} name - The `.name` value to look for * @returns {{ key: string, value: Function } | undefined} */ function findExportByName (pkg, name) { for (const [key, value] of Object.entries(pkg)) { if (value?.name === name) { return { key, value } } } } function getBaseSequencerExport (vitestPackage) { return findExportByName(vitestPackage, 'BaseSequencer') } function getChannelPromise (channelToPublishTo, frameworkVersion) { return new Promise(resolve => { channelToPublishTo.publish({ onDone: resolve, frameworkVersion }) }) } function isCliApiPackage (vitestPackage) { return !!findExportByName(vitestPackage, 'startVitest') } function getTestRunnerExport (testPackage) { return findExportByName(testPackage, 'VitestTestRunner') || findExportByName(testPackage, 'TestRunner') } function getVitestExport (vitestPackage) { return findExportByName(vitestPackage, 'Vitest') } function getForksPoolWorkerExport (vitestPackage) { return findExportByName(vitestPackage, 'ForksPoolWorker') } function getThreadsPoolWorkerExport (vitestPackage) { return findExportByName(vitestPackage, 'ThreadsPoolWorker') } function getSessionStatus (state) { if (state.getCountOfFailedTests() > 0) { return 'fail' } if (state.pathsSet.size === 0) { return 'skip' } return 'pass' } // From https://github.com/vitest-dev/vitest/blob/51c04e2f44d91322b334f8ccbcdb368facc3f8ec/packages/runner/src/run.ts#L243-L250 function getVitestTestStatus (test, retryCount) { if (test.result.state !== 'fail' && (!test.repeats || (test.retry ?? 0) === retryCount)) { return 'pass' } return 'fail' } function getTypeTasks (fileTasks, type = 'test') { const typeTasks = [] function getTasks (tasks) { for (const task of tasks) { if (task.type === type) { typeTasks.push(task) } else if (task.tasks) { getTasks(task.tasks) } } } getTasks(fileTasks) return typeTasks } function getTestName (task) { let testName = task.name let currentTask = task.suite while (currentTask) { if (currentTask.name) { testName = `${currentTask.name} ${testName}` } currentTask = currentTask.suite } return testName } function getFinalAttemptToFixStatus (task, state, isSwitchedStatus, testCtx) { if (isSwitchedStatus && attemptToFixTasks.has(task) && testCtx?.status) { return testCtx.status } return state === 'fail' ? 'fail' : 'pass' } function recordFinalAttemptToFixExecution (task, status, providedContext) { const statuses = attemptToFixTaskToStatuses.get(task) if (statuses && statuses.length <= providedContext.testManagementAttemptToFixRetries) { statuses.push(status) } recordAttemptToFixExecution(attemptToFixExecutions, { testSuite: getTestSuitePath(task.file.filepath, process.cwd()), testName: getTestName(task), status, isDisabled: disabledTasks.has(task), isQuarantined: quarantinedTasks.has(task), }) } function disableFrameworkRetries (task) { task.retry = 0 } /** * Vitest accumulates retry and repeat errors on one task result. The first error added since * the last reported attempt is the primary error for the failed attempt currently being reported. * * @param {object} task * @param {Array<object> | undefined} errors * @returns {object | undefined} */ function getCurrentAttemptTestError (task, errors) { if (!errors?.length) return const previousErrorCount = taskToReportedErrorCount.get(task) ?? 0 const testError = errors[previousErrorCount] ?? errors[0] taskToReportedErrorCount.set(task, errors.length) return testError } /** * Wraps a function so it runs inside the current test span context. * @param {object} task * @param {Function} fn * @returns {Function} */ function wrapTestScopedFn (task, fn) { return shimmer.wrapFunction(fn, fn => function (...args) { return testFnCh.runStores(taskToCtx.get(task), () => fn.apply(this, args)) }) } /** * Wraps a `beforeEach` cleanup callback so it inherits the test span context. * Vitest allows `beforeEach` to return a cleanup function, including via a promise. * @param {object} task * @param {unknown} result * @returns {unknown} */ function wrapBeforeEachCleanupResult (task, result) { if (typeof result === 'function') { return wrapTestScopedFn(task, result) } if (result && typeof result.then === 'function') { return result.then(cleanupFn => wrapBeforeEachCleanupResult(task, cleanupFn)) } return result } function getWorkspaceProject (ctx) { return ctx.getCoreWorkspaceProject ? ctx.getCoreWorkspaceProject() : ctx.getRootProject() } function setProvidedContext (ctx, values, warningMessage) { try { Object.assign(getWorkspaceProject(ctx)._provided, values) } catch { log.warn(warningMessage) } } function getTestFilepathsFromSpecifications (testSpecifications) { if (!Array.isArray(testSpecifications) || !testSpecifications.length) { return } return testSpecifications.map(testSpecification => { const testFile = Array.isArray(testSpecification) ? testSpecification[1] : testSpecification return testFile?.moduleId || testFile?.filepath || testFile }) } function getTestFilepaths (ctx, testSpecifications) { const testFilepaths = getTestFilepathsFromSpecifications(testSpecifications) if (testFilepaths) { return testFilepaths } const getFilePaths = ctx.getTestFilepaths || ctx._globTestFilepaths return getFilePaths.call(ctx) } function wrapCoverageProvider (ctx) { const { coverageProvider } = ctx if (!coverageProvider?.generateCoverage || coverageWrappedProviders.has(coverageProvider)) { return } coverageWrappedProviders.add(coverageProvider) // Capture coverage root directory from config (default is 'coverage' in cwd) try { const coverageConfig = ctx.config?.coverage const reportsDirectory = coverageConfig?.reportsDirectory || 'coverage' const rootDir = ctx.config?.root || process.cwd() coverageRootDir = path.isAbsolute(reportsDirectory) ? reportsDirectory : path.join(rootDir, reportsDirectory) } catch { // Fallback to cwd if we can't get config coverageRootDir = process.cwd() } shimmer.wrap(coverageProvider, 'generateCoverage', generateCoverage => async function () { const totalCodeCoverage = await generateCoverage.apply(this, arguments) try { testCodeCoverageLinesTotal = totalCodeCoverage.getCoverageSummary().lines.pct } catch { // ignore errors } return totalCodeCoverage }) } function wrapSessionFinish (ctx) { if (finishWrappedContexts.has(ctx)) { return } finishWrappedContexts.add(ctx) shimmer.wrap(ctx, 'exit', getFinishWrapper) shimmer.wrap(ctx, 'close', getFinishWrapper) } async function runMainProcessSetup (ctx, frameworkVersion, testSpecifications) { if (!testSessionFinishCh.hasSubscribers) { return } try { const { err, libraryConfig } = await getChannelPromise(libraryConfigurationCh, frameworkVersion) if (!err) { isFlakyTestRetriesEnabled = libraryConfig.isFlakyTestRetriesEnabled flakyTestRetriesCount = libraryConfig.flakyTestRetriesCount isEarlyFlakeDetectionEnabled = libraryConfig.isEarlyFlakeDetectionEnabled earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries earlyFlakeDetectionSlowTestRetries = libraryConfig.earlyFlakeDetectionSlowTestRetries ?? {} isDiEnabled = libraryConfig.isDiEnabled isKnownTestsEnabled = libraryConfig.isKnownTestsEnabled isTestManagementTestsEnabled = libraryConfig.isTestManagementEnabled testManagementAttemptToFixRetries = libraryConfig.testManagementAttemptToFixRetries isImpactedTestsEnabled = libraryConfig.isImpactedTestsEnabled } } catch { isFlakyTestRetriesEnabled = false isEarlyFlakeDetectionEnabled = false isDiEnabled = false isKnownTestsEnabled = false isImpactedTestsEnabled = false } if (testSessionConfigurationCh.hasSubscribers) { const { testSessionId, testModuleId, testCommand, repositoryRoot, codeOwnersEntries } = await getChannelPromise( testSessionConfigurationCh, frameworkVersion ) setProvidedContext(ctx, { _ddTestSessionId: testSessionId, _ddTestModuleId: testModuleId, _ddTestCommand: testCommand, _ddRepositoryRoot: repositoryRoot, _ddCodeOwnersEntries: codeOwnersEntries, }, 'Could not send test session configuration to workers.') } const { knownTestsResponse, testManagementTestsResponse, } = await getTestOptimizationRequestResults({ isKnownTestsEnabled, isTestManagementTestsEnabled, getKnownTests: () => getChannelPromise(knownTestsCh), getTestManagementTests: () => getChannelPromise(testManagementTestsCh), }) const flakyTestRetriesConfiguration = configureFlakyTestRetries(ctx, testSpecifications) if (flakyTestRetriesConfiguration) { setProvidedContext(ctx, { _ddIsFlakyTestRetriesEnabled: isFlakyTestRetriesEnabled, _ddFlakyTestRetriesCount: flakyTestRetriesCount, _ddFlakyTestRetriesIncludesUnnamedProject: flakyTestRetriesConfiguration.includesUnnamedProject, _ddFlakyTestRetriesProjectNames: flakyTestRetriesConfiguration.projectNames, }, 'Could not send library configuration to workers.') } if (isKnownTestsEnabled) { const currentKnownTestsResponse = knownTestsResponse || await getChannelPromise(knownTestsCh) if (currentKnownTestsResponse.err) { isEarlyFlakeDetectionEnabled = false } else { const knownTests = currentKnownTestsResponse.knownTests const testFilepaths = await getTestFilepaths(ctx, testSpecifications) if (isValidKnownTests(knownTests)) { isEarlyFlakeDetectionFaultyCh.publish({ knownTests: knownTests.vitest, testFilepaths, onDone: (isFaulty) => { isEarlyFlakeDetectionFaulty = isFaulty }, }) if (isEarlyFlakeDetectionFaulty) { isEarlyFlakeDetectionEnabled = false log.warn('New test detection is disabled because the number of new tests is too high.') } else { setProvidedContext(ctx, { _ddIsKnownTestsEnabled: isKnownTestsEnabled, _ddKnownTests: knownTests, _ddIsEarlyFlakeDetectionEnabled: isEarlyFlakeDetectionEnabled, _ddEarlyFlakeDetectionNumRetries: getConfiguredEfdRetryCount(earlyFlakeDetectionSlowTestRetries, earlyFlakeDetectionNumRetries), _ddEarlyFlakeDetectionSlowTestRetries: earlyFlakeDetectionSlowTestRetries, }, 'Could not send known tests to workers so Early Flake Detection will not work.') } } else { isEarlyFlakeDetectionFaulty = true isEarlyFlakeDetectionEnabled = false } } } if (isDiEnabled) { setProvidedContext(ctx, { _ddIsDiEnabled: isDiEnabled, }, 'Could not send Dynamic Instrumentation configuration to workers.') } if (isTestManagementTestsEnabled) { const { err, testManagementTests: receivedTestManagementTests } = testManagementTestsResponse || await getChannelPromise(testManagementTestsCh) if (err) { isTestManagementTestsEnabled = false log.error('Could not get test management tests.') } else { setProvidedContext(ctx, { _ddIsTestManagementTestsEnabled: isTestManagementTestsEnabled, _ddTestManagementAttemptToFixRetries: testManagementAttemptToFixRetries, _ddTestManagementTests: receivedTestManagementTests, }, 'Could not send test management tests to workers so Test Management will not work.') } } if (isImpactedTestsEnabled) { const { err, modifiedFiles } = await getChannelPromise(modifiedFilesCh) if (err) { log.error('Could not get modified tests.') } else { setProvidedContext(ctx, { _ddIsImpactedTestsEnabled: isImpactedTestsEnabled, _ddModifiedFiles: modifiedFiles, }, 'Could not send modified tests to workers so Impacted Tests will not work.') } } wrapCoverageProvider(ctx) wrapSessionFinish(ctx) } function ensureMainProcessSetup (ctx, frameworkVersion, testSpecifications) { let setupPromise = mainProcessSetupPromises.get(ctx) if (!setupPromise) { setupPromise = runMainProcessSetup(ctx, frameworkVersion, testSpecifications) mainProcessSetupPromises.set(ctx, setupPromise) } return setupPromise } /** * Configure Vitest retries for the root project and resolved workspace projects. * * @param {object} ctx * @param {unknown[]|undefined} testSpecifications * @returns {{ projectNames: string[], includesUnnamedProject: boolean }|undefined} */ function configureFlakyTestRetries (ctx, testSpecifications) { if (!isFlakyTestRetriesEnabled || flakyTestRetriesCount <= 0) return let configured = false let includesUnnamedProject = false const projectNames = [] for (const { config, projectName } of getVitestProjectConfigs(ctx, testSpecifications)) { if (!config.retry) { config.retry = flakyTestRetriesCount configured = true if (projectName) { projectNames.push(projectName) } else { includesUnnamedProject = true } } } if (!configured) return return { includesUnnamedProject, projectNames, } } /** * Return unique Vitest configs that can be used to run tests. * * @param {object} ctx * @param {unknown[]|undefined} testSpecifications * @returns {{ config: object, projectName?: string }[]} */ function getVitestProjectConfigs (ctx, testSpecifications) { const entries = [] addTestSpecificationConfigs(entries, testSpecifications) if (entries.length > 0) { return entries } const selectedProjectNames = getSelectedProjectNames() addSelectedInlineProjectConfigs(entries, safeConfig(ctx), selectedProjectNames) addSelectedRuntimeProjectConfigs(entries, ctx?.projects, selectedProjectNames) if (entries.length > 0) { return entries } if (Array.isArray(ctx?.projects)) { for (const project of ctx.projects) { addConfig(entries, safeConfig(project), getProjectName(project)) } if (entries.length > 0) { return entries } } addConfig(entries, safeConfig(ctx)) addConfig(entries, safeConfig(safeWorkspaceProject(ctx))) return entries } /** * Add configs from runnable test specifications once. * * @param {{ config: object, projectName?: string }[]} entries * @param {unknown[]|undefined} testSpecifications */ function addTestSpecificationConfigs (entries, testSpecifications) { if (!Array.isArray(testSpecifications)) return for (const testSpecification of testSpecifications) { const project = getTestSpecificationProject(testSpecification) addConfig(entries, safeConfig(project), getProjectName(project)) } } /** * Add selected inline project configs from the root Vitest config once. * * @param {{ config: object, projectName?: string }[]} entries * @param {object|undefined} rootConfig * @param {string[]} selectedProjectNames */ function addSelectedInlineProjectConfigs (entries, rootConfig, selectedProjectNames) { if (selectedProjectNames.length === 0 || !Array.isArray(rootConfig?.projects)) return for (const project of rootConfig.projects) { const config = getInlineProjectConfig(project) const projectName = getProjectName(project) if (selectedProjectNames.includes(projectName)) { addConfig(entries, config, projectName) } } } /** * Add selected resolved project configs once. * * @param {{ config: object, projectName?: string }[]} entries * @param {unknown[]|undefined} projects * @param {string[]} selectedProjectNames */ function addSelectedRuntimeProjectConfigs (entries, projects, selectedProjectNames) { if (selectedProjectNames.length === 0 || !Array.isArray(projects)) return for (const project of projects) { const projectName = getProjectName(project) if (selectedProjectNames.includes(projectName)) { addConfig(entries, safeConfig(project), projectName) } } } /** * Return selected project names from the Vitest CLI arguments. * * @returns {string[]} */ function getSelectedProjectNames () { const names = [] for (let index = 0; index < process.argv.length; index++) { const argument = process.argv[index] if (argument === '--project' && process.argv[index + 1]) { names.push(process.argv[index + 1]) index++ } else if (argument.startsWith('--project=')) { names.push(argument.slice('--project='.length)) } } return names } /** * Return the test config from an inline Vitest project entry. * * @param {unknown} project * @returns {object|undefined} */ function getInlineProjectConfig (project) { return project?.test || project } /** * Return a Vitest project name from runtime or inline project objects. * * @param {unknown} project * @returns {string|undefined} */ function getProjectName (project) { return normalizeProjectName(project?.name || project?.config?.name || project?.test?.name) } /** * Return a normalized Vitest project name. * * @param {unknown} name * @returns {string|undefined} */ function normalizeProjectName (name) { if (typeof name === 'string') return name const label = name?.label return typeof label === 'string' ? label : undefined } /** * Add a config object once. * * @param {{ config: object, projectName?: string }[]} entries * @param {object|undefined} config * @param {string|undefined} projectName */ function addConfig (entries, config, projectName) { if (config && !entries.some(entry => entry.config === config || (projectName && entry.projectName === projectName))) { entries.push({ config, projectName }) } } /** * Read a Vitest config object without assuming the project is initialized. * * @param {object|undefined} project * @returns {object|undefined} */ function safeConfig (project) { let config try { config = project?.config } catch {} return config } /** * Read the workspace project without assuming the root server is initialized. * * @param {object} ctx * @returns {object|undefined} */ function safeWorkspaceProject (ctx) { let project try { project = getWorkspaceProject(ctx) } catch {} return project } /** * Return whether Datadog configured ATR retries for a task. * * @param {object} providedContext * @param {object} task * @returns {boolean} */ function isFlakyTestRetriesEnabledForTask (providedContext, task) { if (!providedContext.isFlakyTestRetriesEnabled) return false const { flakyTestRetriesProjectNames } = providedContext if (!Array.isArray(flakyTestRetriesProjectNames)) return true const projectName = task.file?.projectName if (!projectName) { return providedContext.flakyTestRetriesIncludesUnnamedProject === true } return flakyTestRetriesProjectNames.includes(projectName) } function getSortWrapper (sort, frameworkVersion) { return async function () { await ensureMainProcessSetup(this.ctx, frameworkVersion, arguments[0]) return sort.apply(this, arguments) } } function getFinishWrapper (exitOrClose) { let isClosed = false return async function () { if (isClosed) { // needed because exit calls close return exitOrClose.apply(this, arguments) } isClosed = true if (!testSessionFinishCh.hasSubscribers) { return exitOrClose.apply(this, arguments) } let onFinish const flushPromise = new Promise(resolve => { onFinish = resolve }) const failedSuites = this.state.getFailedFilepaths() let error if (failedSuites.length) { error = new Error(`Test suites failed: ${failedSuites.length}.`) } testSessionFinishCh.publish({ status: getSessionStatus(this.state), testCodeCoverageLinesTotal, error, isEarlyFlakeDetectionEnabled, isEarlyFlakeDetectionFaulty, isTestManagementTestsEnabled, vitestPool, onFinish, }) logTestOptimizationSummary({ attemptToFixExecutions, newTestsWithDynamicNames }) loggedAttemptToFixTests.clear() await flushPromise // If coverage was generated, publish coverage report channel for upload if (coverageRootDir && codeCoverageReportCh.hasSubscribers) { await new Promise((resolve) => { codeCoverageReportCh.publish({ rootDir: coverageRootDir, onDone: resolve }) }) } return exitOrClose.apply(this, arguments) } } function getCliOrStartVitestWrapper (frameworkVersion) { return function (oldCliOrStartVitest) { return function (...args) { if (!testSessionStartCh.hasSubscribers || isSessionStarted) { return oldCliOrStartVitest.apply(this, args) } isSessionStarted = true testSessionStartCh.publish({ command: getTestCommand(), frameworkVersion }) return oldCliOrStartVitest.apply(this, args) } } } function isForkPool (pool) { return pool === 'forks' || pool === 'vmForks' } function isThreadPool (pool) { return pool === 'threads' || pool === 'vmThreads' } /** * Return the project object attached to a Vitest test specification. * * @param {unknown} testSpecification * @returns {object|undefined} */ function getTestSpecificationProject (testSpecification) { if (Array.isArray(testSpecification)) { return testSpecification[0] } return testSpecification?.project } function getTestSpecificationPool (testSpecification) { const project = getTestSpecificationProject(testSpecification) return project?.config?.pool || project?.serializedConfig?.pool || project?.pool || testSpecification?.pool } function hasForkPoolTestSpecification (testSpecifications) { if (!Array.isArray(testSpecifications)) { return false } for (const testSpecification of testSpecifications) { if (isForkPool(getTestSpecificationPool(testSpecification))) { return true } } return false } function shouldMarkVitestWorkerEnv (pool, testSpecifications) { return isForkPool(pool) || hasForkPoolTestSpecification(testSpecifications) || (!testSpecifications && !isThreadPool(pool)) } function markVitestWorkerEnv (ctx, testSpecifications) { const config = ctx?.config if (!config || !shouldMarkVitestWorkerEnv(config.pool, testSpecifications)) { return } config.env = config.env || {} config.env.DD_VITEST_WORKER = '1' } function wrapVitestRunFiles (Vitest, frameworkVersion) { if (!Vitest?.prototype?.runFiles) { return } shimmer.wrap(Vitest.prototype, 'runFiles', runFiles => async function (testSpecifications) { markVitestWorkerEnv(this, testSpecifications) await ensureMainProcessSetup(this, frameworkVersion, testSpecifications) return runFiles.apply(this, arguments) }) if (Vitest.prototype.collectTests) { shimmer.wrap(Vitest.prototype, 'collectTests', collectTests => function () { markVitestWorkerEnv(this) return collectTests.apply(this, arguments) }) } } function getCreateCliWrapper (vitestPackage, frameworkVersion) { const createCliExport = findExportByName(vitestPackage, 'createCLI') if (!createCliExport) { return vitestPackage } shimmer.wrap(vitestPackage, createCliExport.key, getCliOrStartVitestWrapper(frameworkVersion)) return vitestPackage } function threadHandler (thread) { const { runtime } = thread let workerProcess if (runtime === 'child_process') { vitestPool = 'child_process' workerProcess = thread.process } else if (runtime === 'worker_threads') { vitestPool = 'worker_threads' workerProcess = thread.thread } else { vitestPool = 'unknown' } if (!workerProcess) { log.error('Vitest error: could not get process or thread from TinyPool#run') return } if (workerProcesses.has(workerProcess)) { return } workerProcesses.add(workerProcess) workerProcess.on('message', (message) => { if (message.__tinypool_worker_message__ && message.data) { if (message.interprocessCode === VITEST_WORKER_TRACE_PAYLOAD_CODE) { collectTestOptimizationSummariesFromTraces(message.data, { newTestsWithDynamicNames, attemptToFixExecutions, }) workerReportTraceCh.publish(message.data) } else if (message.interprocessCode === VITEST_WORKER_LOGS_PAYLOAD_CODE) { workerReportLogsCh.publish(message.data) } } }) } function wrapTinyPoolRun (TinyPool) { shimmer.wrap(TinyPool.prototype, 'run', run => async function () { // We have to do this before and after because the threads list gets recycled, that is, the processes are re-created // eslint-disable-next-line unicorn/no-array-for-each this.threads.forEach(threadHandler) const runResult = await run.apply(this, arguments) // eslint-disable-next-line unicorn/no-array-for-each this.threads.forEach(threadHandler) return runResult }) } addHook({ name: 'tinypool', // version from tinypool@0.8 was used in vitest@1.6.0 versions: ['>=0.8.0'], }, (TinyPool) => { wrapTinyPoolRun(TinyPool) return TinyPool }) function getWrappedOn (on) { return function (event, callback) { if (event !== 'message') { return on.apply(this, arguments) } // `arguments[1]` is the callback function, which // we modify to intercept our messages to not interfere // with vitest's own messages arguments[1] = shimmer.wrapFunction(callback, callback => function (message) { if (message.type !== 'Buffer' && Array.isArray(message)) { const [interprocessCode, data] = message if (interprocessCode === VITEST_WORKER_TRACE_PAYLOAD_CODE) { collectTestOptimizationSummariesFromTraces(data, { newTestsWithDynamicNames, attemptToFixExecutions, }) workerReportTraceCh.publish(data) } else if (interprocessCode === VITEST_WORKER_LOGS_PAYLOAD_CODE) { workerReportLogsCh.publish(data) } // If we execute the callback vitest crashes, as the message is not supported return } return callback.apply(this, arguments) }) return on.apply(this, arguments) } } function getStartVitestWrapper (cliApiPackage, frameworkVersion) { if (!isCliApiPackage(cliApiPackage)) { return cliApiPackage } const startVitestExport = findExportByName(cliApiPackage, 'startVitest') shimmer.wrap(cliApiPackage, startVitestExport.key, getCliOrStartVitestWrapper(frameworkVersion)) const vitest = getVitestExport(cliApiPackage) if (vitest) { wrapVitestRunFiles(vitest.value, frameworkVersion) } const forksPoolWorker = getForksPoolWorkerExport(cliApiPackage) if (forksPoolWorker) { // function is async shimmer.wrap(forksPoolWorker.value.prototype, 'start', start => function (...args) { vitestPool = 'child_process' this.env.DD_VITEST_WORKER = '1' return start.apply(this, args) }) shimmer.wrap(forksPoolWorker.value.prototype, 'on', getWrappedOn) } const threadsPoolWorker = getThreadsPoolWorkerExport(cliApiPackage) if (threadsPoolWorker) { // function is async shimmer.wrap(threadsPoolWorker.value.prototype, 'start', start => function (...args) { vitestPool = 'worker_threads' this.env.DD_VITEST_WORKER = '1' return start.apply(this, args) }) shimmer.wrap(threadsPoolWorker.value.prototype, 'on', getWrappedOn) } return cliApiPackage } function wrapVitestTestRunner (VitestTestRunner) { // `onBeforeRunTask` is run before any repetition or attempt is run // `onBeforeRunTask` is an async function shimmer.wrap(VitestTestRunner.prototype, 'onBeforeRunTask', onBeforeRunTask => function (task) { const testName = getTestName(task) const { knownTests, isEarlyFlakeDetectionEnabled, isKnownTestsEnabled, numRepeats, isTestManagementTestsEnabled, testManagementAttemptToFixRetries, testManagementTests, isImpactedTestsEnabled, modifiedFiles, } = getProvidedContext() if (isTestManagementTestsEnabled) { isAttemptToFixCh.publish({ testManagementTests, testSuiteAbsolutePath: task.file.filepath, testName, onDone: (isAttemptToFix) => { if (isAttemptToFix) { isRetryReasonAttemptToFix = task.repeats !== testManagementAttemptToFixRetries disableFrameworkRetries(task) task.repeats = testManagementAttemptToFixRetries attemptToFixTasks.add(task) attemptToFixTaskToStatuses.set(task, []) } }, }) isDisabledCh.publish({ testManagementTests, testSuiteAbsolutePath: task.file.filepath, testName, onDone: (isTestDisabled) => { if (isTestDisabled) { disabledTasks.add(task) if (!attemptToFixTasks.has(task)) { // we only actually skip if the test is not being attempted to be fixed task.mode = 'skip' } } }, }) } if (isImpactedTestsEnabled) { isModifiedCh.publish({ modifiedFiles, testSuiteAbsolutePath: task.file.filepath, onDone: (isImpacted) => { if (isImpacted) { if (isEarlyFlakeDetectionEnabled) { isRetryReasonEfd = true disableFrameworkRetries(task) task.repeats = numRepeats } modifiedTasks.add(task) taskToStatuses.set(task, []) } }, }) } if (isKnownTestsEnabled) { isNewTestCh.publish({ knownTests, testSuiteAbsolutePath: task.file.filepath, testName, onDone: (isNew) => { if (isNew && !attemptToFixTasks.has(task)) { if (isEarlyFlakeDetectionEnabled && !modifiedTasks.has(task)) { isRetryReasonEfd = true disableFrameworkRetries(task) task.repeats = numRepeats } newTasks.add(task) taskToStatuses.set(task, []) if (DYNAMIC_NAME_RE.test(testName)) { dynamicNameTasks.add(task) } } }, }) } return onBeforeRunTask.apply(this, arguments) }) // `onAfterRunTask` is run after all repetitions or attempts are run // `onAfterRunTask` is an async function shimmer.wrap(VitestTestRunner.prototype, 'onAfterRunTask', onAfterRunTask => function (task) { const { isEarlyFlakeDetectionEnabled, isTestManagementTestsEnabled } = getProvidedContext() if (isTestManagementTestsEnabled) { const isAttemptingToFix = attemptToFixTasks.has(task) const isQuarantined = quarantinedTasks.has(task) if (isAttemptingToFix) { const statuses = attemptToFixTaskToStatuses.get(task) if (task.result.state === 'pass' && statuses?.includes('fail')) { switchedStatuses.add(task) task.result.state = 'fail' } } if (!isAttemptingToFix && isQuarantined) { if (task.result.state === 'fail') { switchedStatuses.add(task) } task.result.state = 'pass' } } if (isEarlyFlakeDetectionEnabled && taskToStatuses.has(task) && !attemptToFixTasks.has(task)) { const statuses = taskToStatuses.get(task) // If the test has passed at least once, we consider it passed if (statuses.includes('pass')) { if (task.result.state === 'fail') { switchedStatuses.add(task) } task.result.state = 'pass' } } return onAfterRunTask.apply(this, arguments) }) // test start (only tests that are not marked as skip or todo) // `onBeforeTryTask` is run for every repetition and attempt of the test shimmer.wrap(VitestTestRunner.prototype, 'onBeforeTryTask', onBeforeTryTask => async function (task, retryInfo) { if (!testPassCh.hasSubscribers && !testErrorCh.hasSubscribers && !testSkipCh.hasSubscribers) { return onBeforeTryTask.apply(this, arguments) } const testName = getTestName(task) let isNew = false let isQuarantined = false const providedContext = getProvidedContext() const { isKnownTestsEnabled, isEarlyFlakeDetectionEnabled, isDiEnabled, isTestManagementTestsEnabled, testManagementTests, slowTestRetries, } = providedContext if (isKnownTestsEnabled) { isNew = newTasks.has(task) } if (isTestManagementTestsEnabled) { isQuarantinedCh.publish({ testManagementTests, testSuiteAbsolutePath: task.file.filepath, testName, onDone: (isTestQuarantined) => { isQuarantined = isTestQuarantined if (isTestQuarantined) { quarantinedTasks.add(task) } }, }) } const { retry: numAttempt, repeats: numRepetition } = retryInfo const isEfdManagedTask = isEarlyFlakeDetectionEnabled && taskToStatuses.has(task) && !attemptToFixTasks.has(task) if (isEfdManagedTask && numRepetition > 0 && !efdDeterminedRetries.has(task)) { const previousExecutionStart = efdExecutionStartByTask.get(task) const duration = previousExecutionStart === undefined ? task.result?.duration ?? 0 : performance.now() - previousExecutionStart const retryCount = getEfdRetryCount(duration, slowTestRetries) efdDeterminedRetries.set(task, retryCount) task.repeats = retryCount if (retryCount === 0) { efdSlowAbortedTasks.add(task) } } const efdRetryCount = efdDeterminedRetries.get(task) if (isEfdManagedTask && efdRetryCount !== undefined && numRepetition > efdRetryCount) { if (task.result) { efdSkippedRetryResults.set(task, { ...task.result, errors: task.result.errors?.slice(), }) } if (vitestSetFn) { const noop = function () {} noop.__ddTraceWrapped = true vitestSetFn(task, noop) } return onBeforeTryTask.apply(this, arguments) } if (isEfdManagedTask) { efdExecutionStartByTask.set(task, performance.now()) } // We finish the previous test here because we know it has failed already if (numAttempt > 0) { const shouldWaitForHitProbe = isDiEnabled && numAttempt > 1 if (shouldWaitForHitProbe) { await waitForHitProbe() } const promises = {} const shouldSetProbe = isDiEnabled && numAttempt === 1 const ctx = taskToCtx.get(task) const testError = getCurrentAttemptTestError(task, task.result?.errors) if (ctx) { testErrorCh.publish({ error: testError, shouldSetProbe, promises, ...ctx.currentStore, }) // We wait for the probe to be set if (promises.setProbePromise) { await promises.setProbePromise } } } const lastExecutionStatus = task.result.state const isAtf = attemptToFixTasks.has(task) const shouldTrackStatuses = isEarlyFlakeDetectionEnabled || isAtf const shouldFlipStatus = isEarlyFlakeDetectionEnabled || isAtf const statuses = isAtf ? attemptToFixTaskToStatuses.get(task) : taskToStatuses.get(task) // These clauses handle task.repeats, whether EFD is enabled or not // The only thing that EFD does is to forcefully pass the test if it has passed at least once if (numRepetition > 0 && numRepetition < task.repeats) { // it may or may have not failed // Here we finish the earlier iteration, // as long as it's not the _last_ iteration (which will be finished normally) const ctx = taskToCtx.get(task) if (ctx) { if (lastExecutionStatus === 'fail') { const testError = getCurrentAttemptTestError(task, task.result?.errors) testErrorCh.publish({ error: testError, ...ctx.currentStore }) } else { testPassCh.publish({ task, ...ctx.currentStore }) } if (shouldTrackStatuses && statuses) { statuses.push(lastExecutionStatus) } if (shouldFlipStatus) { // If we don't "reset" the result.state to "pass", once a repetition fails, // vitest will always consider the test as failed, so we can't read the actual status // This means that we change vitest's behavior: // if the last attempt passes, vitest would consider the test as failed // but after this change, it will consider the test as passed task.result.state = 'pass' } } } else if (numRepetition === task.repeats) { if (shouldTrackStatuses && statuses) { statuses.push(lastExecutionStatus) } const ctx = taskToCtx.get(task) if (lastExecutionStatus === 'fail') { const testError = getCurrentAttemptTestError(task, task.result?.errors) testErrorCh.publish({ error: testError, ...ctx.currentStore }) } else { testPassCh.publish({ task, ...ctx.currentStore }) } if (shouldFlipStatus) { task.result.state = 'pass' } } const isRetryReasonAtr = numAttempt > 0 && isFlakyTestRetriesEnabledForTask(providedContext, task) && !isRetryReasonAttemptToFix && !isRetryReasonEfd const ctx = { testName, testSuiteAbsolutePath: task.file.filepath, isRetry: numAttempt > 0 || numRepetition > 0, isRetryReasonEfd, isRetryReasonAttemptToFix: isRetryReasonAttemptToFix && numRepetition > 0, isNew, hasDynamicName: dynamicNameTasks.has(task), mightHitProbe: isDiEnabled && numAttempt > 0, isAttemptToFix: attemptToFixTasks.has(task), isDisabled: disabledTasks.has(task), isQuarantined, isRetryReasonAtr, isModified: modifiedTasks.has(task), } taskToCtx.set(task, ctx) if (attemptToFixTasks.has(task)) { logAttemptToFixTestExecution( getTestSuitePath(task.file.filepath, process.cwd()), testName, loggedAttemptToFixTests ) } testStartCh.runStores(ctx, () => {}) // Wrap the test function so it runs inside the test span context. // Without this, HTTP requests during test execution become orphaned root spans. if (vitestGetFn && vitestSetFn) { const originalFn = vitestGetFn(task) if (originalFn && !originalFn.__ddTraceWrapped) { const wrappedFn = wrapTestScopedFn(task, originalFn) wrappedFn.__ddTraceWrapped = true vitestSetFn(task, wrappedFn) } } // Wrap beforeEach/afterEach hooks so they also run inside the test span context. // In vitest 4+, hooks are in a WeakMap accessed via getHooks(). In older versions, they're on suite.hooks. let currentSuite = task.suite while (currentSuite) { const hooks = vitestGetHooks ? vitestGetHooks(currentSuite) : currentSuite.hooks if (hooks) { for (const hookType of ['beforeEach', 'afterEach']) { const hookArray = hooks[hookType] if (!hookArray) continue for (let i = 0; i < hookArray.length; i++) { const currentFn = hookArray[i] const originalFn = originalHookFns.get(currentFn) || currentFn const wrappedFn = shimmer.wrapFunction(originalFn, fn => function (...args) { const result = testFnCh.runStores(taskToCtx.get(task), () => fn.apply(this, args)) if (hookType === 'beforeEach') { return wrapBeforeEachCleanupResult(task, result) } return result }) originalHookFns.set(wrappedFn, originalFn) hookArray[i] = wrappedFn } } } currentSuite = currentSuite.suite } return onBeforeTryTask.apply(this, arguments) }) // test finish (only passed tests) shimmer.wrap(VitestTestRunner.prototype, 'onAfterTryTask', onAfterTryTask => async function (task, retryInfo) { if (!testPassCh.hasSubscribers && !testErrorCh.hasSubscribers && !testSkipCh.hasSubscribers) { return onAfterTryTask.apply(this, arguments) } const result = await onAfterTryTask.apply(this, arguments) const { isEarlyFlakeDetectionEnabled, testManagementAttemptToFixRetries, slowTestRetries, } = getProvidedContext() const status = getVitestTestStatus(task, retryInfo.retry) const ctx = taskToCtx.get(task) const { isDiEnabled } = getProvidedContext() if (efdSkippedRetryResults.has(task)) { task.result = efdSkippedRetryResults.get(task) efdSkippedRetryResults.delete(task) return result } if (isDiEnabled && retryInfo.retry > 1) { await waitForHitProbe() } if ( isEarlyFlakeDetectionEnabled && (retryInfo.repeats ?? 0) === 0 && taskToStatuses.has(task) && !attemptToFixTasks.has(task) && !efdDeterminedRetries.has(task) ) { const executionStart = efdExecutionStartByTask.get(task) const duration = executionStart === undefined ? task.result?.duration ?? 0 : performance.now() - executionStart const retryCount = getEfdRetryCount(duration, slowTestRetries) efdDeterminedRetries.set(task, retryCount) task.repeats = retryCount if (retryCount === 0) { efdSlowAbortedTasks.add(task) } } let attemptToFixPassed = false let attemptToFixFailed = false if (attemptToFixTasks.has(task)) { const statuses = attemptToFixTaskToStatuses.get(task) if (statuses.length === testManagementAttemptToFixRetries) { if (status === 'pass' && statuses.every(status => status === 'pass')) { attemptToFixPassed = true } else if (status === 'fail' || statuses.includes('fail')) { attemptToFixFailed = true } } } if (ctx) { // We don't finish here because the test might fail in a later hook (afterEach) ctx.status = status ctx.task = task ctx.attemptToFixPassed = attemptToFixPassed ctx.attemptToFixFailed = attemptToFixFailed testFinishTimeCh.runStores(ctx, () =