codeceptjs
Version:
Supercharged End 2 End Testing Framework for NodeJS
1,290 lines (1,141 loc) • 115 kB
JavaScript
// @ts-nocheck
// TypeScript: Import Node.js types for process, fs, path, etc.
/// <reference types="node" />
const fs = require('fs')
const path = require('path')
const mkdirp = require('mkdirp')
const crypto = require('crypto')
const { threadId } = require('worker_threads')
const { template } = require('../utils')
const { getMachineInfo } = require('../command/info')
const event = require('../event')
const output = require('../output')
const Codecept = require('../codecept')
const defaultConfig = {
output: typeof global !== 'undefined' && global.output_dir ? global.output_dir : './output',
reportFileName: 'report.html',
includeArtifacts: true,
showSteps: true,
showSkipped: true,
showMetadata: true,
showTags: true,
showRetries: true,
exportStats: false,
exportStatsPath: './stats.json',
keepHistory: false,
historyPath: './test-history.json',
maxHistoryEntries: 50,
}
/**
* HTML Reporter Plugin for CodeceptJS
*
* Generates comprehensive HTML reports showing:
* - Test statistics
* - Feature/Scenario details
* - Individual step results
* - Test artifacts (screenshots, etc.)
*
* ## Configuration
*
* ```js
* "plugins": {
* "htmlReporter": {
* "enabled": true,
* "output": "./output",
* "reportFileName": "report.html",
* "includeArtifacts": true,
* "showSteps": true,
* "showSkipped": true,
* "showMetadata": true,
* "showTags": true,
* "showRetries": true,
* "exportStats": false,
* "exportStatsPath": "./stats.json",
* "keepHistory": false,
* "historyPath": "./test-history.json",
* "maxHistoryEntries": 50
* }
* }
* ```
*/
module.exports = function (config) {
const options = { ...defaultConfig, ...config }
/**
* TypeScript: Explicitly type reportData arrays as any[] to avoid 'never' errors
*/
let reportData = {
stats: {},
tests: [],
failures: [],
hooks: [],
startTime: null,
endTime: null,
retries: [],
config: options,
}
let currentTestSteps = []
let currentTestHooks = []
let currentBddSteps = [] // Track BDD/Gherkin steps
let testRetryAttempts = new Map() // Track retry attempts per test
let currentSuite = null // Track current suite for BDD detection
// Initialize report directory
const reportDir = options.output ? path.resolve(global.codecept_dir, options.output) : path.resolve(global.output_dir || './output')
mkdirp.sync(reportDir)
// Track overall test execution
event.dispatcher.on(event.all.before, () => {
reportData.startTime = new Date().toISOString()
output.print('HTML Reporter: Starting HTML report generation...')
})
// Track test start to initialize steps and hooks collection
event.dispatcher.on(event.test.before, test => {
currentTestSteps = []
currentTestHooks = []
currentBddSteps = []
// Track current suite for BDD detection
currentSuite = test.parent
// Enhanced retry detection with priority-based approach
const testId = generateTestId(test)
// Only set retry count if not already set, using priority order
if (!testRetryAttempts.has(testId)) {
// Method 1: Check retryNum property (most reliable)
if (test.retryNum && test.retryNum > 0) {
testRetryAttempts.set(testId, test.retryNum)
output.debug(`HTML Reporter: Retry count detected (retryNum) for ${test.title}, attempts: ${test.retryNum}`)
}
// Method 2: Check currentRetry property
else if (test.currentRetry && test.currentRetry > 0) {
testRetryAttempts.set(testId, test.currentRetry)
output.debug(`HTML Reporter: Retry count detected (currentRetry) for ${test.title}, attempts: ${test.currentRetry}`)
}
// Method 3: Check if this is a retried test
else if (test.retriedTest && test.retriedTest()) {
const originalTest = test.retriedTest()
const originalTestId = generateTestId(originalTest)
if (!testRetryAttempts.has(originalTestId)) {
testRetryAttempts.set(originalTestId, 1) // Start with 1 retry
} else {
testRetryAttempts.set(originalTestId, testRetryAttempts.get(originalTestId) + 1)
}
output.debug(`HTML Reporter: Retry detected (retriedTest) for ${originalTest.title}, attempts: ${testRetryAttempts.get(originalTestId)}`)
}
// Method 4: Check if test has been seen before (indicating a retry)
else if (reportData.tests.some(t => t.id === testId)) {
testRetryAttempts.set(testId, 1) // First retry detected
output.debug(`HTML Reporter: Retry detected (duplicate test) for ${test.title}, attempts: 1`)
}
}
})
// Collect step information
event.dispatcher.on(event.step.started, step => {
step.htmlReporterStartTime = Date.now()
})
event.dispatcher.on(event.step.finished, step => {
if (step.htmlReporterStartTime) {
step.duration = Date.now() - step.htmlReporterStartTime
}
// Serialize args immediately to preserve them through worker serialization
let serializedArgs = []
if (step.args && Array.isArray(step.args)) {
serializedArgs = step.args.map(arg => {
try {
// Try to convert to JSON-friendly format
if (typeof arg === 'string') return arg
if (typeof arg === 'number') return arg
if (typeof arg === 'boolean') return arg
if (arg === null || arg === undefined) return arg
// For objects, try to serialize them
return JSON.parse(JSON.stringify(arg))
} catch (e) {
// If serialization fails, convert to string
return String(arg)
}
})
}
currentTestSteps.push({
name: step.name,
actor: step.actor,
args: serializedArgs,
status: step.failed ? 'failed' : 'success',
duration: step.duration || 0,
})
})
// Collect hook information
event.dispatcher.on(event.hook.started, hook => {
hook.htmlReporterStartTime = Date.now()
})
event.dispatcher.on(event.hook.finished, hook => {
if (hook.htmlReporterStartTime) {
hook.duration = Date.now() - hook.htmlReporterStartTime
}
// Enhanced hook info: include type, name, location, error, and context
const hookInfo = {
title: hook.title,
type: hook.type || 'unknown', // before, after, beforeSuite, afterSuite
status: hook.err ? 'failed' : 'passed',
duration: hook.duration || 0,
error: hook.err ? hook.err.message || hook.err.toString() : null,
location: hook.file || hook.location || (hook.ctx && hook.ctx.test && hook.ctx.test.file) || null,
context: hook.ctx
? {
testTitle: hook.ctx.test?.title,
suiteTitle: hook.ctx.test?.parent?.title,
feature: hook.ctx.test?.parent?.feature?.name,
}
: null,
}
currentTestHooks.push(hookInfo)
reportData.hooks.push(hookInfo)
})
// Collect BDD/Gherkin step information
event.dispatcher.on(event.bddStep.started, step => {
step.htmlReporterStartTime = Date.now()
})
event.dispatcher.on(event.bddStep.finished, step => {
if (step.htmlReporterStartTime) {
step.duration = Date.now() - step.htmlReporterStartTime
}
currentBddSteps.push({
keyword: step.actor || 'Given',
text: step.name,
status: step.failed ? 'failed' : 'success',
duration: step.duration || 0,
comment: step.comment,
})
})
// Collect skipped tests
event.dispatcher.on(event.test.skipped, test => {
const testId = generateTestId(test)
// Detect if this is a BDD/Gherkin test
const suite = test.parent || test.suite || currentSuite
const isBddTest = isBddGherkinTest(test, suite)
const featureInfo = isBddTest ? getBddFeatureInfo(test, suite) : null
// Extract parent/suite title
const parentTitle = test.parent?.title || test.suite?.title || (suite && suite.title) || null
const suiteTitle = test.suite?.title || (suite && suite.title) || null
const testData = {
...test,
id: testId,
state: 'pending', // Use 'pending' as the state for skipped tests
duration: 0,
steps: [],
hooks: [],
artifacts: [],
tags: test.tags || [],
meta: test.meta || {},
opts: test.opts || {},
notes: test.notes || [],
retryAttempts: 0,
uid: test.uid,
isBdd: isBddTest,
feature: featureInfo,
parentTitle: parentTitle,
suiteTitle: suiteTitle,
}
reportData.tests.push(testData)
output.debug(`HTML Reporter: Added skipped test - ${test.title}`)
})
// Collect test results
event.dispatcher.on(event.test.finished, test => {
const testId = generateTestId(test)
let retryAttempts = testRetryAttempts.get(testId) || 0
// Additional retry detection in test.finished event
// Check if this test has retry indicators we might have missed
if (retryAttempts === 0) {
if (test.retryNum && test.retryNum > 0) {
retryAttempts = test.retryNum
testRetryAttempts.set(testId, retryAttempts)
output.debug(`HTML Reporter: Late retry detection (retryNum) for ${test.title}, attempts: ${retryAttempts}`)
} else if (test.currentRetry && test.currentRetry > 0) {
retryAttempts = test.currentRetry
testRetryAttempts.set(testId, retryAttempts)
output.debug(`HTML Reporter: Late retry detection (currentRetry) for ${test.title}, attempts: ${retryAttempts}`)
} else if (test._retries && test._retries > 0) {
retryAttempts = test._retries
testRetryAttempts.set(testId, retryAttempts)
output.debug(`HTML Reporter: Late retry detection (_retries) for ${test.title}, attempts: ${retryAttempts}`)
}
}
// Debug logging
output.debug(`HTML Reporter: Test finished - ${test.title}, State: ${test.state}, Retries: ${retryAttempts}`)
// Detect if this is a BDD/Gherkin test - use test.parent directly instead of currentSuite
const suite = test.parent || test.suite || currentSuite
const isBddTest = isBddGherkinTest(test, suite)
const steps = isBddTest ? currentBddSteps : currentTestSteps
const featureInfo = isBddTest ? getBddFeatureInfo(test, suite) : null
// Check if this test already exists in reportData.tests (from a previous retry)
const existingTestIndex = reportData.tests.findIndex(t => t.id === testId)
const hasFailedBefore = existingTestIndex >= 0 && reportData.tests[existingTestIndex] && reportData.tests[existingTestIndex].state === 'failed'
const currentlyFailed = test.state === 'failed'
// Debug artifacts collection (but don't process them yet - screenshots may not be ready)
output.debug(`HTML Reporter: Test ${test.title} artifacts at test.finished: ${JSON.stringify(test.artifacts)}`)
// Extract parent/suite title before serialization (for worker mode)
// This ensures the feature name is preserved when test data is JSON stringified
const parentTitle = test.parent?.title || test.suite?.title || (suite && suite.title) || null
const suiteTitle = test.suite?.title || (suite && suite.title) || null
const testData = {
...test,
id: testId,
duration: test.duration || 0,
steps: [...steps], // Copy the steps (BDD or regular)
hooks: [...currentTestHooks], // Copy the hooks
artifacts: test.artifacts || [], // Keep original artifacts for now
tags: test.tags || [],
meta: test.meta || {},
opts: test.opts || {},
notes: test.notes || [],
retryAttempts: currentlyFailed || hasFailedBefore ? retryAttempts : 0, // Only show retries for failed tests
uid: test.uid,
isBdd: isBddTest,
feature: featureInfo,
// Store parent/suite titles as simple strings for worker mode serialization
parentTitle: parentTitle,
suiteTitle: suiteTitle,
}
if (existingTestIndex >= 0) {
// Update existing test with final result (including failed state)
if (existingTestIndex >= 0) reportData.tests[existingTestIndex] = testData
output.debug(`HTML Reporter: Updated existing test - ${test.title}, Final state: ${test.state}`)
} else {
// Add new test
reportData.tests.push(testData)
output.debug(`HTML Reporter: Added new test - ${test.title}, State: ${test.state}`)
}
// Track retry information - only add if there were actual retries AND the test failed at some point
const existingRetryIndex = reportData.retries.findIndex(r => r.testId === testId)
// Only track retries if:
// 1. There are retry attempts detected AND (test failed now OR failed before)
// 2. OR there's an existing retry record (meaning it failed before)
if ((retryAttempts > 0 && (currentlyFailed || hasFailedBefore)) || existingRetryIndex >= 0) {
// If no retry attempts detected but we have an existing retry record, increment it
if (retryAttempts === 0 && existingRetryIndex >= 0) {
retryAttempts = reportData.retries[existingRetryIndex].attempts + 1
testRetryAttempts.set(testId, retryAttempts)
output.debug(`HTML Reporter: Incremented retry count for duplicate test ${test.title}, attempts: ${retryAttempts}`)
}
// Remove existing retry info for this test and add updated one
reportData.retries = reportData.retries.filter(r => r.testId !== testId)
reportData.retries.push({
testId: testId,
testTitle: test.title,
attempts: retryAttempts,
finalState: test.state,
duration: test.duration || 0,
})
output.debug(`HTML Reporter: Added retry info for ${test.title}, attempts: ${retryAttempts}, state: ${test.state}`)
}
// Fallback: If this test already exists and either failed before or is failing now, it's a retry
else if (existingTestIndex >= 0 && (hasFailedBefore || currentlyFailed)) {
const fallbackAttempts = 1
testRetryAttempts.set(testId, fallbackAttempts)
reportData.retries.push({
testId: testId,
testTitle: test.title,
attempts: fallbackAttempts,
finalState: test.state,
duration: test.duration || 0,
})
output.debug(`HTML Reporter: Fallback retry detection for failed test ${test.title}, attempts: ${fallbackAttempts}`)
}
})
// Generate final report
event.dispatcher.on(event.all.result, async result => {
reportData.endTime = new Date().toISOString()
reportData.duration = new Date(reportData.endTime).getTime() - new Date(reportData.startTime).getTime()
// Process artifacts now that all async tasks (including screenshots) are complete
output.debug(`HTML Reporter: Processing artifacts for ${reportData.tests.length} tests after all async tasks complete`)
reportData.tests.forEach(test => {
const originalArtifacts = test.artifacts
let collectedArtifacts = []
output.debug(`HTML Reporter: Processing test "${test.title}" (ID: ${test.id})`)
output.debug(`HTML Reporter: Test ${test.title} final artifacts: ${JSON.stringify(originalArtifacts)}`)
if (originalArtifacts) {
if (Array.isArray(originalArtifacts)) {
collectedArtifacts = originalArtifacts
output.debug(`HTML Reporter: Using array artifacts: ${collectedArtifacts.length} items`)
} else if (typeof originalArtifacts === 'object') {
// Convert object properties to array (screenshotOnFail plugin format)
collectedArtifacts = Object.values(originalArtifacts).filter(artifact => artifact)
output.debug(`HTML Reporter: Converted artifacts object to array: ${collectedArtifacts.length} items`)
output.debug(`HTML Reporter: Converted artifacts: ${JSON.stringify(collectedArtifacts)}`)
}
}
// Only use filesystem fallback if no artifacts found from screenshotOnFail plugin
if (collectedArtifacts.length === 0 && test.state === 'failed') {
output.debug(`HTML Reporter: No artifacts from plugin, trying filesystem for test "${test.title}"`)
collectedArtifacts = collectScreenshotsFromFilesystem(test, test.id)
output.debug(`HTML Reporter: Collected ${collectedArtifacts.length} screenshots from filesystem for failed test "${test.title}"`)
if (collectedArtifacts.length > 0) {
output.debug(`HTML Reporter: Filesystem screenshots for "${test.title}": ${JSON.stringify(collectedArtifacts)}`)
}
}
// Update test with processed artifacts
test.artifacts = collectedArtifacts
output.debug(`HTML Reporter: Final artifacts for "${test.title}": ${JSON.stringify(test.artifacts)}`)
})
// Calculate stats from our collected test data instead of using result.stats
const passedTests = reportData.tests.filter(t => t.state === 'passed').length
const failedTests = reportData.tests.filter(t => t.state === 'failed').length
// Combine pending and skipped tests (both represent tests that were not run)
const pendingTests = reportData.tests.filter(t => t.state === 'pending' || t.state === 'skipped').length
// Calculate flaky tests (passed but had retries)
const flakyTests = reportData.tests.filter(t => t.state === 'passed' && t.retryAttempts > 0).length
// Count total artifacts
const totalArtifacts = reportData.tests.reduce((sum, t) => sum + (t.artifacts?.length || 0), 0)
// Populate failures from our collected test data with enhanced details
reportData.failures = reportData.tests
.filter(t => t.state === 'failed')
.map(t => {
const testName = t.title || 'Unknown Test'
// Try to get feature name from BDD, preserved titles (worker mode), or direct access
let featureName = t.feature?.name || t.parentTitle || t.suiteTitle || t.parent?.title || t.suite?.title || 'Unknown Feature'
if (featureName === 'Unknown Feature' && t.suite && t.suite.feature && t.suite.feature.name) {
featureName = t.suite.feature.name
}
if (t.err) {
const errorMessage = t.err.message || t.err.toString() || 'Test failed'
const errorStack = t.err.stack || ''
const filePath = t.file || t.parent?.file || ''
// Create enhanced failure object with test details
return {
testName: testName,
featureName: featureName,
message: errorMessage,
stack: errorStack,
filePath: filePath,
toString: () => `${testName} (${featureName})\n${errorMessage}\n${errorStack}`.trim(),
}
}
return {
testName: testName,
featureName: featureName,
message: `Test failed: ${testName}`,
stack: '',
filePath: t.file || t.parent?.file || '',
toString: () => `${testName} (${featureName})\nTest failed: ${testName}`,
}
})
reportData.stats = {
tests: reportData.tests.length,
passes: passedTests,
failures: failedTests,
pending: pendingTests,
duration: reportData.duration,
failedHooks: result.stats?.failedHooks || 0,
flaky: flakyTests,
artifacts: totalArtifacts,
}
// Debug logging for final stats
output.debug(`HTML Reporter: Calculated stats - Tests: ${reportData.stats.tests}, Passes: ${reportData.stats.passes}, Failures: ${reportData.stats.failures}`)
output.debug(`HTML Reporter: Collected ${reportData.tests.length} tests in reportData`)
output.debug(`HTML Reporter: Failures array has ${reportData.failures.length} items`)
output.debug(`HTML Reporter: Retries array has ${reportData.retries.length} items`)
output.debug(`HTML Reporter: testRetryAttempts Map size: ${testRetryAttempts.size}`)
// Log retry attempts map contents
for (const [testId, attempts] of testRetryAttempts.entries()) {
output.debug(`HTML Reporter: testRetryAttempts - ${testId}: ${attempts} attempts`)
}
reportData.tests.forEach(test => {
output.debug(`HTML Reporter: Test in reportData - ${test.title}, State: ${test.state}, Retries: ${test.retryAttempts}`)
})
// Check if running with workers
if (process.env.RUNS_WITH_WORKERS) {
// In worker mode, save results to a JSON file for later consolidation
const workerId = threadId
const jsonFileName = `worker-${workerId}-results.json`
const jsonPath = path.join(reportDir, jsonFileName)
try {
// Always overwrite the file with the latest complete data from this worker
// This prevents double-counting when the event is triggered multiple times
fs.writeFileSync(jsonPath, safeJsonStringify(reportData))
output.debug(`HTML Reporter: Generated worker JSON results: ${jsonFileName}`)
} catch (error) {
output.debug(`HTML Reporter: Failed to write worker JSON: ${error.message}`)
}
return
}
// Single process mode - generate report normally
try {
await generateHtmlReport(reportData, options)
} catch (error) {
output.print(`Failed to generate HTML report: ${error.message}`)
output.debug(`HTML Reporter error stack: ${error.stack}`)
}
// Export stats if configured
if (options.exportStats) {
exportTestStats(reportData, options)
}
// Save history if configured
if (options.keepHistory) {
saveTestHistory(reportData, options)
}
})
// Handle worker consolidation after all workers complete
event.dispatcher.on(event.workers.result, async result => {
if (process.env.RUNS_WITH_WORKERS) {
// Only run consolidation in main process
await consolidateWorkerJsonResults(options)
}
})
/**
* Safely serialize data to JSON, handling circular references
*/
function safeJsonStringify(data) {
const seen = new WeakSet()
return JSON.stringify(
data,
(key, value) => {
if (typeof value === 'object' && value !== null) {
if (seen.has(value)) {
// For error objects, try to extract useful information instead of "[Circular Reference]"
if (key === 'err' || key === 'error') {
return {
message: value.message || 'Error occurred',
stack: value.stack || '',
name: value.name || 'Error',
}
}
// Skip circular references for other objects
return undefined
}
seen.add(value)
// Special handling for error objects to preserve important properties
if (value instanceof Error || (value.message && value.stack)) {
return {
message: value.message || '',
stack: value.stack || '',
name: value.name || 'Error',
toString: () => value.message || 'Error occurred',
}
}
}
return value
},
2,
)
}
function generateTestId(test) {
return crypto
.createHash('sha256')
.update(`${test.parent?.title || 'unknown'}_${test.title}`)
.digest('hex')
.substring(0, 8)
}
function collectScreenshotsFromFilesystem(test, testId) {
const screenshots = []
try {
// Common screenshot locations to check
const possibleDirs = [
reportDir, // Same as report directory
global.output_dir || './output', // Global output directory
path.resolve(global.codecept_dir || '.', 'output'), // Codecept output directory
path.resolve('.', 'output'), // Current directory output
path.resolve('.', '_output'), // Alternative output directory
path.resolve('output'), // Relative output directory
path.resolve('qa', 'output'), // QA project output directory
path.resolve('..', 'qa', 'output'), // Parent QA project output directory
]
// Use the exact same logic as screenshotOnFail plugin's testToFileName function
const originalTestName = test.title || 'test'
const originalFeatureName = test.parent?.title || 'feature'
// Replicate testToFileName logic from lib/mocha/test.js
function replicateTestToFileName(testTitle) {
let fileName = testTitle
// Slice to 100 characters first
fileName = fileName.slice(0, 100)
// Handle data-driven tests: remove everything from '{' onwards (with 3 chars before)
if (fileName.indexOf('{') !== -1) {
fileName = fileName.substr(0, fileName.indexOf('{') - 3).trim()
}
// Apply clearString logic from utils.js
if (fileName.endsWith('.')) {
fileName = fileName.slice(0, -1)
}
fileName = fileName
.replace(/ /g, '_')
.replace(/"/g, "'")
.replace(/\//g, '_')
.replace(/</g, '(')
.replace(/>/g, ')')
.replace(/:/g, '_')
.replace(/\\/g, '_')
.replace(/\|/g, '_')
.replace(/\?/g, '.')
.replace(/\*/g, '^')
.replace(/'/g, '')
// Final slice to 100 characters
return fileName.slice(0, 100)
}
const testName = replicateTestToFileName(originalTestName)
const featureName = replicateTestToFileName(originalFeatureName)
output.debug(`HTML Reporter: Original test title: "${originalTestName}"`)
output.debug(`HTML Reporter: CodeceptJS filename: "${testName}"`)
// Generate possible screenshot names based on CodeceptJS patterns
const possibleNames = [
`${testName}.failed.png`, // Primary CodeceptJS screenshotOnFail pattern
`${testName}.failed.jpg`,
`${featureName}_${testName}.failed.png`,
`${featureName}_${testName}.failed.jpg`,
`Test_${testName}.failed.png`, // Alternative pattern
`Test_${testName}.failed.jpg`,
`${testName}.png`,
`${testName}.jpg`,
`${featureName}_${testName}.png`,
`${featureName}_${testName}.jpg`,
`failed_${testName}.png`,
`failed_${testName}.jpg`,
`screenshot_${testId}.png`,
`screenshot_${testId}.jpg`,
'screenshot.png',
'screenshot.jpg',
'failure.png',
'failure.jpg',
]
output.debug(`HTML Reporter: Checking ${possibleNames.length} possible screenshot names for "${testName}"`)
// Search for screenshots in possible directories
for (const dir of possibleDirs) {
output.debug(`HTML Reporter: Checking directory: ${dir}`)
if (!fs.existsSync(dir)) {
output.debug(`HTML Reporter: Directory does not exist: ${dir}`)
continue
}
try {
const files = fs.readdirSync(dir)
output.debug(`HTML Reporter: Found ${files.length} files in ${dir}`)
// Look for exact matches first
for (const name of possibleNames) {
if (files.includes(name)) {
const fullPath = path.join(dir, name)
if (!screenshots.includes(fullPath)) {
screenshots.push(fullPath)
output.debug(`HTML Reporter: Found screenshot: ${fullPath}`)
}
}
}
// Look for screenshot files that are specifically for this test
// Be more strict to avoid cross-test contamination
const screenshotFiles = files.filter(file => {
const lowerFile = file.toLowerCase()
const lowerTestName = testName.toLowerCase()
const lowerFeatureName = featureName.toLowerCase()
return (
file.match(/\.(png|jpg|jpeg|gif|webp|bmp)$/i) &&
// Exact test name matches with .failed pattern (most specific)
(file === `${testName}.failed.png` ||
file === `${testName}.failed.jpg` ||
file === `${featureName}_${testName}.failed.png` ||
file === `${featureName}_${testName}.failed.jpg` ||
file === `Test_${testName}.failed.png` ||
file === `Test_${testName}.failed.jpg` ||
// Word boundary checks for .failed pattern
(lowerFile.includes('.failed.') &&
(lowerFile.startsWith(lowerTestName + '.') || lowerFile.startsWith(lowerFeatureName + '_' + lowerTestName + '.') || lowerFile.startsWith('test_' + lowerTestName + '.'))))
)
})
for (const file of screenshotFiles) {
const fullPath = path.join(dir, file)
if (!screenshots.includes(fullPath)) {
screenshots.push(fullPath)
output.debug(`HTML Reporter: Found related screenshot: ${fullPath}`)
}
}
} catch (error) {
// Ignore directory read errors
output.debug(`HTML Reporter: Could not read directory ${dir}: ${error.message}`)
}
}
} catch (error) {
output.debug(`HTML Reporter: Error collecting screenshots: ${error.message}`)
}
return screenshots
}
function isBddGherkinTest(test, suite) {
// Check if the suite has BDD/Gherkin properties
return !!(suite && (suite.feature || suite.file?.endsWith('.feature')))
}
function getBddFeatureInfo(test, suite) {
if (!suite) return null
return {
name: suite.feature?.name || suite.title,
description: suite.feature?.description || suite.comment || '',
language: suite.feature?.language || 'en',
tags: suite.tags || [],
file: suite.file || '',
}
}
function exportTestStats(data, config) {
const statsPath = path.resolve(reportDir, config.exportStatsPath)
const exportData = {
timestamp: data.endTime, // Already an ISO string
duration: data.duration,
stats: data.stats,
retries: data.retries,
testCount: data.tests.length,
passedTests: data.tests.filter(t => t.state === 'passed').length,
failedTests: data.tests.filter(t => t.state === 'failed').length,
pendingTests: data.tests.filter(t => t.state === 'pending').length,
tests: data.tests.map(test => ({
id: test.id,
title: test.title,
feature: test.parent?.title || 'Unknown',
state: test.state,
duration: test.duration,
tags: test.tags,
meta: test.meta,
retryAttempts: test.retryAttempts,
uid: test.uid,
})),
}
try {
fs.writeFileSync(statsPath, JSON.stringify(exportData, null, 2))
output.print(`Test stats exported to: ${statsPath}`)
} catch (error) {
output.print(`Failed to export test stats: ${error.message}`)
}
}
function saveTestHistory(data, config) {
const historyPath = path.resolve(reportDir, config.historyPath)
let history = []
// Load existing history
try {
if (fs.existsSync(historyPath)) {
history = JSON.parse(fs.readFileSync(historyPath, 'utf8'))
}
} catch (error) {
output.print(`Failed to load existing history: ${error.message}`)
}
// Add current run to history
history.unshift({
timestamp: data.endTime, // Already an ISO string
duration: data.duration,
stats: data.stats,
retries: data.retries.length,
testCount: data.tests.length,
})
// Limit history entries
if (history.length > config.maxHistoryEntries) {
history = history.slice(0, config.maxHistoryEntries)
}
try {
fs.writeFileSync(historyPath, JSON.stringify(history, null, 2))
output.print(`Test history saved to: ${historyPath}`)
} catch (error) {
output.print(`Failed to save test history: ${error.message}`)
}
}
/**
* Consolidates JSON reports from multiple workers into a single HTML report
*/
async function consolidateWorkerJsonResults(config) {
const jsonFiles = fs.readdirSync(reportDir).filter(file => file.startsWith('worker-') && file.endsWith('-results.json'))
if (jsonFiles.length === 0) {
output.debug('HTML Reporter: No worker JSON results found to consolidate')
return
}
output.debug(`HTML Reporter: Found ${jsonFiles.length} worker JSON files to consolidate`)
// Initialize consolidated data structure
const consolidatedData = {
stats: {
tests: 0,
passes: 0,
failures: 0,
pending: 0,
skipped: 0,
duration: 0,
failedHooks: 0,
},
tests: [],
failures: [],
hooks: [],
startTime: new Date(),
endTime: new Date(),
retries: [],
duration: 0,
}
try {
// Process each worker's JSON file
for (const jsonFile of jsonFiles) {
const jsonPath = path.join(reportDir, jsonFile)
try {
const workerData = JSON.parse(fs.readFileSync(jsonPath, 'utf8'))
// Extract worker ID from filename (e.g., "worker-0-results.json" -> 0)
const workerIdMatch = jsonFile.match(/worker-(\d+)-results\.json/)
const workerIndex = workerIdMatch ? parseInt(workerIdMatch[1], 10) : undefined
// Merge stats
if (workerData.stats) {
consolidatedData.stats.passes += workerData.stats.passes || 0
consolidatedData.stats.failures += workerData.stats.failures || 0
consolidatedData.stats.tests += workerData.stats.tests || 0
consolidatedData.stats.pending += workerData.stats.pending || 0
consolidatedData.stats.skipped += workerData.stats.skipped || 0
consolidatedData.stats.duration += workerData.stats.duration || 0
consolidatedData.stats.failedHooks += workerData.stats.failedHooks || 0
}
// Merge tests and add worker index
if (workerData.tests) {
const testsWithWorkerIndex = workerData.tests.map(test => ({
...test,
workerIndex: workerIndex,
}))
consolidatedData.tests.push(...testsWithWorkerIndex)
}
if (workerData.failures) consolidatedData.failures.push(...workerData.failures)
if (workerData.hooks) consolidatedData.hooks.push(...workerData.hooks)
if (workerData.retries) consolidatedData.retries.push(...workerData.retries)
// Update timestamps
if (workerData.startTime) {
const workerStart = new Date(workerData.startTime).getTime()
const currentStart = new Date(consolidatedData.startTime).getTime()
if (workerStart < currentStart) {
consolidatedData.startTime = workerData.startTime
}
}
if (workerData.endTime) {
const workerEnd = new Date(workerData.endTime).getTime()
const currentEnd = new Date(consolidatedData.endTime).getTime()
if (workerEnd > currentEnd) {
consolidatedData.endTime = workerData.endTime
}
}
// Update duration
if (workerData.duration) {
consolidatedData.duration = Math.max(consolidatedData.duration, workerData.duration)
}
// Clean up the worker JSON file
try {
fs.unlinkSync(jsonPath)
} catch (error) {
output.print(`Failed to delete worker JSON file ${jsonFile}: ${error.message}`)
}
} catch (error) {
output.print(`Failed to process worker JSON file ${jsonFile}: ${error.message}`)
}
}
// Generate the final HTML report
generateHtmlReport(consolidatedData, config)
// Export stats if configured
if (config.exportStats) {
exportTestStats(consolidatedData, config)
}
// Save history if configured
if (config.keepHistory) {
saveTestHistory(consolidatedData, config)
}
output.debug(`HTML Reporter: Successfully consolidated ${jsonFiles.length} worker reports`)
} catch (error) {
output.debug(`HTML Reporter: Failed to consolidate worker reports: ${error.message}`)
}
}
async function generateHtmlReport(data, config) {
const reportPath = path.join(reportDir, config.reportFileName)
// Load history if available
let history = []
if (config.keepHistory) {
const historyPath = path.resolve(reportDir, config.historyPath)
try {
if (fs.existsSync(historyPath)) {
history = JSON.parse(fs.readFileSync(historyPath, 'utf8')) // Show all available history
}
} catch (error) {
output.print(`Failed to load history for report: ${error.message}`)
}
// Add current run to history for chart display (before saving to file)
const currentRun = {
timestamp: data.endTime, // Already an ISO string
duration: data.duration,
stats: data.stats,
retries: data.retries.length,
testCount: data.tests.length,
}
history.unshift(currentRun)
// Limit history entries for chart display
if (history.length > config.maxHistoryEntries) {
history = history.slice(0, config.maxHistoryEntries)
}
}
// Get system information
const systemInfo = await getMachineInfo()
const html = template(getHtmlTemplate(), {
title: `CodeceptJS Test Report v${Codecept.version()}`,
timestamp: data.endTime, // Already an ISO string
duration: formatDuration(data.duration),
stats: JSON.stringify(data.stats),
history: JSON.stringify(history),
statsHtml: generateStatsHtml(data.stats),
testsHtml: generateTestsHtml(data.tests, config),
retriesHtml: config.showRetries ? generateRetriesHtml(data.retries) : '',
cssStyles: getCssStyles(),
jsScripts: getJsScripts(),
showRetries: config.showRetries ? 'block' : 'none',
showHistory: config.keepHistory && history.length > 0 ? 'block' : 'none',
codeceptVersion: Codecept.version(),
systemInfoHtml: generateSystemInfoHtml(systemInfo),
})
fs.writeFileSync(reportPath, html)
output.print(`HTML Report saved to: ${reportPath}`)
}
function generateStatsHtml(stats) {
const passed = stats.passes || 0
const failed = stats.failures || 0
const pending = stats.pending || 0
const total = stats.tests || 0
const flaky = stats.flaky || 0
const artifactCount = stats.artifacts || 0
const passRate = total > 0 ? ((passed / total) * 100).toFixed(1) : '0.0'
const failRate = total > 0 ? ((failed / total) * 100).toFixed(1) : '0.0'
return `
<div class="stats-cards">
<div class="stat-card total">
<h3>Total</h3>
<span class="stat-number">${total}</span>
</div>
<div class="stat-card passed">
<h3>Passed</h3>
<span class="stat-number">${passed}</span>
</div>
<div class="stat-card failed">
<h3>Failed</h3>
<span class="stat-number">${failed}</span>
</div>
<div class="stat-card pending">
<h3>Skipped</h3>
<span class="stat-number">${pending}</span>
</div>
<div class="stat-card flaky">
<h3>Flaky</h3>
<span class="stat-number">${flaky}</span>
</div>
<div class="stat-card artifacts">
<h3>Artifacts</h3>
<span class="stat-number">${artifactCount}</span>
</div>
</div>
<div class="metrics-summary">
<span>Pass Rate: <strong>${passRate}%</strong></span>
<span>Fail Rate: <strong>${failRate}%</strong></span>
</div>
<div class="pie-chart-container">
<canvas id="statsChart" width="300" height="300"></canvas>
<script>
// Pie chart data will be rendered by JavaScript
window.chartData = {
passed: ${passed},
failed: ${failed},
pending: ${pending}
};
</script>
</div>
`
}
function generateTestsHtml(tests, config) {
if (!tests || tests.length === 0) {
return '<p>No tests found.</p>'
}
// Group tests by feature name
const grouped = {}
tests.forEach(test => {
const feature = test.isBdd && test.feature ? test.feature.name : test.parentTitle || test.suiteTitle || test.parent?.title || test.suite?.title || 'Unknown Feature'
if (!grouped[feature]) grouped[feature] = []
grouped[feature].push(test)
})
// Render each feature section
return Object.entries(grouped)
.map(([feature, tests]) => {
const featureId = feature.replace(/[^a-zA-Z0-9]/g, '-').toLowerCase()
return `
<section class="feature-group">
<h3 class="feature-group-title" onclick="toggleFeatureGroup('${featureId}')">
${escapeHtml(feature)}
<span class="toggle-icon">▼</span>
</h3>
<div class="feature-tests" id="feature-${featureId}">
${tests
.map(test => {
const statusClass = test.state || 'unknown'
const steps = config.showSteps && test.steps ? (test.isBdd ? generateBddStepsHtml(test.steps) : generateStepsHtml(test.steps)) : ''
const featureDetails = test.isBdd && test.feature ? generateBddFeatureHtml(test.feature) : ''
const hooks = test.hooks && test.hooks.length > 0 ? generateHooksHtml(test.hooks) : ''
const artifacts = config.includeArtifacts && test.artifacts ? generateArtifactsHtml(test.artifacts, test.state === 'failed') : ''
const metadata = config.showMetadata && (test.meta || test.opts) ? generateMetadataHtml(test.meta, test.opts) : ''
const tags = config.showTags && test.tags && test.tags.length > 0 ? generateTagsHtml(test.tags) : ''
const retries = config.showRetries && test.retryAttempts > 0 ? generateTestRetryHtml(test.retryAttempts, test.state) : ''
const notes = test.notes && test.notes.length > 0 ? generateNotesHtml(test.notes) : ''
// Worker badge - show worker index if test has worker info
const workerBadge = test.workerIndex !== undefined ? `<span class="worker-badge worker-${test.workerIndex}">Worker ${test.workerIndex}</span>` : ''
return `
<div class="test-item ${statusClass}${test.isBdd ? ' bdd-test' : ''}" id="test-${test.id}" data-feature="${escapeHtml(feature)}" data-status="${statusClass}" data-tags="${(test.tags || []).join(',')}" data-retries="${test.retryAttempts || 0}" data-type="${test.isBdd ? 'bdd' : 'regular'}">
<div class="test-header" onclick="toggleTestDetails('test-${test.id}')">
<span class="test-status ${statusClass}">●</span>
<div class="test-info">
<h3 class="test-title">${test.isBdd ? `Scenario: ${test.title}` : test.title}</h3>
<div class="test-meta-line">
${workerBadge}
${test.uid ? `<span class="test-uid">${test.uid}</span>` : ''}
<span class="test-duration">${formatDuration(test.duration)}</span>
${test.retryAttempts > 0 ? `<span class="retry-badge">${test.retryAttempts} retries</span>` : ''}
${test.isBdd ? '<span class="bdd-badge">Gherkin</span>' : ''}
</div>
</div>
</div>
<div class="test-details" id="details-test-${test.id}">
${test.err ? `<div class="error-message"><pre>${escapeHtml(getErrorMessage(test))}</pre></div>` : ''}
${featureDetails}
${tags}
${metadata}
${retries}
${notes}
${hooks}
${steps}
${artifacts}
</div>
</div>
`
})
.join('')}
</div>
</section>
`
})
.join('')
}
function generateStepsHtml(steps) {
if (!steps || steps.length === 0) return ''
const stepsHtml = steps
.map(step => {
const statusClass = step.status || 'unknown'
const args = step.args ? step.args.map(arg => JSON.stringify(arg)).join(', ') : ''
const stepName = step.name || 'unknown step'
const actor = step.actor || 'I'
return `
<div class="step-item ${statusClass}">
<span class="step-status ${statusClass}">●</span>
<span class="step-title">${actor}.${stepName}(${args})</span>
<span class="step-duration">${formatDuration(step.duration)}</span>
</div>
`
})
.join('')
return `
<div class="steps-section">
<h4>Steps:</h4>
<div class="steps-list">${stepsHtml}</div>
</div>
`
}
function generateBddStepsHtml(steps) {
if (!steps || steps.length === 0) return ''
const stepsHtml = steps
.map(step => {
const statusClass = step.status || 'unknown'
const keyword = step.keyword || 'Given'
const text = step.text || ''
const comment = step.comment ? `<div class="step-comment">${escapeHtml(step.comment)}</div>` : ''
return `
<div class="bdd-step-item ${statusClass}">
<span class="step-status ${statusClass}">●</span>
<span class="bdd-keyword">${keyword}</span>
<span class="bdd-step-text">${escapeHtml(text)}</span>
<span class="step-duration">${formatDuration(step.duration)}</span>
${comment}
</div>
`
})
.join('')
return `
<div class="bdd-steps-section">
<h4>Scenario Steps:</h4>
<div class="bdd-steps-list">${stepsHtml}</div>
</div>
`
}
function generateBddFeatureHtml(feature) {
if (!feature) return ''
const description = feature.description ? `<div class="feature-description">${escapeHtml(feature.description)}</div>` : ''
const featureTags = feature.tags && feature.tags.length > 0 ? `<div class="feature-tags">${feature.tags.map(tag => `<span class="feature-tag">${escapeHtml(tag)}</span>`).join('')}</div>` : ''
return `
<div class="bdd-feature-section">
<h4>Feature Information:</h4>
<div class="feature-info">
<div class="feature-name">Feature: ${escapeHtml(feature.name)}</div>
${description}
${featureTags}
${feature.file ? `<div class="feature-file">File: ${escapeHtml(feature.file)}</div>` : ''}
</div>
</div>
`
}
function generateHooksHtml(hooks) {
if (!hooks || hooks.length === 0) return ''
const hooksHtml = hooks
.map(hook => {
const statusClass = hook.status || 'unknown'
const hookType = hook.type || 'hook'
const hookTitle = hook.title || `${hookType} hook`
const location = hook.location ? `<div class="hook-location">Location: ${escapeHtml(hook.location)}</div>` : ''
const context = hook.context ? `<div class="hook-context">Test: ${escapeHtml(hook.context.testTitle || 'N/A')}, Suite: ${escapeHtml(hook.context.suiteTitle || 'N/A')}</div>` : ''
return `
<div class="hook-item ${statusClass}">
<span class="hook-status ${statusClass}">●</span>
<div class="hook-content">
<span class="hook-title">${hookType}: ${hookTitle}</span>
<span class="hook-duration">${formatDuration(hook.duration)}</span>
${location}
${context}
${hook.error ? `<div class="hook-error">${escapeHtml(hook.error)}</div>` : ''}
</div>
</div>
`
})
.join('')
return `
<div class="hooks-section">
<h4>Hooks:</h4>
<div class="hooks-list">${hooksHtml}</div>
</div>
`
}
function generateMetadataHtml(meta, opts) {
const allMeta = { ...(opts || {}), ...(meta || {}) }
if (!allMeta || Object.keys(allMeta).length === 0) return ''
const metaHtml = Object.entries(allMeta)
.filter(([key, value]) => value !== undefined && value !== null)
.map(([key, value]) => {
const displayValue = typeof value === 'object' ? JSON.stringify(value) : value.toString()
return `<div class="meta-item"><span class="meta-key">${escapeHtml(key)}:</span> <span class="meta-value">${escapeHtml(displayValue)}</span></div>`
})
.join('')
return `
<div class="metadata-section">
<h4>Metadata:</h4>
<div class="metadata-list">${metaHtml}</div>
</div>
`
}
function generateTagsHtml(tags) {
if (!tags || tags.length === 0) return ''
const tagsHtml = tags.map(tag => `<span class="test-tag">${escapeHtml(tag)}</span>`).join('')
return `
<div class="tags-section">
<h4>Tags:</h4>
<div class="tags-list">${tagsHtml}</div>
</div>
`
}
function generateNotesHtml(notes) {
if (!notes || notes.length === 0) return ''
const notesHtml = notes.map(note => `<div class="note-item note-${note.type || 'info'}"><span class="note-type">${note.type || 'info'}:</span> <span class="note-text">${escapeHtml(note.text)}</span></div>`).join('')
return `
<div class="notes-section">
<h4>Notes:</h4>
<div class="notes-list">${notesHtml}</div>
</div>
`
}
function generateTestRetryHtml(retryAttempts, testState) {
// Enhanced retry history display showing whether test eventually passed or failed
const statusBadge = testState === 'passed' ? '<span class="retry-status-badge passed">✓ Eventually Passed</span>' : '<span class="retry-status-badge failed">✗ Eventually Failed</span>'
return `
<div class="retry-section">
<h4>Retry History:</h4>
<div class="retry-info">
<div class="retry-summary">
<span class="retry-count">Total retry attempts: <strong>${r