UNPKG

wdio-workflo-allure-reporter

Version:

This is a customized version of wdio-workflo-allure-reporter for use with workflo framework.

839 lines (677 loc) 29.2 kB
import events from 'events' import Allure from 'allure-js-commons-workflo' import Step from 'allure-js-commons-workflo/beans/step' import * as fs from 'fs' import * as path from 'path' import * as _ from 'lodash' function isEmpty (object) { return !object || Object.keys(object).length === 0 } const LOGGING_HOOKS = ['"before all" hook', '"after all" hook'] let debug = false let debugSeleniumCommand = false function logger () { if (debug) { console.log(arguments) } } function error () { console.error(arguments) } /** * Initialize a new `Allure` test reporter. * * @param {Runner} runner * @api public */ class AllureReporter extends events.EventEmitter { constructor (baseReporter, config, options = {}) { super() this.baseReporter = baseReporter this.config = config this.options = options if (options) { debug = options.debug debugSeleniumCommand = options.debugSeleniumCommand } this.allures = {} this.currentTestId = '' this.startedSpecs = false this.testIds = {} this.errorCtr = 0 this.stepLevel = 0 this.retryMessage = undefined this.startStep = function (cid, message) { const allure = this.getAllure(cid) allure.startStep(message) this.stepLevel++ } this.endStep = function (cid, status) { const allure = this.getAllure(cid) allure.endStep(status) this.stepLevel-- } // const { epilogue } = this.baseReporter this.dumpFailuresAndErrors = function (allure, _assertions) { const assertions = {} // ensure that the same assertions are not being logged multiple times // assertions' equality is defined by equal stacks and matcherNames _assertions.forEach(assertion => { const _assertion = _.cloneDeep(assertion) if (_assertion.stack.trim().indexOf('at') === 0) { _assertion.stack = _assertion.message + '\n' + _assertion.stack } delete _assertion.specObj delete _assertion.errorType delete _assertion.passed delete _assertion.actual delete _assertion.expected assertions[_assertion.stack] = _assertion }) var failures = Object.values(assertions).filter(function (assertion) { return typeof assertion.matcherName !== 'undefined' }) var errors = Object.values(assertions).filter(function (assertion) { return typeof assertion.matcherName === 'undefined' }) if (failures.length > 0) { this.dumpJSON(allure, 'Validation Failures', failures) } if (errors.length > 0) { this.dumpJSON(allure, 'Errors', errors) } } this.dumpScreenshots = function (allure, _screenshots) { for (var screenshotMessage in _screenshots) { const screenshots = {} for (var i = 0; i < _screenshots[screenshotMessage].length; ++i) { if (_screenshots[screenshotMessage] && _screenshots[screenshotMessage][i]) { screenshots[_screenshots[screenshotMessage][i].id] = _screenshots[screenshotMessage][i] } } Object.values(screenshots).forEach(screenshot => { allure.addAttachment('Screenshot_' + screenshot.id + ': \'' + screenshotMessage + '\'', new Buffer(fs.readFileSync(screenshot.filename, { encoding: 'base64' }), 'base64')) }) } } this.on('end', () => { // epilogue.call(baseReporter) if (this.startedSpecs) { this.addResults() } // add allure properties file this.addAllurePropertiesFile() }) this.on('startSpecs', (runner) => { if (!this.startedSpecs) { this.startedSpecs = true } }) this.on('suite:start', (suite) => { const allure = this.getAllure(suite.cid) const currentSuite = allure.getCurrentSuite() const prefix = currentSuite ? currentSuite.name + ' ' : '' process.workflo.currentCid = suite.cid allure.startSuite(prefix + suite.title) }) this.on('suite:end', (suite) => { this.getAllure(suite.cid).endSuite() }) this.on('test:setCurrentId', (test) => { const allure = this.getAllure(test.cid) this.currentTestId = test.id const currentTest = allure.getCurrentTest() if (this.currentTestId) { if (test.testcase) { const traceInfo = this.config.traceInfo.testcases[this.currentTestId] currentTest.addParameter('argument', 'Testcase File', traceInfo.testcaseFile) currentTest.addParameter('argument', 'Validates Specs', traceInfo.specs.join(', ')) } else if (test.spec) { const parts = this.currentTestId.split('|') const spec = parts[0] const criteria = parts[1] const traceInfo = this.config.traceInfo.specs[spec] currentTest.addParameter('argument', 'Spec File', traceInfo.specFile) const manualFile = traceInfo.criteriaValidationFiles[criteria].manualFile const testcases = traceInfo.criteriaValidationFiles[criteria].testcases if (manualFile) { currentTest.addParameter('argument', 'Validated by Manual Results', manualFile) } else if (testcases.length > 0) { currentTest.addParameter('argument', 'Validated in Testcases', testcases.join(', ')) } } } }) this.on('test:start', (test) => { const allure = this.getAllure(test.cid) allure.startCase(test.title) this.lastCid = test.cid this.stepLevel = 0 const currentTest = allure.getCurrentTest() if (test.metadata) { for (const key in test.metadata) { const meta = { event: 'test:meta', cid: test.cid } meta[key] = test.metadata[key] this.handleMetadata(meta) } } // add spec ids let printObject = this.config.printObject const filterFilesStr = `${printObject['Spec Files']} specFiles, ${printObject['Testcase Files']} testcaseFiles, ${printObject['Manual Result Files']} manualResultFiles` const filtersStr = `${printObject['Features']} features, ${printObject['Specs']} specs, ${printObject['Suites']} suites, ${printObject['Testcases']} testcases` const coverageStr = `${printObject['Automated Criteria'].count} automated ${printObject['Automated Criteria'].percentage}, ${printObject['Manual Criteria'].count} manual ${printObject['Manual Criteria'].percentage}, ${printObject['Uncovered Criteria'].count} uncovered ${printObject['Uncovered Criteria'].percentage}` let uncoveredCriteriaStr = '' for (const spec in printObject['Uncovered Criteria Object']) { uncoveredCriteriaStr += `${spec}: [${printObject['Uncovered Criteria Object'][spec].join(', ')}], ` } uncoveredCriteriaStr = uncoveredCriteriaStr.substring(0, uncoveredCriteriaStr.length - 2) // these will be added on dashboard page currentTest.addParameter('environment-variable', 'BROWSER', test.runner[test.cid].browserName) currentTest.addParameter('environment-variable', 'BASE URL', this.config.baseUrl) if (this.config.manualOnly) { currentTest.addParameter('environment-variable', 'EXECUTION MODE: ', 'Manual Only') } else if (this.config.automaticOnly) { currentTest.addParameter('environment-variable', 'EXECUTION MODE: ', 'Automatic Only') } else { currentTest.addParameter('environment-variable', 'EXECUTION MODE: ', 'Automatic and Manual') } // currentTest.addParameter('environment-variable', 'SELENIUM CAPABILITIES', JSON.stringify(test.runner[test.cid])) // currentTest.addParameter('environment-variable', 'WEBDRIVER HOST', this.config.host) // currentTest.addParameter('environment-variable', 'WEBDRIVER PORT', this.config.port) currentTest.addParameter('environment-variable', 'CRITERIA COVERAGE: ', coverageStr) currentTest.addParameter('environment-variable', 'FILTER FILES: ', filterFilesStr) currentTest.addParameter('environment-variable', 'FILTERS: ', filtersStr) if (uncoveredCriteriaStr) { currentTest.addParameter('environment-variable', 'UNCOVERED CRITERIA: ', uncoveredCriteriaStr) } }) this.on('test:pass', (test) => { const allure = this.getAllure(test.cid) this.addArguments(test) this.addResultsDummy(test) allure.endCase('passed') }) this.on('test:fail', (test) => { const allure = this.getAllure(test.cid) if (!allure.getCurrentTest()) { allure.startCase(test.title) } else { allure.getCurrentTest().name = test.title } while (allure.getCurrentSuite().currentStep instanceof Step) { this.endStep(test.cid, 'failed') } this.addArguments(test) this.addResultsDummy(test) allure.endCase('failed', { message: '\n' + test.err.message, stack: '\n' + test.err.stack }) }) this.on('test:broken', (test) => { const allure = this.getAllure(test.cid) if (!allure.getCurrentTest()) { allure.startCase(test.title) } else { allure.getCurrentTest().name = test.title } while (allure.getCurrentSuite().currentStep instanceof Step) { this.endStep(test.cid, 'broken') } this.addArguments(test) this.addResultsDummy(test) allure.endCase('broken', { message: '\n' + test.err.message, stack: '\n' + test.err.stack }) }) this.on('test:pending', (test) => { const allure = this.getAllure(test.cid) this.addArguments(test) this.addResultsDummy(test) allure.endCase('pending', {message: 'Test ignored', stack: ''}) }) this.on('test:unvalidated', (test) => { const allure = this.getAllure(test.cid) if (!allure.getCurrentTest()) { allure.startCase(test.title) } else { allure.getCurrentTest().name = test.title } this.addArguments(test) this.addResultsDummy(test) allure.endCase('unknown', { message: '\n' + test.err.message }) }) this.on('runner:command', (command) => { const allure = this.getAllure(command.cid) if (!this.isAnyTestRunning(allure)) { return } if (debugSeleniumCommand) { this.startStep(command.cid, `${command.method} ${command.uri.path}`) if (!isEmpty(command.data)) { this.dumpJSON(allure, 'Request', command.data) } } }) this.on('runner:result', (command) => { const allure = this.getAllure(command.cid) let status = 'passed' if (!this.isAnyTestRunning(allure)) { return } if (debugSeleniumCommand) { if (command.body.screenshot) { allure.addAttachment('Screenshot', new Buffer(command.body.screenshot, 'base64')) delete command.body.screenshot } if (command.body.type && command.body.type === 'RuntimeError') { status = 'broken' } this.dumpJSON(allure, 'Response', command.body) this.endStep(command.cid, status) } }) /* meta : { (mandatory) cid : ..., (mandatory) event : 'test:meta' (optional) description : string (optional) feature : string | array (optional) story : string | array (optional) issue : string | array // bugs??? (optional) severity : [ 'blocker','critical','normal','minor','trivial' ] (optional) argument : {name: value, name2: value } (optional) environment : {name: value, name2: value } } */ this.on('test:meta', function (meta) { this.handleMetadata(meta) }) this.on('step:start', function (step) { const allure = this.getAllure(step.cid) logger('step:start', step) if (!this.isAnyTestRunning(allure)) { error('ERROR', 'cannot start step because no test is running', step) return } this.startStep(step.cid, step.title != null ? step.title : 'No name defined') if (step.arg) { this.dumpJSON(allure, 'Step Args', step.arg) } }) this.on('step:end', function (step) { const allure = this.getAllure(step.cid) logger('step:end', step) if (!this.isAnyTestRunning(allure)) { error('ERROR', 'cannot end step because no test is running', step) return } if (step.arg) { this.dumpJSON(allure, 'Step Result', step.arg) } if (step.assertionFailures) { this.dumpFailuresAndErrors(allure, step.assertionFailures) } if (step.screenshots) { this.dumpScreenshots(allure, step.screenshots) } this.endStep(step.cid, step.status) }) this.on('step:pass', function (step) { logger('step:pass', step) this.endStep(step.cid, 'passed') }) this.on('step:fail', function (step) { const allure = this.getAllure(step.cid) logger('step:fail', step) this.dumpFailuresAndErrors(allure, step.errs) if (step.screenshots) { this.dumpScreenshots(allure, step.screenshots) } this.endStep(step.cid, 'failed') }) this.on('step:broken', function (step) { const allure = this.getAllure(step.cid) logger('step:broken', step) if (step.assertion) { this.dumpFailuresAndErrors(allure, [step.assertion]) if (step.assertion.screenshotFilename) { allure.addAttachment('Screenshot_' + step.assertion.screenshotId + ': \'' + step.assertion.message + '\'', new Buffer(fs.readFileSync(step.assertion.screenshotFilename, { encoding: 'base64' }), 'base64')) } } else { this.dumpFailuresAndErrors(allure, step.errs) if (step.screenshots) { this.dumpScreenshots(allure, step.screenshots) } } this.endStep(step.cid, 'broken') }) this.on('step:unvalidated', function (step) { const allure = this.getAllure(step.cid) logger('step:unvalidated', step) this.dumpJSON(allure, 'unvalidated Specifications', step.err) this.endStep(step.cid, 'unknown') }) this.on('retry:failed', function (step) { const allure = this.getAllure(step.cid) this.dumpFailuresAndErrors(allure, [this.retryMessage.assertion]) if (this.retryMessage.assertion.screenshotFilename) { allure.addAttachment( 'Screenshot_' + this.retryMessage.assertion.screenshotId + ': \'' + this.retryMessage.assertion.message + '\'', new Buffer(fs.readFileSync(this.retryMessage.assertion.screenshotFilename, { encoding: 'base64' }), 'base64') ) } while (this.stepLevel > 0) { this.endStep(step.cid, 'failed') } this.startStep(step.cid, `RETRY ${step.retry}`) this.endStep(step.cid, 'skipped') }) this.on('retry:broken', function (step) { const allure = this.getAllure(step.cid) this.dumpFailuresAndErrors(allure, [step.assertion]) if (step.assertion.screenshotFilename) { allure.addAttachment('Screenshot_' + step.assertion.screenshotId + ': \'' + step.assertion.message + '\'', new Buffer(fs.readFileSync(step.assertion.screenshotFilename, { encoding: 'base64' }), 'base64')) } while (this.stepLevel > 0) { this.endStep(step.cid, 'broken') } this.startStep(step.cid, `RETRY ${step.retry}`) this.endStep(step.cid, 'skipped') }) this.on('retry:validateFailure', function (message) { this.retryMessage = message }) /* attachment : { cid: ..., event: 'test:attach', title: string, file: string, type: string } */ this.on('test:attach', function (attachment) { const allure = this.getAllure(attachment.cid) logger('test:attach', attachment) if (this.isAnyTestRunning(allure)) { error('ERROR', 'cannot attach because no test is running', attachment) return } allure.addAttachment( attachment.title, fs.readFileSync(attachment.file), attachment.type ) }) /* log : { cid: ..., event: 'test:log', message: string, detail: object } */ this.on('test:log', function (log) { const allure = this.getAllure(log.cid) logger('test:log', log) if (this.isAnyTestRunning(allure)) { error('ERROR', 'cannot log because no test is running', log) return } const content = log.detail != null ? JSON.stringify(log.detail, null, ' ') : '' allure.addAttachment( log.message, content, 'application/json' ) }) this.on('runner:screenshot', function (command) { // const allure = this.getAllure(command.cid) }) this.on('hook:start', (hook) => { const allure = this.getAllure(hook.cid) if (!allure.getCurrentSuite() || LOGGING_HOOKS.indexOf(hook.title) === -1) { return } allure.startCase(hook.title) }) this.on('hook:end', (hook) => { const allure = this.getAllure(hook.cid) if (!allure.getCurrentSuite() || LOGGING_HOOKS.indexOf(hook.title) === -1) { return } allure.endCase('passed') if (allure.getCurrentTest().steps.length === 0) { allure.getCurrentSuite().testcases.pop() } }) } handleMetadata (meta) { var allure = this.getAllure(meta.cid) logger('test:meta', meta) if (!this.isAnyTestRunning(allure)) { error('ERROR', 'test:meta : NO TEST RUNNING') return } const currentTest = allure.getCurrentTest() // manage description if (meta.description) { currentTest.setDescription(meta.description) } if (meta.epic) { if (typeof (meta.epic) === 'string') { currentTest.addLabel('epic', meta.epic) } else { for (const i in meta.epic) { currentTest.addLabel('epic', meta.epic[i]) } } } // manage labels ( feature, story, issue ) if (meta.feature) { if (typeof (meta.feature) === 'string') { currentTest.addLabel('feature', meta.feature) } else { for (const i in meta.feature) { currentTest.addLabel('feature', meta.feature[i]) } } } if (meta.story) { if (typeof (meta.story) === 'string') { currentTest.addLabel('story', meta.story) } else { for (const i in meta.story) { currentTest.addLabel('story', meta.story[i]) } } } if (meta.issue) { if (typeof (meta.issue) === 'string') { currentTest.addLabel('issue', this.getIssue(meta.issue)) } else { for (const issue of meta.issue) { // string array, previously "in" currentTest.addLabel('issue', this.getIssue(issue)) } } } if (meta.bug) { if (typeof (meta.bug) === 'string') { currentTest.addLabel('issue', this.getBug(meta.bug)) // check if 'bug' can be used instead! } else { for (const bug of meta.bug) { currentTest.addLabel('issue', this.getBug(bug)) } } } if (meta.testId) { if (typeof (meta.testId) === 'string') { currentTest.addLabel('testId', meta.testId) // check if 'bug' can be used instead! } } if (meta.severity) { if (typeof (meta.severity) === 'string') { currentTest.addLabel('severity', meta.severity) } else { error('ERROR : meta.severity should be a string', meta) } } // manage parameters if (meta.argument) { if (typeof (meta.argument) === 'object') { const keys = Object.keys(meta.argument) for (const i in keys) { const key = keys[i] const val = meta.argument[key] currentTest.addParameter('argument', key, val) } } else { error('ERROR : meta.argument should be an object { name2: val1, name2: val2.. }', meta) } } if (meta.environment) { if (typeof (meta.environment) === 'object') { const keys = Object.keys(meta.environment) for (const i in keys) { const key = keys[i] const val = meta.environment[key] currentTest.addParameter('environment-variable', key, val) } } else { error('ERROR : meta.environment should be an object { name2: val1, name2: val2.. }', meta) } } } getBug (bug) { if (this.config.allure && this.config.allure.bugAppendix) { bug = `${bug}${this.config.allure.bugAppendix}` } if (this.config.allure && this.config.allure.bugPrefix) { bug = `${this.config.allure.bugPrefix}${bug}` } return bug } getIssue (issue) { if (this.config.allure && this.config.allure.issueAppendix) { issue = `${issue}${this.config.allure.issueAppendix}` } if (this.config.allure && this.config.allure.issuePrefix) { issue = `${this.config.allure.issuePrefix}${issue}` } return issue } // add test.arguments to current test addArguments (test) { if (test.arguments) { for (const key in test.arguments) { if (test.arguments[key].value) { this.getAllure(test.cid).getCurrentTest().addParameter('argument', test.arguments[key].caption, test.arguments[key].value) } } } } buildResultStr (results) { let str = '' str += `${results.passing.count} passing (~${Math.round(results.passing.percentage)}%)` if (results.skipped.count > 0) { str += `, ${results.skipped.count} skipped (~${Math.round(results.skipped.percentage)}%)` } if (results.unvalidated.count > 0) { str += `, ${results.unvalidated.count} unvalidated (~${Math.round(results.unvalidated.percentage)}%)` } if (results.failing.count > 0) { str += `, ${results.failing.count} failing (~${Math.round(results.failing.percentage)}%)` } if (results.broken.count > 0) { str += `, ${results.broken.count} broken (~${Math.round(results.broken.percentage)}%)` } return str } addResultsDummy (test) { if (test.cid) { const allure = this.getAllure(test.cid) const currentTest = allure.getCurrentTest() this.testIds[currentTest] = true currentTest.addParameter('environment-variable', 'TESTCASE RESULTS: ', 'TESTCASE_RESULTS_STR') currentTest.addParameter('environment-variable', 'SPEC RESULTS: ', 'SPEC_RESULTS_STR') } } addResults () { const tcResults = this.baseReporter.getTestcaseResults() const specResults = this.baseReporter.getSpecResults() let tcResultsStr = this.buildResultStr(tcResults) let specResultsStr = this.buildResultStr(specResults) const latestRun = fs.readFileSync(this.config.latestRunPath, 'utf8') // replace TESTCASE_RESULTS_STR and SPEC_RESULTS_STR in all xml and json files in results folder // unfortunately there is not better way to do this at the moment getAllFiles(path.join(this.config.resultsPath, latestRun), '.xml').forEach( file => { let content = fs.readFileSync(file, 'utf8') content = content.replace(/TESTCASE_RESULTS_STR/g, tcResultsStr) content = content.replace(/SPEC_RESULTS_STR/g, specResultsStr) fs.unlinkSync(file) fs.writeFileSync(file, content, 'utf8') } ) } getAllure (cid) { if (typeof cid === 'undefined') { cid = process.workflo.cid } else { if (typeof process.workflo === 'undefined') { process.workflo = { } } else { process.workflo.currentCid = cid // REMOVE??? } } if (this.allures[cid]) { return this.allures[cid] } const allure = new Allure() allure.setOptions({ targetDir: this.options.outputDir || 'allure-results' }) this.allures[cid] = allure return this.allures[cid] } isAnyTestRunning (allure) { return allure.getCurrentSuite() && allure.getCurrentTest() } dumpJSON (allure, name, json) { const jsonStr = JSON.stringify(json, null, ' ').split(`\\\"`).join(`'`).replace(/(?:\\n)/g, '') allure.addAttachment(name, jsonStr, 'application/json') } addAllurePropertiesFile () { if (this.config.allure) { try { let allurePropertiesContents = [] if (this.config.allure.issueTrackerPattern) { allurePropertiesContents.push('allure.issues.tracker.pattern=' + this.config.allure.issueTrackerPattern) } if (this.config.allure.testManagementPattern) { allurePropertiesContents.push('allure.tests.management.pattern=' + this.config.allure.testManagementPattern) } fs.writeFileSync(path.join(this.options.outputDir, 'allure.properties'), allurePropertiesContents.join('\n'), { mode: 0o755 }) } catch (err) { console.error('Writing allure.properties file failed: \n') console.error(err) process.exit(1) } } } } const read = (dir) => fs.readdirSync(dir) .reduce( (files, file) => fs.statSync(path.join(dir, file)).isDirectory() ? files.concat(read(path.join(dir, file))) : files.concat(path.join(dir, file)), [] ) function getAllFiles (dirPath, extension) { return read(dirPath).filter( (fileName) => fileName.endsWith(extension) ) } export default AllureReporter