loadmill
Version:
A node.js module for running load tests and functional tests on loadmill.com
506 lines (505 loc) • 21.7 kB
JavaScript
"use strict";
exports.__esModule = true;
exports.mochawesomeReport = exports.junitReport = void 0;
var tslib_1 = require("tslib");
var fs = require("fs");
var path = require("path");
var http_request_1 = require("./http-request");
var pLimit = require('p-limit');
var flatMap = require("lodash/flatMap");
var isEmpty = require("lodash/isEmpty");
var find = require("lodash/find");
var forEach = require("lodash/forEach");
var includes = require("lodash/includes");
var utils_1 = require("./utils");
var POLLING_INTERVAL_MS = 5000;
var MAX_POLLING = 36; // 3 minutes
var MOCHA_AWESOME_RETRY_INTERVAL = 3000;
var generateJunitReport = function (testId, runType, token) { return tslib_1.__awaiter(void 0, void 0, void 0, function () {
var junitReportId, err_1;
return tslib_1.__generator(this, function (_a) {
switch (_a.label) {
case 0:
_a.trys.push([0, 2, , 3]);
return [4 /*yield*/, http_request_1.sendHttpRequest({
method: http_request_1.HttpMethods.POST,
url: junitReportAPI,
body: {
testId: testId,
runType: runType
},
token: token
})];
case 1:
junitReportId = (_a.sent()).body.junitReportId;
return [2 /*return*/, junitReportId];
case 2:
err_1 = _a.sent();
handleJunitFailed(err_1.message);
return [3 /*break*/, 3];
case 3: return [2 /*return*/];
}
});
}); };
var waitForAndSaveJunitReport = function (reportId, token, path) { return tslib_1.__awaiter(void 0, void 0, void 0, function () {
var polling_count, junitReport_1, err_2;
return tslib_1.__generator(this, function (_a) {
switch (_a.label) {
case 0:
polling_count = 0;
_a.label = 1;
case 1:
if (!(polling_count < MAX_POLLING)) return [3 /*break*/, 7];
_a.label = 2;
case 2:
_a.trys.push([2, 4, , 5]);
return [4 /*yield*/, http_request_1.sendHttpRequest({
url: junitReportAPI + "/" + reportId,
token: token
})];
case 3:
junitReport_1 = (_a.sent()).body.junitReport;
saveJunitReport(junitReport_1, path);
return [3 /*break*/, 7];
case 4:
err_2 = _a.sent();
if (err_2.status !== 404) {
handleJunitFailed(err_2.message);
return [3 /*break*/, 7];
}
return [3 /*break*/, 5];
case 5:
polling_count++;
return [4 /*yield*/, utils_1.sleep(POLLING_INTERVAL_MS)];
case 6:
_a.sent();
return [3 /*break*/, 1];
case 7:
if (polling_count === MAX_POLLING) {
handleJunitFailed('Generating report took too long. Please contact support');
}
return [2 /*return*/];
}
});
}); };
var saveJunitReport = function (junitReport, path) {
var resolvedPath = resolvePath(path ? path : './test-results', 'xml');
ensureDirectoryExistence(resolvedPath);
fs.writeFileSync(resolvedPath, junitReport);
};
var handleJunitFailed = function (errMsg) {
console.log("Failed to generate JUnit report" + (errMsg ? ": " + errMsg : ''));
};
var ensureDirectoryExistence = function (filePath) {
var dirname = path.dirname(filePath);
if (fs.existsSync(dirname)) {
return true;
}
ensureDirectoryExistence(dirname);
fs.mkdirSync(dirname);
};
var resolvePath = function (path, suffix) {
if (path.charAt(path.length - 1) == '/') {
path = path.substr(0, path.length - 1);
}
return path + "/loadmill/results." + suffix;
};
// TODO this all flow should come from @loadmill package
var toFailedFlowRunReport = function (flowRun, formater) {
var errs = [];
var result = flowRun.result, redactableResult = flowRun.redactableResult;
if (result.flow) {
var flow = result.flow, afterEach = result.afterEach;
appendFlowRunFailures(errs, formater, flow, redactableResult.flow);
if (afterEach) {
appendFlowRunFailures(errs, formater, afterEach, redactableResult.afterEach, flow.resolvedRequests.length);
}
}
else {
appendFlowRunFailures(errs, formater, result, redactableResult);
}
return errs;
};
var appendFlowRunFailures = function (errs, formater, result, redactableResult, offset) {
if (offset === void 0) { offset = 0; }
var _a = result, resolvedRequests = _a.resolvedRequests, failures = _a.failures, err = _a.err;
if (Array.isArray(resolvedRequests) && resolvedRequests.length > 0) {
resolvedRequests.map(function (req, i) {
var description = req.description, method = req.method, url = req.url, _a = req.assert, assert = _a === void 0 ? [] : _a;
var postParameters = redactableResult && redactableResult[i].postParameters;
var reqFailures = failures && failures[i];
var numSuccesses = 1;
var _b = reqFailures || {}, _c = _b.histogram, histogram = _c === void 0 ? {} : _c, _d = _b.numFailures, numFailures = _d === void 0 ? 0 : _d;
var totalNumRequests = numSuccesses + numFailures;
if (numFailures > 0) {
var flowFailedText_1 = genReqDesc(i + offset) + " " + (description ? genReqDesc(i + offset, description) : '') + " " + method + " " + url + " =>";
var assertionNames_1 = Object.keys(assert);
var requestErrorNames = Object.keys(histogram).filter(function (name) { return !includes(assertionNames_1, name); });
requestErrorNames.map(function (name) {
flowFailedText_1 += " " + name + " ";
});
errs.push(flowFailedText_1);
var flatPostParameters_1 = flatMap(postParameters);
var assertionItems = getItems(assertionNames_1, histogram, totalNumRequests);
forEach(assertionItems, function (assertion) {
if (assert[assertion.name]) {
var check = assert[assertion.name].check;
var actual = getActualValue(assertion.errorRate, flatPostParameters_1, check);
if (actual) {
var assErr = generateAssertionName(assert[assertion.name], actual, formater);
errs.push(assErr);
}
}
});
}
});
}
else if (err) {
errs.push(typeof err === 'string' ? err : err.message);
}
};
function generateAssertionName(_a, actual, formatAssertion) {
var check = _a.check, equals = _a.equals, notEquals = _a.notEquals, contains = _a.contains, notContains = _a.notContains, matches = _a.matches, falsy = _a.falsy, greater = _a.greater, lesser = _a.lesser, JSONSchema = _a.JSONSchema, JSONContains = _a.JSONContains;
if (equals != null) {
return formatAssertion(check, 'Equals', equals, actual);
}
else if (notEquals != null) {
return formatAssertion(check, 'Doesn\'t equal', notEquals, actual);
}
else if (contains != null) {
return formatAssertion(check, 'Contains', contains, actual);
}
else if (notContains != null) {
return formatAssertion(check, 'Doesn\'t contain', notContains, actual);
}
else if (matches != null) {
return formatAssertion(check, 'Matches', matches, actual);
}
else if (greater != null) {
return formatAssertion(check, 'Greater than', greater, actual);
}
else if (lesser != null) {
return formatAssertion(check, 'Less than', lesser, actual);
}
else if (falsy != null) {
return formatAssertion(check, 'Doesn\'t exist', null, actual);
}
else if (JSONSchema != null) {
return formatAssertion(check, 'JSON Schema', JSONSchema, actual);
}
else if (JSONContains != null) {
return formatAssertion(check, 'JSON Contains', JSONContains, actual);
}
else {
return formatAssertion(check, 'Exists', null, actual);
}
}
/**
* null - we dont want to show actual value - load test or successfull test.
* NULL_VAL - we want to show there was a null value.
* else - show the actual value.
*/
function getActualValue(errorRate, postParameters, check) {
if (errorRate && !isEmpty(postParameters)) {
// empty means load test
var exists = find(postParameters, function (p) { return p[check]; });
return exists ? exists[check] : 'null';
}
return null;
}
function getItems(names, histogram, totalNumRequests) {
var items = names.sort().map(function (name) {
var numFailures = histogram[name];
var errorRate = calculateErrorRate(numFailures, totalNumRequests - numFailures);
return { name: name, errorRate: errorRate };
});
return isEmpty(items) ? [{ name: 'None', errorRate: 0 }] : items;
}
function calculateErrorRate(failures, successes) {
failures = numberify(failures);
successes = numberify(successes);
if (failures === 0) {
return 0;
}
else if (successes === 0) {
return 1;
}
else {
return failures / (failures + successes);
}
}
function numberify(num, defaultValue) {
if (defaultValue === void 0) { defaultValue = 0; }
if (num == null) {
return defaultValue;
}
else {
return Number(num);
}
}
function genReqDesc(index, description) {
return (description ? description : "Request #" + (index + 1)) + " -";
}
function getFlowRunAPI(f) {
return utils_1.TESTING_ORIGIN + "/api/test-suites-runs/flows/" + f.id;
}
function generateCodeBlock(s, f) {
var res = "URL: " + getFlowRunWebURL(s, f);
if (f.retries) {
res += "\nRetries: " + f.retries;
}
return res;
}
function getFlowRunWebURL(s, f) {
return utils_1.TESTING_ORIGIN + "/app/api-tests/test-suite-runs/" + s.id + "/flows/" + f.id;
}
var junitReportAPI = utils_1.TESTING_ORIGIN + "/api/reports/junit";
var toMochawesomeFailedFlow = function (flowRun) {
var errs = toFailedFlowRunReport(flowRun, function (check, operation, value, actual) {
var text = '';
if (actual != null) {
text += "\n+ Expected: " + check + " " + operation + " " + (value != null ? value : '') + " ";
text += "\n- Actual: " + (actual !== 'null' ? actual : 'null') + " ";
}
return text;
});
return {
"showDiff": true,
"actual": "",
"negate": false,
"_message": "",
"generatedMessage": false,
"diff": errs.join('\n')
};
};
var flowToMochawesone = function (suite, flow, token) { return tslib_1.__awaiter(void 0, void 0, void 0, function () {
var url, flowData, hasPassed, hasFailed, res;
return tslib_1.__generator(this, function (_a) {
switch (_a.label) {
case 0:
url = getFlowRunAPI(flow);
return [4 /*yield*/, fetchFlowRunData(url, token)];
case 1:
flowData = _a.sent();
hasPassed = _hasPassed(flow);
hasFailed = flow.status === 'FAILED';
res = {
"title": flow.description,
"fullTitle": flow.description,
"timedOut": false,
"duration": flow.duration,
"state": hasPassed ? 'passed' : 'failed',
"pass": hasPassed,
"fail": hasFailed,
"isHook": false,
"skipped": false,
"pending": false,
"code": generateCodeBlock(suite, flow),
"err": hasFailed ? toMochawesomeFailedFlow(flowData) : {},
"uuid": flow.id
};
return [2 /*return*/, res];
}
});
}); };
var suiteToMochawesone = function (suite, token) { return tslib_1.__awaiter(void 0, void 0, void 0, function () {
var flows, passedFlows, failedFlows, limit, _a;
var _b;
return tslib_1.__generator(this, function (_c) {
switch (_c.label) {
case 0:
flows = suite.flowRuns || [];
passedFlows = flows.filter(function (f) { return _hasPassed(f); }).map(function (f) { return f.id; });
failedFlows = flows.filter(function (f) { return f.status === 'FAILED'; }).map(function (f) { return f.id; });
limit = pLimit(3);
_b = {
"title": suite.description
};
_a = "tests";
return [4 /*yield*/, Promise.all(flows.filter(function (flow) { return _hasPassed(flow) || flow.status === 'FAILED'; })
.map(function (f) { return limit(function () { return flowToMochawesone(suite, f, token); }); }))];
case 1: return [2 /*return*/, (_b[_a] = _c.sent(),
_b["duration"] = ((+suite.endTime || Date.now()) - +suite.startTime),
_b["suites"] = [],
_b["uuid"] = suite.id,
_b["passes"] = passedFlows,
_b["failures"] = failedFlows,
_b["root"] = false,
_b["_timeout"] = 0,
_b["file"] = "",
_b["fullFile"] = "",
_b["beforeHooks"] = [],
_b["afterHooks"] = [],
_b["skipped"] = [],
_b["pending"] = [],
_b)];
}
});
}); };
var generateMochawesomeReport = function (testResult, token) { return tslib_1.__awaiter(void 0, void 0, void 0, function () {
var suites, passedSuites, failedSuites, duration, suitesLength, limit, res, _a, _b;
var _c, _d;
return tslib_1.__generator(this, function (_e) {
switch (_e.label) {
case 0:
suites = testResult.testSuitesRuns || [testResult];
passedSuites = suites.filter(function (t) { return t.passed; }).length;
failedSuites = suites.filter(function (t) { return !t.passed; }).length;
duration = suites.reduce(function (acc, s) { return acc + ((+s.endTime || Date.now()) - +s.startTime); }, 0);
suitesLength = suites.length;
limit = pLimit(3);
_c = {
"stats": {
"suites": suitesLength,
"tests": suitesLength,
"passes": passedSuites,
"failures": failedSuites,
"start": (suites[0] ? getFirstExecutedSuiteTime(suites) : new Date()).toISOString(),
"end": new Date().toISOString(),
"pending": 0,
"testsRegistered": suitesLength,
"pendingPercent": 0,
"passPercent": suitesLength == 0 ? 0 : (passedSuites / suitesLength) * 100,
"other": 0,
"hasOther": false,
"skipped": 0,
"hasSkipped": false,
"duration": duration
}
};
_a = "results";
_d = {
"title": "Loadmill API tests"
};
_b = "suites";
return [4 /*yield*/, Promise.all(suites.map(function (s) { return limit(function () { return suiteToMochawesone(s, token); }); }))];
case 1:
res = (_c[_a] = [
(_d[_b] = _e.sent(),
_d["tests"] = [],
_d["pending"] = [],
_d["root"] = true,
_d["_timeout"] = 0,
_d["uuid"] = suites[0] ? suites[0].id : '123e4567-e89b-12d3-a456-426652340000',
_d["beforeHooks"] = [],
_d["afterHooks"] = [],
_d["fullFile"] = "",
_d["file"] = "",
_d["passes"] = [],
_d["failures"] = [],
_d["skipped"] = [],
_d["duration"] = duration,
_d["rootEmpty"] = true,
_d)
],
_c);
return [2 /*return*/, res];
}
});
}); };
var junitReport = function (testResult, token, path) { return tslib_1.__awaiter(void 0, void 0, void 0, function () {
var reportId, _a;
return tslib_1.__generator(this, function (_b) {
switch (_b.label) {
case 0:
if (!testResult) {
return [2 /*return*/];
}
console.log('Generating JUnit report...');
return [4 /*yield*/, generateJunitReport(testResult.id, testResult.type, token)];
case 1:
reportId = _b.sent();
_a = reportId;
if (!_a) return [3 /*break*/, 3];
return [4 /*yield*/, waitForAndSaveJunitReport(reportId, token, path)];
case 2:
_a = (_b.sent());
_b.label = 3;
case 3:
_a;
console.log('Finished generating JUnit report');
return [2 /*return*/];
}
});
}); };
exports.junitReport = junitReport;
var mochawesomeReport = function (testResult, token, path) { return tslib_1.__awaiter(void 0, void 0, void 0, function () {
var jsonResults, resolvedPath;
return tslib_1.__generator(this, function (_a) {
switch (_a.label) {
case 0:
console.log('Generating Mochawesome report...');
if (!testResult) {
console.log('No test result to generate report');
return [2 /*return*/];
}
return [4 /*yield*/, generateMochawesomeReport(testResult, token)];
case 1:
jsonResults = _a.sent();
resolvedPath = resolvePath(path ? path : './mochawesome-results', 'json');
ensureDirectoryExistence(resolvedPath);
console.log("Writing Mochawesome report to " + resolvedPath);
fs.writeFileSync(resolvedPath, JSON.stringify(jsonResults, null, 2));
console.log('Finished generating Mochawesome report');
return [2 /*return*/];
}
});
}); };
exports.mochawesomeReport = mochawesomeReport;
function fetchFlowRunData(url, token) {
return tslib_1.__awaiter(this, void 0, void 0, function () {
var body, err_3, body, error_1, body, error_2;
return tslib_1.__generator(this, function (_a) {
switch (_a.label) {
case 0:
_a.trys.push([0, 2, , 13]);
return [4 /*yield*/, http_request_1.sendHttpRequest({ url: url, token: token })];
case 1:
body = (_a.sent()).body;
return [2 /*return*/, body];
case 2:
err_3 = _a.sent();
_a.label = 3;
case 3:
_a.trys.push([3, 6, , 12]);
console.log("Failed to fetch flow run data for " + url + ". Retrying...");
return [4 /*yield*/, utils_1.sleep(MOCHA_AWESOME_RETRY_INTERVAL)];
case 4:
_a.sent();
return [4 /*yield*/, http_request_1.sendHttpRequest({ url: url, token: token })];
case 5:
body = (_a.sent()).body;
return [2 /*return*/, body];
case 6:
error_1 = _a.sent();
_a.label = 7;
case 7:
_a.trys.push([7, 10, , 11]);
console.log("Failed to fetch flow run data for " + url + ". Retrying last time...");
return [4 /*yield*/, utils_1.sleep(MOCHA_AWESOME_RETRY_INTERVAL)];
case 8:
_a.sent();
return [4 /*yield*/, http_request_1.sendHttpRequest({ url: url, token: token })];
case 9:
body = (_a.sent()).body;
return [2 /*return*/, body];
case 10:
error_2 = _a.sent();
console.log("Failed to fetch flow run data for " + url);
throw error_2;
case 11: return [3 /*break*/, 12];
case 12: return [3 /*break*/, 13];
case 13: return [2 /*return*/];
}
});
});
}
function getFirstExecutedSuiteTime(suites) {
var firstSuite = suites.reduce(function (prev, curr) {
return prev.startTime < curr.startTime ? prev : curr;
});
return new Date(firstSuite.startTime);
}
var _hasPassed = function (flow) { return flow.status === 'PASSED' || _isFlaky(flow); };
// check by status alone is not enough because of backward compatibility
var _isFlaky = function (flow) { return flow.status === 'FLAKY' && !!flow.retries; };