intern
Version:
Intern. A next-generation code testing stack for JavaScript.
277 lines • 12.2 kB
JavaScript
(function (factory) {
if (typeof module === "object" && typeof module.exports === "object") {
var v = factory(require, exports);
if (v !== undefined) module.exports = v;
}
else if (typeof define === "function" && define.amd) {
define(["require", "exports", "tslib", "fs", "./Reporter"], factory);
}
})(function (require, exports) {
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
var tslib_1 = require("tslib");
var fs_1 = require("fs");
var Reporter_1 = tslib_1.__importStar(require("./Reporter"));
var BenchmarkReporter = (function (_super) {
tslib_1.__extends(BenchmarkReporter, _super);
function BenchmarkReporter(executor, options) {
if (options === void 0) { options = {}; }
var _this = _super.call(this, executor, options) || this;
_this.mode = options.mode || 'test';
_this.filename = options.filename || '';
_this.thresholds = options.thresholds || {};
if (_this.mode === 'test') {
try {
_this.baseline = JSON.parse(fs_1.readFileSync(_this.filename, { encoding: 'utf8' }));
}
catch (error) {
_this.console.warn('Unable to load benchmark baseline data from ' + _this.filename);
_this.console.warn('Switching to "baseline" mode');
_this.mode = 'baseline';
}
}
if (!_this.baseline) {
_this.baseline = {};
}
_this.sessions = {};
return _this;
}
BenchmarkReporter.prototype._getSession = function (testOrSuite) {
var sessionId = testOrSuite.sessionId || 'local';
var session = this.sessions[sessionId];
if (!session) {
var client = void 0;
var version = void 0;
var platform_1;
if (testOrSuite.sessionId) {
var environmentType = testOrSuite.remote.environmentType;
client = environmentType.browserName;
version = environmentType.version;
platform_1 = environmentType.platform;
}
else {
client = process.title;
version = process.version;
platform_1 = process.platform;
}
session = this.sessions[sessionId] = {
suites: {},
environment: {
client: client,
version: version,
platform: platform_1,
id: client + ':' + version + ':' + platform_1
}
};
}
return session;
};
BenchmarkReporter.prototype.runEnd = function () {
if (this.mode === 'baseline') {
var existingBaseline_1;
try {
existingBaseline_1 = JSON.parse(fs_1.readFileSync(this.filename, { encoding: 'utf8' }));
}
catch (error) {
existingBaseline_1 = {};
}
var baseline_1 = this.baseline;
Object.keys(baseline_1).forEach(function (environmentId) {
existingBaseline_1[environmentId] = baseline_1[environmentId];
});
fs_1.writeFileSync(this.filename, JSON.stringify(existingBaseline_1, null, ' '));
}
};
BenchmarkReporter.prototype.suiteEnd = function (suite) {
var session = this._getSession(suite);
if (!suite.hasParent) {
var environment = session.environment;
this.console.log('Finished benchmarking ' +
environment.client +
' ' +
environment.version +
' on ' +
environment.platform);
}
else if (this.mode === 'test') {
var suiteInfo = session.suites[suite.id];
var numTests = suiteInfo.numBenchmarks;
if (numTests > 0) {
var numFailedTests = suiteInfo.numFailedBenchmarks;
var message = numFailedTests + '/' + numTests + ' benchmarks failed in ' + suite.id;
if (numFailedTests > 0) {
this.console.warn(message);
}
else {
this.console.log(message);
}
}
}
};
BenchmarkReporter.prototype.suiteStart = function (suite) {
var session = this._getSession(suite);
if (!suite.hasParent) {
var environment = session.environment;
var environmentName = environment.client +
' ' +
environment.version +
' on ' +
environment.platform;
var baselineEnvironments = this.baseline;
this.console.log((this.mode === 'baseline' ? 'Baselining' : 'Benchmarking') +
' ' +
environmentName);
if (this.mode === 'baseline') {
baselineEnvironments[environment.id] = {
client: environment.client,
version: environment.version,
platform: environment.platform,
tests: {}
};
}
else if (!baselineEnvironments[environment.id]) {
this.console.warn('No baseline data for ' + environmentName + '!');
}
}
else {
session.suites[suite.id] = {
numBenchmarks: 0,
numFailedBenchmarks: 0
};
}
};
BenchmarkReporter.prototype.testEnd = function (test) {
var _this = this;
var benchmarkTest = test;
if (benchmarkTest.benchmark == null) {
return;
}
if (benchmarkTest.error) {
var session = this._getSession(benchmarkTest);
var suiteInfo = session.suites[benchmarkTest.parentId];
suiteInfo.numBenchmarks++;
suiteInfo.numFailedBenchmarks++;
this.console.error('FAIL: ' + benchmarkTest.id);
this.console.error(this.executor.formatError(benchmarkTest.error, { space: ' ' }));
}
else {
var checkTest = function (baseline, benchmark) {
var warn = [];
var fail = [];
var list;
var baselineMean = baseline.stats.mean;
var thresholds = _this.thresholds || {};
var percentDifference = (100 * (benchmark.stats.mean - baselineMean)) / baselineMean;
if (thresholds.warn &&
thresholds.warn.mean &&
Math.abs(percentDifference) > thresholds.warn.mean) {
list = warn;
if (thresholds.fail &&
thresholds.fail.mean &&
Math.abs(percentDifference) > thresholds.fail.mean) {
list = fail;
}
list.push('Execution time is ' + percentDifference.toFixed(1) + '% off');
}
var baselineRme = baseline.stats.rme;
percentDifference = benchmark.stats.rme - baselineRme;
if (thresholds.warn &&
thresholds.warn.rme &&
Math.abs(percentDifference) > thresholds.warn.rme) {
list = warn;
if (thresholds.fail &&
thresholds.fail.rme &&
Math.abs(percentDifference) > thresholds.fail.rme) {
list = fail;
}
list.push('RME is ' + percentDifference.toFixed(1) + '% off');
}
if (fail.length) {
_this.console.error('FAIL ' + benchmarkTest.id + ' (' + fail.join(', ') + ')');
return false;
}
else if (warn.length) {
_this.console.warn('WARN ' + benchmarkTest.id + ' (' + warn.join(', ') + ')');
}
else {
_this.console.log('PASS ' + benchmarkTest.id);
}
return true;
};
var benchmark = benchmarkTest.benchmark;
var session = this._getSession(benchmarkTest);
var environment = session.environment;
var suiteInfo = session.suites[benchmarkTest.parentId];
suiteInfo.numBenchmarks++;
var baseline = this.baseline[environment.id];
if (this.mode === 'baseline') {
baseline.tests[benchmarkTest.id] = {
hz: benchmark.hz,
times: benchmark.times,
stats: {
rme: benchmark.stats.rme,
moe: benchmark.stats.moe,
mean: benchmark.stats.mean
}
};
this.console.log('Baselined ' + benchmarkTest.name);
this.executor.log('Time per run:', formatSeconds(benchmark.stats.mean), '\xb1', benchmark.stats.rme.toFixed(2), '%');
}
else {
if (baseline) {
var testData = baseline.tests[benchmarkTest.id];
var result = checkTest(testData, benchmark);
var baselineStats = testData.stats;
var benchmarkStats = benchmark.stats;
this.executor.log('Expected time per run:', formatSeconds(baselineStats.mean), '\xb1', baselineStats.rme.toFixed(2), '%');
this.executor.log('Actual time per run:', formatSeconds(benchmarkStats.mean), '\xb1', benchmarkStats.rme.toFixed(2), '%');
if (!result) {
suiteInfo.numFailedBenchmarks++;
}
}
}
}
};
tslib_1.__decorate([
Reporter_1.eventHandler()
], BenchmarkReporter.prototype, "runEnd", null);
tslib_1.__decorate([
Reporter_1.eventHandler()
], BenchmarkReporter.prototype, "suiteEnd", null);
tslib_1.__decorate([
Reporter_1.eventHandler()
], BenchmarkReporter.prototype, "suiteStart", null);
tslib_1.__decorate([
Reporter_1.eventHandler()
], BenchmarkReporter.prototype, "testEnd", null);
return BenchmarkReporter;
}(Reporter_1.default));
exports.default = BenchmarkReporter;
function formatSeconds(value) {
if (value == null) {
return null;
}
var units = 's';
if (value < 1) {
var places = Math.ceil(Math.log(value) / Math.log(10)) - 1;
if (places < -9) {
value *= Math.pow(10, 12);
units = 'ps';
}
else if (places < -6) {
value *= Math.pow(10, 9);
units = 'ns';
}
else if (places < -3) {
value *= Math.pow(10, 6);
units = 'µs';
}
else if (places < 0) {
value *= Math.pow(10, 3);
units = 'ms';
}
}
return value.toFixed(3) + units;
}
});
//# sourceMappingURL=Benchmark.js.map