@angular/benchpress
Version:
Benchpress - a framework for e2e performance tests
1,357 lines (1,336 loc) • 67.7 kB
JavaScript
/**
* @license Angular v0.0.0
* (c) 2010-2022 Google LLC. https://angular.io/
* License: MIT
*/
import 'reflect-metadata';
import { InjectionToken, Injector, Injectable, Inject } from '@angular/core';
export { InjectionToken, Injector, ReflectiveInjector } from '@angular/core';
import * as fs from 'fs';
import { __decorate, __param, __metadata } from 'tslib';
/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
class Options {
}
Options.SAMPLE_ID = new InjectionToken('Options.sampleId');
Options.DEFAULT_DESCRIPTION = new InjectionToken('Options.defaultDescription');
Options.SAMPLE_DESCRIPTION = new InjectionToken('Options.sampleDescription');
Options.FORCE_GC = new InjectionToken('Options.forceGc');
Options.NO_PREPARE = () => true;
Options.PREPARE = new InjectionToken('Options.prepare');
Options.EXECUTE = new InjectionToken('Options.execute');
Options.CAPABILITIES = new InjectionToken('Options.capabilities');
Options.USER_AGENT = new InjectionToken('Options.userAgent');
Options.MICRO_METRICS = new InjectionToken('Options.microMetrics');
Options.USER_METRICS = new InjectionToken('Options.userMetrics');
Options.NOW = new InjectionToken('Options.now');
Options.WRITE_FILE = new InjectionToken('Options.writeFile');
Options.RECEIVED_DATA = new InjectionToken('Options.receivedData');
Options.REQUEST_COUNT = new InjectionToken('Options.requestCount');
Options.CAPTURE_FRAMES = new InjectionToken('Options.frameCapture');
Options.RAW_PERFLOG_PATH = new InjectionToken('Options.rawPerflogPath');
Options.DEFAULT_PROVIDERS = [
{ provide: Options.DEFAULT_DESCRIPTION, useValue: {} },
{ provide: Options.SAMPLE_DESCRIPTION, useValue: {} },
{ provide: Options.FORCE_GC, useValue: false },
{ provide: Options.PREPARE, useValue: Options.NO_PREPARE },
{ provide: Options.MICRO_METRICS, useValue: {} }, { provide: Options.USER_METRICS, useValue: {} },
{ provide: Options.NOW, useValue: () => new Date() },
{ provide: Options.RECEIVED_DATA, useValue: false },
{ provide: Options.REQUEST_COUNT, useValue: false },
{ provide: Options.CAPTURE_FRAMES, useValue: false },
{ provide: Options.WRITE_FILE, useValue: writeFile },
{ provide: Options.RAW_PERFLOG_PATH, useValue: null }
];
function writeFile(filename, content) {
return new Promise(function (resolve, reject) {
fs.writeFile(filename, content, (error) => {
if (error) {
reject(error);
}
else {
resolve();
}
});
});
}
/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
class MeasureValues {
constructor(runIndex, timeStamp, values) {
this.runIndex = runIndex;
this.timeStamp = timeStamp;
this.values = values;
}
toJson() {
return {
'timeStamp': this.timeStamp.toJSON(),
'runIndex': this.runIndex,
'values': this.values,
};
}
}
/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
/**
* A metric is measures values
*/
class Metric {
/**
* Starts measuring
*/
beginMeasure() {
throw new Error('NYI');
}
/**
* Ends measuring and reports the data
* since the begin call.
* @param restart: Whether to restart right after this.
*/
endMeasure(restart) {
throw new Error('NYI');
}
/**
* Describes the metrics provided by this metric implementation.
* (e.g. units, ...)
*/
describe() {
throw new Error('NYI');
}
}
/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
class MultiMetric extends Metric {
constructor(_metrics) {
super();
this._metrics = _metrics;
}
static provideWith(childTokens) {
return [
{
provide: _CHILDREN$2,
useFactory: (injector) => childTokens.map(token => injector.get(token)),
deps: [Injector]
},
{
provide: MultiMetric,
useFactory: (children) => new MultiMetric(children),
deps: [_CHILDREN$2]
}
];
}
/**
* Starts measuring
*/
beginMeasure() {
return Promise.all(this._metrics.map(metric => metric.beginMeasure()));
}
/**
* Ends measuring and reports the data
* since the begin call.
* @param restart: Whether to restart right after this.
*/
endMeasure(restart) {
return Promise.all(this._metrics.map(metric => metric.endMeasure(restart)))
.then(values => mergeStringMaps(values));
}
/**
* Describes the metrics provided by this metric implementation.
* (e.g. units, ...)
*/
describe() {
return mergeStringMaps(this._metrics.map((metric) => metric.describe()));
}
}
function mergeStringMaps(maps) {
const result = {};
maps.forEach(map => {
Object.keys(map).forEach(prop => {
result[prop] = map[prop];
});
});
return result;
}
const _CHILDREN$2 = new InjectionToken('MultiMetric.children');
/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
/**
* A WebDriverExtension implements extended commands of the webdriver protocol
* for a given browser, independent of the WebDriverAdapter.
* Needs one implementation for every supported Browser.
*/
class WebDriverExtension {
static provideFirstSupported(childTokens) {
const res = [
{
provide: _CHILDREN$1,
useFactory: (injector) => childTokens.map(token => injector.get(token)),
deps: [Injector]
},
{
provide: WebDriverExtension,
useFactory: (children, capabilities) => {
let delegate = undefined;
children.forEach(extension => {
if (extension.supports(capabilities)) {
delegate = extension;
}
});
if (!delegate) {
throw new Error('Could not find a delegate for given capabilities!');
}
return delegate;
},
deps: [_CHILDREN$1, Options.CAPABILITIES]
}
];
return res;
}
gc() {
throw new Error('NYI');
}
timeBegin(name) {
throw new Error('NYI');
}
timeEnd(name, restartName) {
throw new Error('NYI');
}
/**
* Format:
* - cat: category of the event
* - name: event name: 'script', 'gc', 'render', ...
* - ph: phase: 'B' (begin), 'E' (end), 'X' (Complete event), 'I' (Instant event)
* - ts: timestamp in ms, e.g. 12345
* - pid: process id
* - args: arguments, e.g. {heapSize: 1234}
*
* Based on [Chrome Trace Event
*Format](https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/edit)
**/
readPerfLog() {
throw new Error('NYI');
}
perfLogFeatures() {
throw new Error('NYI');
}
supports(capabilities) {
return true;
}
}
class PerfLogFeatures {
constructor({ render = false, gc = false, frameCapture = false, userTiming = false } = {}) {
this.render = render;
this.gc = gc;
this.frameCapture = frameCapture;
this.userTiming = userTiming;
}
}
const _CHILDREN$1 = new InjectionToken('WebDriverExtension.children');
/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
var PerflogMetric_1;
/**
* A metric that reads out the performance log
*/
let PerflogMetric = PerflogMetric_1 = class PerflogMetric extends Metric {
/**
* @param driverExtension
* @param setTimeout
* @param microMetrics Name and description of metrics provided via console.time / console.timeEnd
* @param ignoreNavigation If true, don't measure from navigationStart events. These events are
* usually triggered by a page load, but can also be triggered when adding iframes to the DOM.
**/
constructor(_driverExtension, _setTimeout, _microMetrics, _forceGc, _captureFrames, _receivedData, _requestCount, _ignoreNavigation) {
super();
this._driverExtension = _driverExtension;
this._setTimeout = _setTimeout;
this._microMetrics = _microMetrics;
this._forceGc = _forceGc;
this._captureFrames = _captureFrames;
this._receivedData = _receivedData;
this._requestCount = _requestCount;
this._ignoreNavigation = _ignoreNavigation;
this._remainingEvents = [];
this._measureCount = 0;
this._perfLogFeatures = _driverExtension.perfLogFeatures();
if (!this._perfLogFeatures.userTiming) {
// User timing is needed for navigationStart.
this._receivedData = false;
this._requestCount = false;
}
}
describe() {
const res = {
'scriptTime': 'script execution time in ms, including gc and render',
'pureScriptTime': 'script execution time in ms, without gc nor render'
};
if (this._perfLogFeatures.render) {
res['renderTime'] = 'render time in ms';
}
if (this._perfLogFeatures.gc) {
res['gcTime'] = 'gc time in ms';
res['gcAmount'] = 'gc amount in kbytes';
res['majorGcTime'] = 'time of major gcs in ms';
if (this._forceGc) {
res['forcedGcTime'] = 'forced gc time in ms';
res['forcedGcAmount'] = 'forced gc amount in kbytes';
}
}
if (this._receivedData) {
res['receivedData'] = 'encoded bytes received since navigationStart';
}
if (this._requestCount) {
res['requestCount'] = 'count of requests sent since navigationStart';
}
if (this._captureFrames) {
if (!this._perfLogFeatures.frameCapture) {
const warningMsg = 'WARNING: Metric requested, but not supported by driver';
// using dot syntax for metric name to keep them grouped together in console reporter
res['frameTime.mean'] = warningMsg;
res['frameTime.worst'] = warningMsg;
res['frameTime.best'] = warningMsg;
res['frameTime.smooth'] = warningMsg;
}
else {
res['frameTime.mean'] = 'mean frame time in ms (target: 16.6ms for 60fps)';
res['frameTime.worst'] = 'worst frame time in ms';
res['frameTime.best'] = 'best frame time in ms';
res['frameTime.smooth'] = 'percentage of frames that hit 60fps';
}
}
for (const name in this._microMetrics) {
res[name] = this._microMetrics[name];
}
return res;
}
beginMeasure() {
let resultPromise = Promise.resolve(null);
if (this._forceGc) {
resultPromise = resultPromise.then((_) => this._driverExtension.gc());
}
return resultPromise.then((_) => this._beginMeasure());
}
endMeasure(restart) {
if (this._forceGc) {
return this._endPlainMeasureAndMeasureForceGc(restart);
}
else {
return this._endMeasure(restart);
}
}
/** @internal */
_endPlainMeasureAndMeasureForceGc(restartMeasure) {
return this._endMeasure(true).then((measureValues) => {
// disable frame capture for measurements during forced gc
const originalFrameCaptureValue = this._captureFrames;
this._captureFrames = false;
return this._driverExtension.gc()
.then((_) => this._endMeasure(restartMeasure))
.then((forceGcMeasureValues) => {
this._captureFrames = originalFrameCaptureValue;
measureValues['forcedGcTime'] = forceGcMeasureValues['gcTime'];
measureValues['forcedGcAmount'] = forceGcMeasureValues['gcAmount'];
return measureValues;
});
});
}
_beginMeasure() {
return this._driverExtension.timeBegin(this._markName(this._measureCount++));
}
_endMeasure(restart) {
const markName = this._markName(this._measureCount - 1);
const nextMarkName = restart ? this._markName(this._measureCount++) : null;
return this._driverExtension.timeEnd(markName, nextMarkName)
.then((_) => this._readUntilEndMark(markName));
}
_readUntilEndMark(markName, loopCount = 0, startEvent = null) {
if (loopCount > _MAX_RETRY_COUNT) {
throw new Error(`Tried too often to get the ending mark: ${loopCount}`);
}
return this._driverExtension.readPerfLog().then((events) => {
this._addEvents(events);
const result = this._aggregateEvents(this._remainingEvents, markName);
if (result) {
this._remainingEvents = events;
return result;
}
let resolve;
const promise = new Promise(res => {
resolve = res;
});
this._setTimeout(() => resolve(this._readUntilEndMark(markName, loopCount + 1)), 100);
return promise;
});
}
_addEvents(events) {
let needSort = false;
events.forEach(event => {
if (event['ph'] === 'X') {
needSort = true;
const startEvent = {};
const endEvent = {};
for (const prop in event) {
startEvent[prop] = event[prop];
endEvent[prop] = event[prop];
}
startEvent['ph'] = 'B';
endEvent['ph'] = 'E';
endEvent['ts'] = startEvent['ts'] + startEvent['dur'];
this._remainingEvents.push(startEvent);
this._remainingEvents.push(endEvent);
}
else {
this._remainingEvents.push(event);
}
});
if (needSort) {
// Need to sort because of the ph==='X' events
this._remainingEvents.sort((a, b) => {
const diff = a['ts'] - b['ts'];
return diff > 0 ? 1 : diff < 0 ? -1 : 0;
});
}
}
_aggregateEvents(events, markName) {
const result = { 'scriptTime': 0, 'pureScriptTime': 0 };
if (this._perfLogFeatures.gc) {
result['gcTime'] = 0;
result['majorGcTime'] = 0;
result['gcAmount'] = 0;
}
if (this._perfLogFeatures.render) {
result['renderTime'] = 0;
}
if (this._captureFrames) {
result['frameTime.mean'] = 0;
result['frameTime.best'] = 0;
result['frameTime.worst'] = 0;
result['frameTime.smooth'] = 0;
}
for (const name in this._microMetrics) {
result[name] = 0;
}
if (this._receivedData) {
result['receivedData'] = 0;
}
if (this._requestCount) {
result['requestCount'] = 0;
}
let markStartEvent = null;
let markEndEvent = null;
events.forEach((event) => {
const ph = event['ph'];
const name = event['name'];
// Here we are determining if this is the event signaling the start or end of our performance
// testing (this is triggered by us calling #timeBegin and #timeEnd).
//
// Previously, this was done by checking that the event name matched our mark name and that
// the phase was either "B" or "E" ("begin" or "end"). However, since Chrome v90 this is
// showing up as "-bpstart" and "-bpend" ("benchpress start/end"), which is what one would
// actually expect since that is the mark name used in ChromeDriverExtension - see the
// #timeBegin and #timeEnd implementations in chrome_driver_extension.ts. For
// backwards-compatibility with Chrome v89 (and older), we do both checks: the phase-based
// one ("B" or "E") and event name-based (the "-bp(start/end)" suffix).
const isStartEvent = (ph === 'B' && name === markName) || name === markName + '-bpstart';
const isEndEvent = (ph === 'E' && name === markName) || name === markName + '-bpend';
if (isStartEvent) {
markStartEvent = event;
}
else if (ph === 'I' && name === 'navigationStart' && !this._ignoreNavigation) {
// if a benchmark measures reload of a page, use the last
// navigationStart as begin event
markStartEvent = event;
}
else if (isEndEvent) {
markEndEvent = event;
}
});
if (!markStartEvent || !markEndEvent) {
// not all events have been received, no further processing for now
return null;
}
if (markStartEvent.pid !== markEndEvent.pid) {
result['invalid'] = 1;
}
let gcTimeInScript = 0;
let renderTimeInScript = 0;
const frameTimestamps = [];
const frameTimes = [];
let frameCaptureStartEvent = null;
let frameCaptureEndEvent = null;
const intervalStarts = {};
const intervalStartCount = {};
let inMeasureRange = false;
events.forEach((event) => {
const ph = event['ph'];
let name = event['name'];
let microIterations = 1;
const microIterationsMatch = name.match(_MICRO_ITERATIONS_REGEX);
if (microIterationsMatch) {
name = microIterationsMatch[1];
microIterations = parseInt(microIterationsMatch[2], 10);
}
if (event === markStartEvent) {
inMeasureRange = true;
}
else if (event === markEndEvent) {
inMeasureRange = false;
}
if (!inMeasureRange || event['pid'] !== markStartEvent['pid']) {
return;
}
if (this._requestCount && name === 'sendRequest') {
result['requestCount'] += 1;
}
else if (this._receivedData && name === 'receivedData' && ph === 'I') {
result['receivedData'] += event['args']['encodedDataLength'];
}
if (ph === 'B' && name === _MARK_NAME_FRAME_CAPTURE) {
if (frameCaptureStartEvent) {
throw new Error('can capture frames only once per benchmark run');
}
if (!this._captureFrames) {
throw new Error('found start event for frame capture, but frame capture was not requested in benchpress');
}
frameCaptureStartEvent = event;
}
else if (ph === 'E' && name === _MARK_NAME_FRAME_CAPTURE) {
if (!frameCaptureStartEvent) {
throw new Error('missing start event for frame capture');
}
frameCaptureEndEvent = event;
}
if (ph === 'I' && frameCaptureStartEvent && !frameCaptureEndEvent && name === 'frame') {
frameTimestamps.push(event['ts']);
if (frameTimestamps.length >= 2) {
frameTimes.push(frameTimestamps[frameTimestamps.length - 1] -
frameTimestamps[frameTimestamps.length - 2]);
}
}
if (ph === 'B') {
if (!intervalStarts[name]) {
intervalStartCount[name] = 1;
intervalStarts[name] = event;
}
else {
intervalStartCount[name]++;
}
}
else if ((ph === 'E') && intervalStarts[name]) {
intervalStartCount[name]--;
if (intervalStartCount[name] === 0) {
const startEvent = intervalStarts[name];
const duration = (event['ts'] - startEvent['ts']);
intervalStarts[name] = null;
if (name === 'gc') {
result['gcTime'] += duration;
const amount = (startEvent['args']['usedHeapSize'] - event['args']['usedHeapSize']) / 1000;
result['gcAmount'] += amount;
const majorGc = event['args']['majorGc'];
if (majorGc && majorGc) {
result['majorGcTime'] += duration;
}
if (intervalStarts['script']) {
gcTimeInScript += duration;
}
}
else if (name === 'render') {
result['renderTime'] += duration;
if (intervalStarts['script']) {
renderTimeInScript += duration;
}
}
else if (name === 'script') {
result['scriptTime'] += duration;
}
else if (this._microMetrics[name]) {
result[name] += duration / microIterations;
}
}
}
});
if (frameCaptureStartEvent && !frameCaptureEndEvent) {
throw new Error('missing end event for frame capture');
}
if (this._captureFrames && !frameCaptureStartEvent) {
throw new Error('frame capture requested in benchpress, but no start event was found');
}
if (frameTimes.length > 0) {
this._addFrameMetrics(result, frameTimes);
}
result['pureScriptTime'] = result['scriptTime'] - gcTimeInScript - renderTimeInScript;
return result;
}
_addFrameMetrics(result, frameTimes) {
result['frameTime.mean'] = frameTimes.reduce((a, b) => a + b, 0) / frameTimes.length;
const firstFrame = frameTimes[0];
result['frameTime.worst'] = frameTimes.reduce((a, b) => a > b ? a : b, firstFrame);
result['frameTime.best'] = frameTimes.reduce((a, b) => a < b ? a : b, firstFrame);
result['frameTime.smooth'] =
frameTimes.filter(t => t < _FRAME_TIME_SMOOTH_THRESHOLD).length / frameTimes.length;
}
_markName(index) {
return `${_MARK_NAME_PREFIX}${index}`;
}
};
PerflogMetric.SET_TIMEOUT = new InjectionToken('PerflogMetric.setTimeout');
PerflogMetric.IGNORE_NAVIGATION = new InjectionToken('PerflogMetric.ignoreNavigation');
PerflogMetric.PROVIDERS = [
{
provide: PerflogMetric_1,
deps: [
WebDriverExtension, PerflogMetric_1.SET_TIMEOUT, Options.MICRO_METRICS, Options.FORCE_GC,
Options.CAPTURE_FRAMES, Options.RECEIVED_DATA, Options.REQUEST_COUNT,
PerflogMetric_1.IGNORE_NAVIGATION
]
},
{
provide: PerflogMetric_1.SET_TIMEOUT,
useValue: (fn, millis) => setTimeout(fn, millis)
},
{ provide: PerflogMetric_1.IGNORE_NAVIGATION, useValue: false }
];
PerflogMetric = PerflogMetric_1 = __decorate([
Injectable(),
__param(1, Inject(PerflogMetric_1.SET_TIMEOUT)),
__param(2, Inject(Options.MICRO_METRICS)),
__param(3, Inject(Options.FORCE_GC)),
__param(4, Inject(Options.CAPTURE_FRAMES)),
__param(5, Inject(Options.RECEIVED_DATA)),
__param(6, Inject(Options.REQUEST_COUNT)),
__param(7, Inject(PerflogMetric_1.IGNORE_NAVIGATION)),
__metadata("design:paramtypes", [WebDriverExtension,
Function, Object, Boolean, Boolean, Boolean, Boolean, Boolean])
], PerflogMetric);
const _MICRO_ITERATIONS_REGEX = /(.+)\*(\d+)$/;
const _MAX_RETRY_COUNT = 20;
const _MARK_NAME_PREFIX = 'benchpress';
const _MARK_NAME_FRAME_CAPTURE = 'frameCapture';
// using 17ms as a somewhat looser threshold, instead of 16.6666ms
const _FRAME_TIME_SMOOTH_THRESHOLD = 17;
/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
/**
* A WebDriverAdapter bridges API differences between different WebDriver clients,
* e.g. JS vs Dart Async vs Dart Sync webdriver.
* Needs one implementation for every supported WebDriver client.
*/
class WebDriverAdapter {
waitFor(callback) {
throw new Error('NYI');
}
executeScript(script) {
throw new Error('NYI');
}
executeAsyncScript(script) {
throw new Error('NYI');
}
capabilities() {
throw new Error('NYI');
}
logs(type) {
throw new Error('NYI');
}
}
/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
var UserMetric_1;
let UserMetric = UserMetric_1 = class UserMetric extends Metric {
constructor(_userMetrics, _wdAdapter) {
super();
this._userMetrics = _userMetrics;
this._wdAdapter = _wdAdapter;
}
/**
* Starts measuring
*/
beginMeasure() {
return Promise.resolve(true);
}
/**
* Ends measuring.
*/
endMeasure(restart) {
let resolve;
let reject;
const promise = new Promise((res, rej) => {
resolve = res;
reject = rej;
});
const adapter = this._wdAdapter;
const names = Object.keys(this._userMetrics);
function getAndClearValues() {
Promise.all(names.map(name => adapter.executeScript(`return window.${name}`)))
.then((values) => {
if (values.every(v => typeof v === 'number')) {
Promise.all(names.map(name => adapter.executeScript(`delete window.${name}`)))
.then((_) => {
const map = {};
for (let i = 0, n = names.length; i < n; i++) {
map[names[i]] = values[i];
}
resolve(map);
}, reject);
}
else {
setTimeout(getAndClearValues, 100);
}
}, reject);
}
getAndClearValues();
return promise;
}
/**
* Describes the metrics provided by this metric implementation.
* (e.g. units, ...)
*/
describe() {
return this._userMetrics;
}
};
UserMetric.PROVIDERS = [{ provide: UserMetric_1, deps: [Options.USER_METRICS, WebDriverAdapter] }];
UserMetric = UserMetric_1 = __decorate([
Injectable(),
__param(0, Inject(Options.USER_METRICS)),
__metadata("design:paramtypes", [Object, WebDriverAdapter])
], UserMetric);
/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
/**
* A reporter reports measure values and the valid sample.
*/
class Reporter {
reportMeasureValues(values) {
throw new Error('NYI');
}
reportSample(completeSample, validSample) {
throw new Error('NYI');
}
}
/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
/**
* A Validator calculates a valid sample out of the complete sample.
* A valid sample is a sample that represents the population that should be observed
* in the correct way.
*/
class Validator {
/**
* Calculates a valid sample out of the complete sample
*/
validate(completeSample) {
throw new Error('NYI');
}
/**
* Returns a Map that describes the properties of the validator
* (e.g. sample size, ...)
*/
describe() {
throw new Error('NYI');
}
}
/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
/**
* SampleDescription merges all available descriptions about a sample
*/
class SampleDescription {
constructor(id, descriptions, metrics) {
this.id = id;
this.metrics = metrics;
this.description = {};
descriptions.forEach(description => {
Object.keys(description).forEach(prop => {
this.description[prop] = description[prop];
});
});
}
toJson() {
return { 'id': this.id, 'description': this.description, 'metrics': this.metrics };
}
}
SampleDescription.PROVIDERS = [{
provide: SampleDescription,
useFactory: (metric, id, forceGc, userAgent, validator, defaultDesc, userDesc) => new SampleDescription(id, [
{ 'forceGc': forceGc, 'userAgent': userAgent }, validator.describe(), defaultDesc,
userDesc
], metric.describe()),
deps: [
Metric, Options.SAMPLE_ID, Options.FORCE_GC, Options.USER_AGENT, Validator,
Options.DEFAULT_DESCRIPTION, Options.SAMPLE_DESCRIPTION
]
}];
/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
class Statistic {
static calculateCoefficientOfVariation(sample, mean) {
return Statistic.calculateStandardDeviation(sample, mean) / mean * 100;
}
static calculateMean(samples) {
let total = 0;
// TODO: use reduce
samples.forEach(x => total += x);
return total / samples.length;
}
static calculateStandardDeviation(samples, mean) {
let deviation = 0;
// TODO: use reduce
samples.forEach(x => deviation += Math.pow(x - mean, 2));
deviation = deviation / (samples.length);
deviation = Math.sqrt(deviation);
return deviation;
}
static calculateRegressionSlope(xValues, xMean, yValues, yMean) {
// See https://en.wikipedia.org/wiki/Simple_linear_regression
let dividendSum = 0;
let divisorSum = 0;
for (let i = 0; i < xValues.length; i++) {
dividendSum += (xValues[i] - xMean) * (yValues[i] - yMean);
divisorSum += Math.pow(xValues[i] - xMean, 2);
}
return dividendSum / divisorSum;
}
}
/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
function formatNum(n) {
return n.toFixed(2);
}
function sortedProps(obj) {
return Object.keys(obj).sort();
}
function formatStats(validSamples, metricName) {
const samples = validSamples.map(measureValues => measureValues.values[metricName]);
const mean = Statistic.calculateMean(samples);
const cv = Statistic.calculateCoefficientOfVariation(samples, mean);
const formattedMean = formatNum(mean);
// Note: Don't use the unicode character for +- as it might cause
// hickups for consoles...
return isNaN(cv) ? formattedMean : `${formattedMean}+-${Math.floor(cv)}%`;
}
/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
var ConsoleReporter_1;
/**
* A reporter for the console
*/
let ConsoleReporter = ConsoleReporter_1 = class ConsoleReporter extends Reporter {
constructor(_columnWidth, sampleDescription, _print) {
super();
this._columnWidth = _columnWidth;
this._print = _print;
this._metricNames = sortedProps(sampleDescription.metrics);
this._printDescription(sampleDescription);
}
static _lpad(value, columnWidth, fill = ' ') {
let result = '';
for (let i = 0; i < columnWidth - value.length; i++) {
result += fill;
}
return result + value;
}
_printDescription(sampleDescription) {
this._print(`BENCHMARK ${sampleDescription.id}`);
this._print('Description:');
const props = sortedProps(sampleDescription.description);
props.forEach((prop) => {
this._print(`- ${prop}: ${sampleDescription.description[prop]}`);
});
this._print('Metrics:');
this._metricNames.forEach((metricName) => {
this._print(`- ${metricName}: ${sampleDescription.metrics[metricName]}`);
});
this._print('');
this._printStringRow(this._metricNames);
this._printStringRow(this._metricNames.map((_) => ''), '-');
}
reportMeasureValues(measureValues) {
const formattedValues = this._metricNames.map(metricName => {
const value = measureValues.values[metricName];
return formatNum(value);
});
this._printStringRow(formattedValues);
return Promise.resolve(null);
}
reportSample(completeSample, validSamples) {
this._printStringRow(this._metricNames.map((_) => ''), '=');
this._printStringRow(this._metricNames.map(metricName => formatStats(validSamples, metricName)));
return Promise.resolve(null);
}
_printStringRow(parts, fill = ' ') {
this._print(parts.map(part => ConsoleReporter_1._lpad(part, this._columnWidth, fill)).join(' | '));
}
};
ConsoleReporter.PRINT = new InjectionToken('ConsoleReporter.print');
ConsoleReporter.COLUMN_WIDTH = new InjectionToken('ConsoleReporter.columnWidth');
ConsoleReporter.PROVIDERS = [
{
provide: ConsoleReporter_1,
deps: [ConsoleReporter_1.COLUMN_WIDTH, SampleDescription, ConsoleReporter_1.PRINT]
},
{ provide: ConsoleReporter_1.COLUMN_WIDTH, useValue: 18 }, {
provide: ConsoleReporter_1.PRINT,
useValue: function (v) {
// tslint:disable-next-line:no-console
console.log(v);
}
}
];
ConsoleReporter = ConsoleReporter_1 = __decorate([
Injectable(),
__param(0, Inject(ConsoleReporter_1.COLUMN_WIDTH)),
__param(2, Inject(ConsoleReporter_1.PRINT)),
__metadata("design:paramtypes", [Number, SampleDescription,
Function])
], ConsoleReporter);
/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
var JsonFileReporter_1;
/**
* A reporter that writes results into a json file.
*/
let JsonFileReporter = JsonFileReporter_1 = class JsonFileReporter extends Reporter {
constructor(_description, _path, _writeFile, _now) {
super();
this._description = _description;
this._path = _path;
this._writeFile = _writeFile;
this._now = _now;
}
reportMeasureValues(measureValues) {
return Promise.resolve(null);
}
reportSample(completeSample, validSample) {
const stats = {};
sortedProps(this._description.metrics).forEach((metricName) => {
stats[metricName] = formatStats(validSample, metricName);
});
const content = JSON.stringify({
'description': this._description,
'stats': stats,
'completeSample': completeSample,
'validSample': validSample,
}, null, 2);
const filePath = `${this._path}/${this._description.id}_${this._now().getTime()}.json`;
return this._writeFile(filePath, content);
}
};
JsonFileReporter.PATH = new InjectionToken('JsonFileReporter.path');
JsonFileReporter.PROVIDERS = [
{
provide: JsonFileReporter_1,
deps: [SampleDescription, JsonFileReporter_1.PATH, Options.WRITE_FILE, Options.NOW]
},
{ provide: JsonFileReporter_1.PATH, useValue: '.' }
];
JsonFileReporter = JsonFileReporter_1 = __decorate([
Injectable(),
__param(1, Inject(JsonFileReporter_1.PATH)),
__param(2, Inject(Options.WRITE_FILE)),
__param(3, Inject(Options.NOW)),
__metadata("design:paramtypes", [SampleDescription, String, Function,
Function])
], JsonFileReporter);
/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
class MultiReporter extends Reporter {
constructor(_reporters) {
super();
this._reporters = _reporters;
}
static provideWith(childTokens) {
return [
{
provide: _CHILDREN,
useFactory: (injector) => childTokens.map(token => injector.get(token)),
deps: [Injector],
},
{
provide: MultiReporter,
useFactory: (children) => new MultiReporter(children),
deps: [_CHILDREN]
}
];
}
reportMeasureValues(values) {
return Promise.all(this._reporters.map(reporter => reporter.reportMeasureValues(values)));
}
reportSample(completeSample, validSample) {
return Promise.all(this._reporters.map(reporter => reporter.reportSample(completeSample, validSample)));
}
}
const _CHILDREN = new InjectionToken('MultiReporter.children');
/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
var Sampler_1;
/**
* The Sampler owns the sample loop:
* 1. calls the prepare/execute callbacks,
* 2. gets data from the metric
* 3. asks the validator for a valid sample
* 4. reports the new data to the reporter
* 5. loop until there is a valid sample
*/
let Sampler = Sampler_1 = class Sampler {
constructor(_driver, _metric, _reporter, _validator, _prepare, _execute, _now) {
this._driver = _driver;
this._metric = _metric;
this._reporter = _reporter;
this._validator = _validator;
this._prepare = _prepare;
this._execute = _execute;
this._now = _now;
}
sample() {
const loop = (lastState) => {
return this._iterate(lastState).then((newState) => {
if (newState.validSample != null) {
return newState;
}
else {
return loop(newState);
}
});
};
return loop(new SampleState([], null));
}
_iterate(lastState) {
let resultPromise;
if (this._prepare !== Options.NO_PREPARE) {
resultPromise = this._driver.waitFor(this._prepare);
}
else {
resultPromise = Promise.resolve(null);
}
if (this._prepare !== Options.NO_PREPARE || lastState.completeSample.length === 0) {
resultPromise = resultPromise.then((_) => this._metric.beginMeasure());
}
return resultPromise.then((_) => this._driver.waitFor(this._execute))
.then((_) => this._metric.endMeasure(this._prepare === Options.NO_PREPARE))
.then((measureValues) => {
if (!!measureValues['invalid']) {
return lastState;
}
return this._report(lastState, measureValues);
});
}
_report(state, metricValues) {
const measureValues = new MeasureValues(state.completeSample.length, this._now(), metricValues);
const completeSample = state.completeSample.concat([measureValues]);
const validSample = this._validator.validate(completeSample);
let resultPromise = this._reporter.reportMeasureValues(measureValues);
if (validSample != null) {
resultPromise =
resultPromise.then((_) => this._reporter.reportSample(completeSample, validSample));
}
return resultPromise.then((_) => new SampleState(completeSample, validSample));
}
};
Sampler.PROVIDERS = [{
provide: Sampler_1,
deps: [
WebDriverAdapter, Metric, Reporter, Validator, Options.PREPARE, Options.EXECUTE,
Options.NOW
]
}];
Sampler = Sampler_1 = __decorate([
Injectable(),
__param(4, Inject(Options.PREPARE)),
__param(5, Inject(Options.EXECUTE)),
__param(6, Inject(Options.NOW)),
__metadata("design:paramtypes", [WebDriverAdapter, Metric, Reporter,
Validator, Function,
Function,
Function])
], Sampler);
class SampleState {
constructor(completeSample, validSample) {
this.completeSample = completeSample;
this.validSample = validSample;
}
}
/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
var RegressionSlopeValidator_1;
/**
* A validator that checks the regression slope of a specific metric.
* Waits for the regression slope to be >=0.
*/
let RegressionSlopeValidator = RegressionSlopeValidator_1 = class RegressionSlopeValidator extends Validator {
constructor(_sampleSize, _metric) {
super();
this._sampleSize = _sampleSize;
this._metric = _metric;
}
describe() {
return { 'sampleSize': this._sampleSize, 'regressionSlopeMetric': this._metric };
}
validate(completeSample) {
if (completeSample.length >= this._sampleSize) {
const latestSample = completeSample.slice(completeSample.length - this._sampleSize, completeSample.length);
const xValues = [];
const yValues = [];
for (let i = 0; i < latestSample.length; i++) {
// For now, we only use the array index as x value.
// TODO(tbosch): think about whether we should use time here instead
xValues.push(i);
yValues.push(latestSample[i].values[this._metric]);
}
const regressionSlope = Statistic.calculateRegressionSlope(xValues, Statistic.calculateMean(xValues), yValues, Statistic.calculateMean(yValues));
return regressionSlope >= 0 ? latestSample : null;
}
else {
return null;
}
}
};
RegressionSlopeValidator.SAMPLE_SIZE = new InjectionToken('RegressionSlopeValidator.sampleSize');
RegressionSlopeValidator.METRIC = new InjectionToken('RegressionSlopeValidator.metric');
RegressionSlopeValidator.PROVIDERS = [
{
provide: RegressionSlopeValidator_1,
deps: [RegressionSlopeValidator_1.SAMPLE_SIZE, RegressionSlopeValidator_1.METRIC]
},
{ provide: RegressionSlopeValidator_1.SAMPLE_SIZE, useValue: 10 },
{ provide: RegressionSlopeValidator_1.METRIC, useValue: 'scriptTime' }
];
RegressionSlopeValidator = RegressionSlopeValidator_1 = __decorate([
Injectable(),
__param(0, Inject(RegressionSlopeValidator_1.SAMPLE_SIZE)),
__param(1, Inject(RegressionSlopeValidator_1.METRIC)),
__metadata("design:paramtypes", [Number, String])
], RegressionSlopeValidator);
/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
var SizeValidator_1;
/**
* A validator that waits for the sample to have a certain size.
*/
let SizeValidator = SizeValidator_1 = class SizeValidator extends Validator {
constructor(_sampleSize) {
super();
this._sampleSize = _sampleSize;
}
describe() {
return { 'sampleSize': this._sampleSize };
}
validate(completeSample) {
if (completeSample.length >= this._sampleSize) {
return completeSample.slice(completeSample.length - this._sampleSize, completeSample.length);
}
else {
return null;
}
}
};
SizeValidator.SAMPLE_SIZE = new InjectionToken('SizeValidator.sampleSize');
SizeValidator.PROVIDERS = [
{ provide: SizeValidator_1, deps: [SizeValidator_1.SAMPLE_SIZE] },
{ provide: SizeValidator_1.SAMPLE_SIZE, useValue: 10 }
];
SizeValidator = SizeValidator_1 = __decorate([
Injectable(),
__param(0, Inject(SizeValidator_1.SAMPLE_SIZE)),
__metadata("design:paramtypes", [Number])
], SizeValidator);
/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
var ChromeDriverExtension_1;
/**
* Set the following 'traceCategories' to collect metrics in Chrome:
* 'v8,blink.console,disabled-by-default-devtools.timeline,devtools.timeline,blink.user_timing'
*
* In order to collect the frame rate related metrics, add 'benchmark'
* to the list above.
*/
let ChromeDriverExtension = ChromeDriverExtension_1 = class ChromeDriverExtension extends WebDriverExtension {
constructor(driver, userAgent, rawPerflogPath) {
super();
this.driver = driver;
this._firstRun = true;
this._majorChromeVersion = this._parseChromeVersion(userAgent);
this._rawPerflogPath = rawPerflogPath;
}
_parseChromeVersion(userAgent) {
if (!userAgent) {
return -1;
}
let v = userAgent.split(/Chrom(e|ium)\//g)[2];
if (!v) {
return -1;
}
v = v.split('.')[0];
if (!v) {
return -1;
}
return parseInt(v, 10);
}
gc() {
return this.driver.executeScript('window.gc()');
}
async timeBegin(name) {
if (this._firstRun) {
this._firstRun = false;
// Before the first run, read out the existing performance logs
// so that the chrome buffer does not fill up.
await this.driver.logs('performance');
}
return this.driver.executeScript(`performance.mark('${name}-bpstart');`);
}
timeEnd(name, restartName = null) {
let script = `performance.mark('${name}-bpend');`;
if (restartName) {
script += `performance.mark('${restartName}-bpstart');`;
}
return this.driver.executeScript(script);
}
// See [Chrome Trace Event
// Format](https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/edit)
readPerfLog() {
// TODO(tbosch): Chromedriver bug https://code.google.com/p/chromedriver/issues/detail?id=1098
// Need to execute at least one command so that the browser logs can be read out!
return this.driver.executeScript('1+1')
.then((_) => this.driver.logs('performance'))
.then((entries) => {
const events = [];
entries.forEach((entry) => {
const message = JSON.parse(entry['message'])['message'];
if (message['method'] === 'Tracing.dataCollected') {
events.push(message['params']);
}
if (message['method'] === 'Tracing.bufferUsage') {
throw new Error('The DevTools trace buffer filled during the test!');
}
});
if (this._rawPerflogPath && events.length) {
fs.appendFileSync(this._rawPerflogPath, JSON.stringify(events));
}
return this._convertPerfRecordsToEvents(events);
});
}
_convertPerfRecordsToEvents(chromeEvents, normalizedEvents = null) {
if (!normalizedEvents) {
normalizedEvents = [];
}
chromeEvents.forEach((event) => {
const categories = this._parseCategories(event['cat']);
const normalizedEvent = this._convertEvent(event, categories);
if (normalizedEvent != null)
normalizedEvents.push(normalizedEvent);
});
return normalizedEvents;
}
_convertEvent(event, categories) {
const name = event['name'];
const args = event['args'];
if (this._isEvent(categories, name, ['blink.console'])) {
return normalizeEvent(event, { 'name': name });
}
else if (this._isEvent(categories, name, ['blink.user_timing'])) {
return normalizeEvent(event, { 'name': name });
}
else if (this._isEvent(categories, name, ['benchmark'], 'BenchmarkInstrumentation::ImplThreadRenderingStats')) {
// TODO(goderbauer): Instead of BenchmarkInstrumentation::ImplThreadRenderingStats the
// following events should be used (if available) for more accurate measurements:
// 1st choice: vsync_before - ground truth on Android
// 2nd choice: BenchmarkInstrumentation::DisplayRenderingStats