playwright-performance-reporter
Version:
Measure and publish performance metrics from browser dev-tools when running playwright
240 lines (239 loc) • 10.3 kB
JavaScript
import { HookOrderToMetricOrder, HookOrderToMeasurementOrder, HookOrderToMeasurementOffsetOrder, testCaseParent, } from '../types/index.js';
import { buildTestCaseIdentifier, buildTestPerformance, buildTestStepIdentifier, JsonChunkWriter, Logger, } from '../helpers/index.js';
import { MetricsEngine } from './index.js';
export class PerformanceReporter {
options;
/**
* Maps unique playwright test ids to the computed name
*/
idToNameMapping = new Map();
/*
* Metrics engine to retrieve metrics from a browser
*/
metricsEngine;
/**
* Writer to stream json chunks
*/
jsonChunkWriter;
/**
* Reference to the unsubscribe function for a hook and test id
*/
samplingRunner = new Map([
['onTest', new Map()],
['onTestStep', new Map()],
]);
constructor(options) {
this.options = options;
this.metricsEngine = new MetricsEngine();
this.jsonChunkWriter = options.customJsonWriter ? options.customJsonWriter : new JsonChunkWriter();
this.jsonChunkWriter.initialize(this.options);
}
onBegin(config, suite) { }
onEnd(result) {
try {
this.jsonChunkWriter.close();
}
catch (error) {
Logger.error('Error writing json report', String(error));
return;
}
if (result.status !== 'passed' && this.options.deleteOnFailure) {
this.jsonChunkWriter.delete();
Logger.info('Test failed and file deleted', this.options.outputDir, this.options.outputFile);
}
else {
Logger.info('Successfully written to json', this.options.outputDir, this.options.outputFile);
}
}
async onTestBegin(test, result) {
const browserDetails = test.parent.project()?.use;
if (browserDetails) {
try {
await this.metricsEngine.setupBrowser(browserDetails.defaultBrowserType, browserDetails);
}
catch { }
}
const browserName = this.metricsEngine.getBrowser();
if (!browserName) {
// Browser not stable
return;
}
const { id, name } = buildTestCaseIdentifier(test);
if (id === '') {
// Root of test suite
return;
}
const results = this.createTestPerformance(id, testCaseParent, name);
await this.executeMetrics(results, id, testCaseParent, 'onTest', 'onStart', browserName);
await this.executeMetrics(results, id, testCaseParent, 'onTest', 'onSampling', browserName);
this.jsonChunkWriter.write(results);
}
async onTestEnd(test, result) {
const browserName = this.metricsEngine.getBrowser();
if (!browserName) {
// Browser not stable
return;
}
const { id, name } = buildTestCaseIdentifier(test);
if (id === '') {
// Root of test suite
return;
}
this.destroySamplingRunner(id, testCaseParent, 'onTest');
const results = this.createTestPerformance(id, testCaseParent, name);
await this.executeMetrics(results, id, testCaseParent, 'onTest', 'onStop', browserName);
this.metricsEngine.destroy();
this.jsonChunkWriter.write(results);
}
async onStepBegin(test, result, step) {
const browserName = this.metricsEngine.getBrowser();
if (!browserName) {
// Browser not stable
return;
}
if (step.category !== 'test.step') {
return;
}
const caseIdentifier = buildTestCaseIdentifier(test);
const stepIdentifier = buildTestStepIdentifier(step);
if (stepIdentifier.name === '') {
// Root of test suite
return;
}
const results = this.createTestPerformance(caseIdentifier.id, stepIdentifier.id, stepIdentifier.name);
await this.executeMetrics(results, caseIdentifier.id, stepIdentifier.id, 'onTestStep', 'onStart', browserName);
await this.executeMetrics(results, caseIdentifier.id, stepIdentifier.id, 'onTestStep', 'onSampling', browserName);
this.jsonChunkWriter.write(results);
}
async onStepEnd(test, result, step) {
const browserName = this.metricsEngine.getBrowser();
if (!browserName) {
// Browser not stable
return;
}
if (step.category !== 'test.step') {
return;
}
const caseIdentifier = buildTestCaseIdentifier(test);
const stepIdentifier = buildTestStepIdentifier(step);
if (stepIdentifier.name === '') {
// Root of test suite
return;
}
this.destroySamplingRunner(caseIdentifier.id, stepIdentifier.id, 'onTestStep');
const results = this.createTestPerformance(caseIdentifier.id, stepIdentifier.id, stepIdentifier.name);
await this.executeMetrics(results, caseIdentifier.id, stepIdentifier.id, 'onTestStep', 'onStop', browserName);
this.jsonChunkWriter.write(results);
}
/**
* Add new `TestPerformance` with identifier
*
* @param caseId identifier for case
* @param stepId identifier for step
* @param name human friendly identifier
*/
createTestPerformance(caseId, stepId, name) {
this.idToNameMapping.set(caseId, name);
return {
[caseId]: {
[stepId]: buildTestPerformance(name),
},
};
}
/**
* Destroys sampling for metrics
*
* @param caseId identifier for case
* @param stepId identifier for step
* @param hook playwright hook
*/
destroySamplingRunner(caseId, stepId, hook) {
this.samplingRunner.get(hook)?.get(caseId + stepId)?.call(this);
this.samplingRunner.get(hook)?.delete(caseId + stepId);
}
/**
* Execute metrics gathering for existing metrics and custom metrics
*
* @param results result accumulator
* @param caseId identifier for case
* @param stepId identifier for step
* @param hook playwright hook
* @param hookOrder playwright hook order
* @param browser which settings and metrics to use
*/
async executeMetrics(results, caseId, stepId, hook, hookOrder, browser) {
if (hookOrder === 'onSampling') {
await this.executeSamplingMetrics(results, caseId, stepId, hook, browser);
return;
}
if (this.options.browsers[browser]?.[hook]?.metrics.length) {
Logger.info('Fetching predefined metrics', ...this.options.browsers[browser][hook].metrics);
}
const startOfTrigger = Date.now();
const metrics = Promise.all(this.options.browsers[browser]?.[hook]?.metrics.map(async (metric) => this.metricsEngine.getMetric(metric, hookOrder)) ?? []);
if (this.options.browsers[browser]?.[hook]?.customMetrics) {
Logger.info('Fetching custom metrics', ...Object.values(this.options.browsers[browser][hook].customMetrics).map(({ name }) => name));
}
const customMetrics = Promise.all(Object.values(this.options.browsers[browser]?.[hook]?.customMetrics ?? {})
.map(async (customMetric) => this.metricsEngine.runCustomMetric(customMetric, hookOrder)) || []);
const resolvedMetrics = await metrics;
const resolvedCustomMetrics = await customMetrics;
const endOfTrigger = Date.now();
results[caseId][stepId][HookOrderToMeasurementOrder[hookOrder]] = endOfTrigger;
results[caseId][stepId][HookOrderToMeasurementOffsetOrder[hookOrder]] = endOfTrigger - startOfTrigger;
results[caseId][stepId][HookOrderToMetricOrder[hookOrder]].push(...resolvedMetrics.filter(m => m !== undefined).flat(), ...resolvedCustomMetrics.filter(m => m !== undefined).flat());
}
/**
* Setup sampling for existing metrics and custom metrics
*
* @param results result accumulator
* @param caseId identifier for case
* @param stepId identifier for step
* @param hook playwright hook
* @param browser which settings and metrics to use
*/
async executeSamplingMetrics(results, caseId, stepId, hook, browser) {
const sampleMetrics = this.options.browsers[browser]?.[hook]?.sampleMetrics;
if (!sampleMetrics) {
return;
}
const samplingArguments = Object.entries(sampleMetrics).map(([metricName, metricSampling]) => {
const registeredMetrics = this.options.browsers[browser]?.[hook]?.metrics;
const customMetrics = this.options.browsers[browser]?.[hook]?.customMetrics;
const isPredefinedMetric = (registeredMetrics ?? []).find(registeredMetric => registeredMetric === metricName);
const isCustomMetric = Object.values(customMetrics ?? {}).find(value => value.name === metricName);
if (registeredMetrics && isPredefinedMetric) {
return [
async () => {
const metricsResponse = await this.metricsEngine.getMetric(metricName, 'onSampling');
if (metricsResponse) {
results[caseId][stepId].samplingMetrics.push(...metricsResponse);
}
},
metricSampling.samplingTimeoutInMilliseconds,
];
}
if (customMetrics && isCustomMetric) {
return [
async () => {
const metricsResponse = await this.metricsEngine.runCustomMetric(customMetrics[metricName], 'onSampling');
if (metricsResponse) {
results[caseId][stepId].samplingMetrics.push(...metricsResponse);
}
},
metricSampling.samplingTimeoutInMilliseconds,
];
}
return undefined;
});
for (const sampling of samplingArguments) {
if (!sampling) {
continue;
}
const interval = setInterval(...sampling);
this.samplingRunner.get(hook)?.set(caseId + stepId, () => {
clearInterval(interval);
});
}
}
}