playwright-performance-reporter
Version:
Measure and publish performance metrics from browser dev-tools when running playwright
275 lines (274 loc) • 10.9 kB
JavaScript
import { HookOrderToMetricOrder, HookOrderToMeasurementOrder, HookOrderToMeasurementOffsetOrder, testCaseParent, } from '../types/index.js';
import { buildTestCaseIdentifier, buildTestPerformance, buildTestStepIdentifier, Logger, } from '../helpers/index.js';
import { nativePresenters, } from '../presenters/index.js';
import { MetricsEngine, } from './index.js';
export class PerformanceReporter {
options;
/**
* Maps unique playwright test ids to the computed name
*/
idToNameMapping = new Map();
/*
* Metrics engine to retrieve metrics from a browser
*/
metricsEngine;
/**
* Writers to stream data to different outputs
*/
presenters;
/**
* Latest test step identifier
*/
latestStepId;
/**
* Latest case identifier
*/
latestCaseId;
/**
* Latest test name
*/
latestName;
/**
* Reference to the unsubscribe function for a hook and test id
*/
samplingRunner = new Map([
['onTest', new Map()],
['onTestStep', new Map()],
]);
constructor(options) {
this.options = options;
this.metricsEngine = new MetricsEngine();
this.presenters = options.presenters && options.presenters.length > 0
? options.presenters
: [
new nativePresenters.jsonChunkPresenter({
outputDir: './',
outputFile: `performance-report-${Date.now()}.json`,
}),
];
}
onBegin(config, suite) { }
async onEnd(result) {
let closeStatus;
try {
closeStatus = await this.closePresenters();
}
catch {
closeStatus = false;
}
if (!closeStatus) {
Logger.error('Error writing report');
return;
}
if (result.status !== 'passed' && this.options.deleteOnFailure) {
await this.deleteFromPresenters();
Logger.info('Test failed and file deleted');
}
else {
Logger.info('Successfully closed presenters');
}
}
async onTestBegin(test, result) {
const browserDetails = test.parent.project()?.use;
if (browserDetails && browserDetails.defaultBrowserType) {
try {
await this.metricsEngine.setupBrowser(browserDetails.defaultBrowserType, browserDetails);
}
catch { }
}
const browserName = this.metricsEngine.getBrowser();
if (!browserName) {
// Browser not stable
return;
}
const { id, name } = buildTestCaseIdentifier(test);
if (id === '') {
// Root of test suite
return;
}
this.latestCaseId = id;
this.latestStepId = testCaseParent;
this.latestName = name;
const results = this.createTestPerformance(id, testCaseParent, name);
await this.executeMetrics(results, id, testCaseParent, 'onTest', 'onStart', browserName);
await this.writeToPresenters(results);
const samplingResults = this.createTestPerformance(id, testCaseParent, name);
await this.executeMetrics(samplingResults, id, testCaseParent, 'onTest', 'onSampling', browserName);
}
async onTestEnd(test, result) {
const browserName = this.metricsEngine.getBrowser();
if (!browserName) {
// Browser not stable
return;
}
const { id, name } = buildTestCaseIdentifier(test);
if (id === '') {
// Root of test suite
return;
}
this.destroySamplingRunner(id, testCaseParent, 'onTest');
const results = this.createTestPerformance(id, testCaseParent, name);
await this.executeMetrics(results, id, testCaseParent, 'onTest', 'onStop', browserName);
this.metricsEngine.destroy();
await this.writeToPresenters(results);
}
async onStepBegin(test, result, step) {
const browserName = this.metricsEngine.getBrowser();
if (!browserName) {
// Browser not stable
return;
}
if (step.category !== 'test.step') {
return;
}
const caseIdentifier = buildTestCaseIdentifier(test);
const stepIdentifier = buildTestStepIdentifier(step);
if (stepIdentifier.name === '') {
// Root of test suite
return;
}
this.latestCaseId = caseIdentifier.id;
this.latestStepId = stepIdentifier.id;
this.latestName = stepIdentifier.name;
const results = this.createTestPerformance(caseIdentifier.id, stepIdentifier.id, stepIdentifier.name);
await this.executeMetrics(results, caseIdentifier.id, stepIdentifier.id, 'onTestStep', 'onStart', browserName);
await this.writeToPresenters(results);
const samplingResults = this.createTestPerformance(caseIdentifier.id, stepIdentifier.id, stepIdentifier.name);
await this.executeMetrics(samplingResults, caseIdentifier.id, stepIdentifier.id, 'onTestStep', 'onSampling', browserName);
}
async onStepEnd(test, result, step) {
const browserName = this.metricsEngine.getBrowser();
if (!browserName) {
// Browser not stable
return;
}
if (step.category !== 'test.step') {
return;
}
const caseIdentifier = buildTestCaseIdentifier(test);
const stepIdentifier = buildTestStepIdentifier(step);
if (stepIdentifier.name === '') {
// Root of test suite
return;
}
this.destroySamplingRunner(caseIdentifier.id, stepIdentifier.id, 'onTestStep');
const results = this.createTestPerformance(caseIdentifier.id, stepIdentifier.id, stepIdentifier.name);
await this.executeMetrics(results, caseIdentifier.id, stepIdentifier.id, 'onTestStep', 'onStop', browserName);
await this.writeToPresenters(results);
}
/**
* Write data to all presenters concurrently
*
* @param content the content to write
*/
async writeToPresenters(content) {
await Promise.allSettled(this.presenters.map(async (presenter) => presenter.write(content)));
}
/**
* Close all presenters
*/
async closePresenters() {
const status = await Promise.allSettled(this.presenters.map(async (presenter) => presenter.close()));
return status.every(result => result.status === 'fulfilled');
}
/**
* Delete files from all presenters
*/
async deleteFromPresenters() {
await Promise.allSettled(this.presenters.map(async (presenter) => presenter.delete()));
}
/**
* Add new `TestPerformance` with identifier
*
* @param caseId identifier for case
* @param stepId identifier for step
* @param name human friendly identifier
*/
createTestPerformance(caseId, stepId, name) {
this.idToNameMapping.set(caseId, name);
return {
[caseId]: {
[stepId]: buildTestPerformance(name),
},
};
}
/**
* Destroys sampling for metrics
*
* @param caseId identifier for case
* @param stepId identifier for step
* @param hook playwright hook
*/
destroySamplingRunner(caseId, stepId, hook) {
this.samplingRunner.get(hook)?.get(caseId + stepId)?.call(this);
this.samplingRunner.get(hook)?.delete(caseId + stepId);
}
/**
* Execute metrics gathering for metrics
*
* @param results result accumulator
* @param caseId identifier for case
* @param stepId identifier for step
* @param hook playwright hook
* @param hookOrder playwright hook order
* @param browser which settings and metrics to use
*/
async executeMetrics(results, caseId, stepId, hook, hookOrder, browser) {
if (hookOrder === 'onSampling') {
await this.executeSamplingMetrics(results, caseId, stepId, hook, browser);
return;
}
const metrics = (this.options.browsers[browser]?.[hook]?.metrics ?? []);
if (metrics.length > 0) {
Logger.info('Fetching metrics', ...metrics.map(m => m.name));
}
const startOfTrigger = Date.now();
const metricsPromises = metrics.map(async (metric) => this.metricsEngine.getMetric(metric, hookOrder));
const resolvedMetrics = await Promise.all(metricsPromises);
const endOfTrigger = Date.now();
results[caseId][stepId][HookOrderToMeasurementOrder[hookOrder]] = endOfTrigger;
results[caseId][stepId][HookOrderToMeasurementOffsetOrder[hookOrder]] = endOfTrigger - startOfTrigger;
results[caseId][stepId][HookOrderToMetricOrder[hookOrder]].push(...resolvedMetrics.filter(m => m !== undefined).flat());
}
/**
* Setup sampling for metrics
*
* @param results result accumulator
* @param caseId identifier for case
* @param stepId identifier for step
* @param hook playwright hook
* @param browser which settings and metrics to use
*/
async executeSamplingMetrics(results, caseId, stepId, hook, browser) {
const samplingConfig = this.options.browsers[browser]?.sampling?.metrics;
if (!samplingConfig) {
return;
}
const samplingArguments = samplingConfig.map(samplingItem => ({
callback: async () => {
const startOfTrigger = Date.now();
const metricsResponse = await this.metricsEngine.getMetric(samplingItem.metric, 'onSampling');
const endOfTrigger = Date.now();
if (metricsResponse && this.latestCaseId && this.latestStepId && this.latestName) {
const clonedResults = structuredClone(results);
clonedResults[this.latestCaseId] ||= {};
clonedResults[this.latestCaseId][this.latestStepId] ||= buildTestPerformance(this.latestName);
clonedResults[this.latestCaseId][this.latestStepId][HookOrderToMeasurementOrder.onStop] = endOfTrigger;
clonedResults[this.latestCaseId][this.latestStepId][HookOrderToMeasurementOffsetOrder.onStop] = endOfTrigger - startOfTrigger;
clonedResults[this.latestCaseId][this.latestStepId].samplingMetrics.push(...metricsResponse);
await this.writeToPresenters(clonedResults);
}
},
delay: samplingItem.samplingTimeoutInMilliseconds,
}));
for (const sampling of samplingArguments) {
if (!sampling) {
continue;
}
const interval = setInterval(sampling.callback, sampling.delay);
this.samplingRunner.get(hook)?.set(caseId + stepId, () => {
clearInterval(interval);
});
}
}
}