toofast/runner-api.js

The Node.js performance testing tool with unit-test-like API.

import { combineHooks, getErrorMessage, noop, sleep } from './utils.js';
import { Histogram } from './Histogram.js';
const MEASURE_TIMEOUT = 10000;
const TARGET_RME = 0.05;
const WARMUP_ITERATION_COUNT = 1;
const BATCH_ITERATION_COUNT = Infinity;
const BATCH_TIMEOUT = 1000;
const BATCH_INTERMISSION_TIMEOUT = 200;
export class Node {
    constructor(callback, testOptions = {}) {
        this.testOptions = testOptions;
        this.parent = null;
        this.children = [];
        this.isSkipped = false;
        this.beforeEachHook = undefined;
        this.afterEachHook = undefined;
        this.beforeWarmupHook = undefined;
        this.afterWarmupHook = undefined;
        this.beforeBatchHook = undefined;
        this.afterBatchHook = undefined;
        this.beforeIterationHook = undefined;
        this.afterIterationHook = undefined;
        this.callback = () => {
            const value = callback();
            this.callback = noop;
            return value;
        };
    }
    beforeEach(hook) {
        this.beforeEachHook = combineHooks(this.beforeEachHook, hook);
    }
    afterEach(hook) {
        this.afterEachHook = combineHooks(this.afterEachHook, hook);
    }
    beforeWarmup(hook) {
        this.beforeWarmupHook = combineHooks(this.beforeWarmupHook, hook);
    }
    afterWarmup(hook) {
        this.afterWarmupHook = combineHooks(this.afterWarmupHook, hook);
    }
    beforeBatch(hook) {
        this.beforeBatchHook = combineHooks(this.beforeBatchHook, hook);
    }
    afterBatch(hook) {
        this.afterBatchHook = combineHooks(this.afterBatchHook, hook);
    }
    beforeIteration(hook) {
        this.beforeIterationHook = combineHooks(this.beforeIterationHook, hook);
    }
    afterIteration(hook) {
        this.afterIterationHook = combineHooks(this.afterIterationHook, hook);
    }
    appendChild(child) {
        if (child.parent !== null) {
            throw new Error('Child already has a parent');
        }
        child.parent = this;
        child.testOptions = Object.assign(Object.assign({}, this.testOptions), child.testOptions);
        child.beforeEachHook = combineHooks(this.beforeEachHook, child.beforeEachHook);
        child.afterEachHook = combineHooks(this.afterEachHook, child.afterEachHook);
        child.beforeWarmupHook = combineHooks(this.beforeWarmupHook, child.beforeWarmupHook);
        child.afterWarmupHook = combineHooks(this.afterWarmupHook, child.afterWarmupHook);
        child.beforeBatchHook = combineHooks(this.beforeBatchHook, child.beforeBatchHook);
        child.afterBatchHook = combineHooks(this.afterBatchHook, child.afterBatchHook);
        child.beforeIterationHook = combineHooks(this.beforeIterationHook, child.beforeIterationHook);
        child.afterIterationHook = combineHooks(this.afterIterationHook, child.afterIterationHook);
        this.children.push(child);
        return this;
    }
    getBlockChildren() {
        const children = [];
        for (const child of this.children) {
            if (child instanceof DescribeNode) {
                children.push({ type: 'describe', name: child.name });
            }
            if (child instanceof TestNode) {
                children.push({ type: 'test', name: child.name });
            }
            if (child instanceof MeasureNode) {
                children.push({ type: 'measure' });
            }
        }
        return children;
    }
}
export class TestSuiteNode extends Node {
    constructor(testOptions = {}) {
        super(noop, testOptions);
    }
}
export class DescribeNode extends Node {
    constructor(name, callback, testOptions = {}) {
        super(callback, testOptions);
        this.name = name;
    }
}
export class TestNode extends Node {
    constructor(name, callback, testOptions = {}) {
        super(callback, testOptions);
        this.name = name;
    }
    get absoluteName() {
        let absoluteName = this.name;
        for (let node = this.parent; node instanceof DescribeNode; node = node.parent) {
            absoluteName = node.name + '.' + absoluteName;
        }
        return absoluteName;
    }
}
export class MeasureNode extends Node {
    constructor(callback, measureOptions = {}) {
        super(callback, measureOptions);
        this.afterWarmupHook = measureOptions.afterWarmup;
        this.beforeBatchHook = measureOptions.beforeBatch;
        this.afterBatchHook = measureOptions.afterBatch;
        this.beforeIterationHook = measureOptions.beforeIteration;
        this.afterIterationHook = measureOptions.afterIteration;
    }
}
export async function bootstrapRunner(options) {
    const { setupFiles, testFile, evalFile, sendMessage } = options;
    try {
        for (const file of setupFiles) {
            await evalFile(file);
        }
        await evalFile(testFile);
    }
    catch (error) {
        sendMessage({ type: 'fatalError', errorMessage: getErrorMessage(error) });
        return false;
    }
    return true;
}
export async function runTest(options) {
    var _a, _b;
    const { startNode, isSkipped, setCurrentNode, runMeasure, sendMessage } = options;
    let node = startNode;
    const nodeLocation = options.nodeLocation.splice(0);
    for (const index of nodeLocation.slice(0, -1)) {
        node = node.children[index];
        setCurrentNode(node);
        await node.callback();
    }
    let depth = 0;
    let startIndex = 0;
    if (nodeLocation.length !== 0) {
        depth = nodeLocation.length - 1;
        startIndex = nodeLocation[depth] + 1;
    }
    if (node instanceof TestSuiteNode) {
        sendMessage({ type: 'testSuiteStart' });
    }
    traversal: while (true) {
        if (startIndex === 0) {
            sendMessage({
                type: 'blockStart',
                kind: node instanceof TestSuiteNode ? 'testSuite' : 'describe',
                children: node.getBlockChildren(),
            });
        }
        for (let i = startIndex; i < node.children.length; ++i) {
            const child = node.children[i];
            nodeLocation[depth] = i;
            if (child.isSkipped) {
                continue;
            }
            if (child instanceof DescribeNode) {
                sendMessage({ type: 'describeStart', name: child.name });
                setCurrentNode(child);
                try {
                    await child.callback();
                }
                catch (error) {
                    // Skip describe block after an error
                    sendMessage({ type: 'error', errorMessage: getErrorMessage(error) });
                    sendMessage({ type: 'blockEnd' });
                    sendMessage({ type: 'describeEnd' });
                    continue;
                }
                depth++;
                startIndex = nodeLocation[depth] = 0;
                node = child;
                continue traversal;
            }
            if (child instanceof TestNode && !isSkipped(child)) {
                sendMessage({
                    type: 'testStart',
                    name: child.name,
                    nodeLocation: nodeLocation.slice(0, depth + 1),
                });
                setCurrentNode(child);
                const durationHistogram = new Histogram();
                const memoryHistogram = new Histogram();
                let isBlockStarted = false;
                try {
                    await ((_a = child.beforeEachHook) === null || _a === void 0 ? void 0 : _a.call(child));
                    await child.callback();
                    sendMessage({ type: 'blockStart', kind: 'test', children: child.getBlockChildren() });
                    isBlockStarted = true;
                    for (const node of child.children) {
                        const result = await runMeasure(node);
                        if (result !== null) {
                            durationHistogram.add(result.durationHistogram);
                            memoryHistogram.add(result.memoryHistogram);
                        }
                    }
                    await ((_b = child.afterEachHook) === null || _b === void 0 ? void 0 : _b.call(child));
                }
                catch (error) {
                    if (!isBlockStarted) {
                        sendMessage({ type: 'blockStart', kind: 'test', children: [] });
                    }
                    sendMessage({ type: 'error', errorMessage: getErrorMessage(error) });
                }
                sendMessage({ type: 'blockEnd' });
                sendMessage({
                    type: 'testEnd',
                    durationStats: durationHistogram.getStats(),
                    memoryStats: memoryHistogram.getStats(),
                });
                return;
            }
        }
        while (node.parent !== null) {
            sendMessage({ type: 'blockEnd' });
            sendMessage({ type: 'describeEnd' });
            node = node.parent;
            nodeLocation[depth] = 0;
            startIndex = ++nodeLocation[--depth];
            if (startIndex < node.children.length) {
                continue traversal;
            }
        }
        break;
    }
    sendMessage({ type: 'blockEnd' });
    sendMessage({ type: 'testSuiteEnd' });
}
export function createRunMeasure(options) {
    const { sendMessage, getMemoryUsed } = options;
    return async (node) => {
        const durationHistogram = new Histogram();
        const memoryHistogram = new Histogram();
        const { testOptions: { measureTimeout = MEASURE_TIMEOUT, targetRme = TARGET_RME, warmupIterationCount = WARMUP_ITERATION_COUNT, batchIterationCount = BATCH_ITERATION_COUNT, batchTimeout = BATCH_TIMEOUT, batchIntermissionTimeout = BATCH_INTERMISSION_TIMEOUT, }, callback, 
        // beforeEachHook,
        // afterEachHook,
        beforeWarmupHook, afterWarmupHook, beforeBatchHook, afterBatchHook, beforeIterationHook, afterIterationHook, } = node;
        // Warmup phase
        if (warmupIterationCount > 0) {
            sendMessage({ type: 'measureWarmupStart' });
            await (beforeWarmupHook === null || beforeWarmupHook === void 0 ? void 0 : beforeWarmupHook());
            await (beforeBatchHook === null || beforeBatchHook === void 0 ? void 0 : beforeBatchHook());
            for (let i = 0; i < warmupIterationCount; ++i) {
                await (beforeIterationHook === null || beforeIterationHook === void 0 ? void 0 : beforeIterationHook());
                try {
                    callback();
                }
                catch (error) {
                    // Skip measure block after an error
                    sendMessage({ type: 'error', errorMessage: getErrorMessage(error) });
                    sendMessage({ type: 'measureWarmupEnd' });
                    return null;
                }
                await (afterIterationHook === null || afterIterationHook === void 0 ? void 0 : afterIterationHook());
            }
            await (afterBatchHook === null || afterBatchHook === void 0 ? void 0 : afterBatchHook());
            await (afterWarmupHook === null || afterWarmupHook === void 0 ? void 0 : afterWarmupHook());
            sendMessage({ type: 'measureWarmupEnd' });
            await sleep(batchIntermissionTimeout);
        }
        let totalIterationCount = 0;
        let prevPercentage = 0;
        const measureTimestamp = Date.now();
        const nextBatch = async () => {
            const batchTimestamp = Date.now();
            let iterationCount = 0;
            while (true) {
                ++iterationCount;
                ++totalIterationCount;
                await (beforeIterationHook === null || beforeIterationHook === void 0 ? void 0 : beforeIterationHook());
                const prevMemoryUsed = getMemoryUsed();
                try {
                    const timestamp = performance.now();
                    callback();
                    durationHistogram.add(performance.now() - timestamp);
                }
                catch (error) {
                    sendMessage({ type: 'error', errorMessage: getErrorMessage(error) });
                }
                const memoryUsed = getMemoryUsed() - prevMemoryUsed;
                if (memoryUsed > 0) {
                    memoryHistogram.add(memoryUsed);
                }
                await (afterIterationHook === null || afterIterationHook === void 0 ? void 0 : afterIterationHook());
                const measureDuration = Date.now() - measureTimestamp;
                const { rme } = durationHistogram;
                if (measureDuration > measureTimeout || (totalIterationCount > 2 && targetRme >= rme)) {
                    sendMessage({ type: 'measureProgress', percentage: 1 });
                    // Measurements completed
                    await (afterBatchHook === null || afterBatchHook === void 0 ? void 0 : afterBatchHook());
                    return;
                }
                const nextPercentage = Math.trunc(Math.max(prevPercentage, measureDuration / measureTimeout || 0, totalIterationCount > 2 ? targetRme / rme || 0 : 0) * 1000) / 1000;
                if (prevPercentage !== nextPercentage) {
                    prevPercentage = nextPercentage;
                    sendMessage({ type: 'measureProgress', percentage: nextPercentage });
                }
                if (Date.now() - batchTimestamp > batchTimeout || iterationCount >= batchIterationCount) {
                    // Schedule the next measurement batch
                    await (afterBatchHook === null || afterBatchHook === void 0 ? void 0 : afterBatchHook());
                    // The pause between batches is required for garbage collection
                    await sleep(batchIntermissionTimeout);
                    await (beforeBatchHook === null || beforeBatchHook === void 0 ? void 0 : beforeBatchHook());
                    return nextBatch();
                }
            }
        };
        sendMessage({ type: 'measureStart' });
        await (beforeBatchHook === null || beforeBatchHook === void 0 ? void 0 : beforeBatchHook());
        sendMessage({ type: 'measureProgress', percentage: 0 });
        await nextBatch();
        sendMessage({
            type: 'measureEnd',
            durationStats: durationHistogram.getStats(),
            memoryStats: memoryHistogram.getStats(),
        });
        return { durationHistogram, memoryHistogram };
    };
}
let currentNode = new TestSuiteNode();
export function getCurrentNode() {
    return currentNode;
}
export function setCurrentNode(node) {
    currentNode = node;
}