artillery/dist/lib/core/ssms.js

Cloud-scale load testing. https://www.artillery.io

/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
// Super simple metric store
// Use our own fork of DDSketch until this PR is merged into main:
// https://github.com/DataDog/sketches-js/pull/13
import { EventEmitter } from 'node:events';
import { DDSketch } from '@artilleryio/sketches-js';
import createDebug from 'debug';
import { clearDriftless, setDriftlessInterval } from 'driftless';
const debug = createDebug('ssms');
const MAX_METRIC_NAME_LENGTH = 1024;
class SSMS extends EventEmitter {
    constructor(_options) {
        super();
        this.opts = _options || {};
        this._counterEarliestMeasurementByPeriod = {};
        this._counterLastMeasurementByPeriod = {};
        this._histogramEarliestMeasurementByPeriod = {};
        this._histogramLastMeasurementByPeriod = {};
        this._aggregationIntervalSec = 5;
        this._aggregateInterval = setDriftlessInterval(this.aggregate.bind(this), this._aggregationIntervalSec * 1000);
        this._lastPeriod = null;
        this.isPullOnly = Boolean(this.opts.pullOnly);
        if (!this.isPullOnly) {
            this._emitInterval = setDriftlessInterval(this._maybeEmitMostRecentPeriod.bind(this), Math.max((this._aggregationIntervalSec * 1000) / 2, 1000));
        }
        this._counters = [];
        this._histograms = [];
        this._rates = [];
        this._active = true;
        this._aggregatedCounters = {};
        this._aggregatedHistograms = {};
        this._aggregatedRates = {};
    }
    stop() {
        this._active = false;
        clearDriftless(this._aggregateInterval);
        if (!this.isPullOnly) {
            clearDriftless(this._emitInterval);
        }
        return this;
    }
    static report(pds) {
        return pds;
    }
    // TODO: first/last metric timestamps should not = period
    static empty(ts) {
        const period = normalizeTs(ts || Date.now());
        return {
            counters: {},
            histograms: {},
            rates: {},
            firstCounterAt: 0,
            firstHistogramAt: 0,
            lastCounterAt: 0,
            lastHistogramAt: 0,
            firstMetricAt: 0,
            lastMetricAt: 0,
            period
        };
    }
    static summarizeHistogram(h) {
        return summarizeHistogram(h);
    }
    // Take metric data for a period and return a summary object with 1.6.x-compatible format
    static legacyReport(pd) {
        const counters = pd.counters || {};
        const histograms = pd.histograms;
        const result = {
            // Custom field compatibility not supported:
            customStats: {},
            counters: {},
            scenariosAvoided: counters['vusers.skipped'] || 0,
            timestamp: new Date(pd.period),
            scenariosCreated: counters['vusers.created'] || 0,
            scenariosCompleted: counters['vusers.completed'] || 0,
            requestsCompleted: counters['http.responses'] ||
                counters['socketio.emit'] ||
                counters['websocket.messages_sent'] ||
                0,
            latency: {},
            rps: {
                mean: pd.rates
                    ? pd.rates['http.response_rate'] ||
                        pd.rates['socketio.emit_rate'] ||
                        0
                    : 0,
                count: counters['http.responses'] || counters['socketio.emit'] || 0
            },
            scenarioDuration: {},
            scenarioCounts: {},
            errors: {},
            codes: {}
        };
        if (histograms &&
            typeof histograms['vusers.session_length'] !== 'undefined') {
            result.scenarioDuration = summarizeHistogram(histograms['vusers.session_length']);
        }
        // scenarioCounts
        const names = Object.keys(counters).filter((k) => k.startsWith('vusers.created_by_name.'));
        for (const n of names) {
            result.scenarioCounts[n.split('vusers.created_by_name.')[1]] =
                counters[n];
        }
        // latency
        const latencyh = histograms
            ? histograms['http.response_time'] || histograms['socketio.response_time']
            : null;
        if (latencyh) {
            result.latency = summarizeHistogram(latencyh);
        }
        // HTTP codes
        const codeNames = Object.keys(counters).filter((k) => k.match(/^(http|socketio)\.codes.*/));
        for (const n of codeNames) {
            const code = parseInt(n.split('.codes.')[1], 10);
            result.codes[code] = counters[n];
        }
        // errors
        const errNames = Object.keys(counters).filter((k) => k.startsWith('errors.'));
        for (const n of errNames) {
            const errName = n.split('errors.')[1];
            result.errors[errName] = counters[n];
        }
        return {
            report: () => result
        };
    }
    // Return object indexed by period (as string):
    static mergeBuckets(periodData) {
        debug(`mergeBuckets // timeslices: ${periodData.map((pd) => pd.period)}`);
        // Returns result[timestamp] = {histograms:{},counters:{},rates:{}}
        // ie. the result is indexed by timeslice
        const result = {};
        for (const pd of periodData) {
            const ts = pd.period;
            if (!result[ts]) {
                result[ts] = {
                    period: ts,
                    counters: {},
                    histograms: {},
                    rates: {}
                };
            }
            // Normalize in place - callers may rely on the input objects
            // having these containers after a merge.
            const counters = pd.counters || {};
            pd.counters = counters;
            const histograms = pd.histograms || {};
            pd.histograms = histograms;
            const rates = pd.rates || {};
            pd.rates = rates;
            //
            // counters
            //
            for (const [name, value] of Object.entries(counters)) {
                if (!result[ts].counters[name]) {
                    result[ts].counters[name] = 0;
                }
                result[ts].counters[name] += value;
            }
            //
            // histograms
            //
            for (const [name, origValue] of Object.entries(histograms)) {
                const value = SSMS.cloneHistogram(origValue);
                if (typeof result[ts].histograms[name] === 'undefined') {
                    result[ts].histograms[name] = value;
                }
                else {
                    // NOTE: this will throw if gamma (accuracy) parameters are different
                    // in those two sketches
                    result[ts].histograms[name].merge(value);
                }
            }
            //
            // rates
            //
            for (const [name, value] of Object.entries(rates)) {
                if (typeof result[ts].rates[name] === 'undefined') {
                    result[ts].rates[name] = 0;
                }
                result[ts].rates[name] += value;
            }
            // NOTE: pre-existing quirk preserved as-is: this writes
            // `result[ts][name]` (rate names directly on the period object,
            // not on `.rates`), and `result[ts][name]` starts undefined so
            // the division yields NaN. Kept byte-for-byte for output
            // compatibility until the metrics model is consolidated.
            for (const name of Object.keys(rates)) {
                const r = result[ts];
                r[name] = round(r[name] / periodData.length, 1);
            }
            result[ts].firstCounterAt = min([
                result[ts].firstCounterAt,
                pd.firstCounterAt
            ]);
            result[ts].firstHistogramAt = min([
                result[ts].firstHistogramAt,
                pd.firstHistogramAt
            ]);
            result[ts].lastCounterAt = max([
                result[ts].lastCounterAt,
                pd.lastCounterAt
            ]);
            result[ts].lastHistogramAt = max([
                result[ts].lastHistogramAt,
                pd.lastHistogramAt
            ]);
            result[ts].firstMetricAt = min([
                result[ts].firstHistogramAt,
                result[ts].firstCounterAt
            ]);
            result[ts].lastMetricAt = max([
                result[ts].lastHistogramAt,
                result[ts].lastCounterAt
            ]);
            result[ts].period = ts;
        }
        return result;
    }
    // Aggregate at lower resolution, i.e. combine three distinct periods of 10s into one of 30s
    // Note: does not check that periods are contiguous, everything is simply merged together
    static pack(periods) {
        const result = {
            period: 0,
            counters: {},
            histograms: {},
            rates: {}
        };
        for (const pd of periods) {
            pd.counters = Object.assign({}, pd.counters || {});
            pd.histograms = Object.assign({}, pd.histograms || {});
            pd.rates = Object.assign({}, pd.rates || {});
            for (const [name, value] of Object.entries(pd.counters)) {
                if (!result.counters[name]) {
                    result.counters[name] = 0;
                }
                result.counters[name] += value;
            }
            for (const [name, origValue] of Object.entries(pd.histograms)) {
                const value = SSMS.cloneHistogram(origValue);
                if (typeof result.histograms[name] === 'undefined') {
                    result.histograms[name] = value;
                }
                else {
                    // NOTE: this will throw if gamma (accuracy) parameters are different
                    // in those two sketches
                    result.histograms[name].merge(value);
                }
            }
            for (const [name, value] of Object.entries(pd.rates)) {
                if (!result.rates[name]) {
                    result.rates[name] = 0;
                }
                // TODO: retain first/last so that we have the duration
                // or retain the duration of the window in which rate events
                // were recorded alongside the average value
                result.rates[name] += value;
            }
        }
        for (const [name, _value] of Object.entries(result.rates)) {
            result.rates[name] = round(result.rates[name] / periods.length, 0);
        }
        result.firstCounterAt = min(periods.map((p) => p.firstCounterAt));
        result.firstHistogramAt = min(periods.map((p) => p.firstHistogramAt));
        result.lastCounterAt = max(periods.map((p) => p.lastCounterAt));
        result.lastHistogramAt = max(periods.map((p) => p.lastHistogramAt));
        result.firstMetricAt = min([
            result.firstHistogramAt,
            result.firstCounterAt
        ]);
        result.lastMetricAt = max([result.lastHistogramAt, result.lastCounterAt]);
        // Kept as-is: max() returns undefined when every period lacks a
        // period value; real inputs always carry one.
        result.period = max(periods.map((p) => p.period));
        return result;
    }
    static cloneHistogram(h) {
        return DDSketch.fromProto(h.toProto());
    }
    static serializeMetrics(pd) {
        // TODO: Add ability to include arbitrary metadata e.g. worker IDs
        const serializedHistograms = {};
        const ph = pd.histograms;
        if (ph) {
            for (const n of Object.keys(ph)) {
                const h = ph[n];
                const buf = h.toProto();
                serializedHistograms[n] = buf;
            }
        }
        // TODO: Mark as serialized, otherwise to check whether we have a serialized object or not
        // is to check if .histograms is a Buffer
        const result = Object.assign({}, pd, { histograms: serializedHistograms });
        return stringify(result);
    }
    static deserializeMetrics(pd) {
        // Serialized form always carries a histograms map (see
        // serializeMetrics) with protobuf buffers as values.
        const object = parse(pd);
        for (const [name, buf] of Object.entries(object.histograms)) {
            const h = DDSketch.fromProto(buf);
            object.histograms[name] = h;
        }
        return object;
    }
    getBucketIds() {
        return [
            ...new Set(Object.keys(this._aggregatedCounters)
                .concat(Object.keys(this._aggregatedHistograms))
                .sort())
        ].reverse();
    }
    // TODO: Deprecate
    counter(name, value) {
        this.incr(name.slice(0, MAX_METRIC_NAME_LENGTH), value);
    }
    incr(name, value, t) {
        this._counters.push(t || Date.now(), name.slice(0, MAX_METRIC_NAME_LENGTH), value);
    }
    // TODO: Deprecate
    summary(name, value) {
        this.histogram(name.slice(0, MAX_METRIC_NAME_LENGTH), value);
    }
    histogram(name, value, t) {
        this._histograms.push(t || Date.now(), name.slice(0, MAX_METRIC_NAME_LENGTH), value);
    }
    rate(name, t) {
        this._rates.push(t || Date.now(), name.slice(0, MAX_METRIC_NAME_LENGTH));
    }
    getMetrics(period) {
        const result = { period };
        const counters = this._aggregatedCounters[period];
        const histograms = this._aggregatedHistograms[period];
        const rates = this._aggregatedRates[period];
        if (counters) {
            result.counters = counters;
        }
        if (histograms) {
            result.histograms = histograms;
        }
        if (rates) {
            result.rates = rates;
        }
        result.firstCounterAt = this._counterEarliestMeasurementByPeriod[period];
        result.firstHistogramAt =
            this._histogramEarliestMeasurementByPeriod[period];
        result.lastCounterAt = this._counterLastMeasurementByPeriod[period];
        result.lastHistogramAt = this._histogramLastMeasurementByPeriod[period];
        result.firstMetricAt = min([
            result.firstHistogramAt,
            result.firstCounterAt
        ]);
        result.lastMetricAt = max([result.lastHistogramAt, result.lastCounterAt]);
        // TODO: Include size of the window, for cases when it's not 10s
        return result;
    }
    _aggregateHistograms(upToTimeslice) {
        for (let i = 0; i < this._histograms.length; i += 3) {
            const ts = this._histograms[i];
            const timeslice = normalizeTs(ts);
            if (timeslice >= upToTimeslice) {
                this._histograms.splice(0, i);
                return;
            }
            const name = this._histograms[i + 1];
            const value = this._histograms[i + 2];
            if (!this._aggregatedHistograms[timeslice]) {
                this._aggregatedHistograms[timeslice] = {};
                this._histogramEarliestMeasurementByPeriod[timeslice] = ts;
            }
            if (!this._aggregatedHistograms[timeslice][name]) {
                this._aggregatedHistograms[timeslice][name] = new DDSketch({
                    relativeAccuracy: 0.01
                });
            }
            // TODO: Benchmark
            this._histogramLastMeasurementByPeriod[timeslice] = ts;
            this._aggregatedHistograms[timeslice][name].accept(value);
        }
        this._histograms.splice(0, this._histograms.length);
    }
    _aggregateCounters(upToTimeslice) {
        // Consider memory-CPU tradeoff. Depending on the length of the buffer, we may want to
        // not exceed N total entries we're processing if we can delay reporting by one or more
        // reporting periods
        for (let i = 0; i < this._counters.length; i += 3) {
            const ts = this._counters[i];
            const timeslice = normalizeTs(ts);
            if (timeslice >= upToTimeslice) {
                this._counters.splice(0, i);
                return;
            }
            const name = this._counters[i + 1];
            const value = this._counters[i + 2];
            if (!this._aggregatedCounters[timeslice]) {
                this._aggregatedCounters[timeslice] = {};
                this._counterEarliestMeasurementByPeriod[timeslice] = ts;
            }
            if (typeof this._aggregatedCounters[timeslice][name] === 'undefined') {
                this._aggregatedCounters[timeslice][name] = value;
            }
            else {
                this._aggregatedCounters[timeslice][name] += value;
            }
            this._counterLastMeasurementByPeriod[timeslice] = ts;
        }
        this._counters.splice(0, this._counters.length);
    }
    _aggregateRates(upToTimeslice) {
        debug('_aggregateRates to', upToTimeslice, new Date(upToTimeslice));
        const a = {};
        let spliceTo = this._rates.length;
        for (let i = 0; i < this._rates.length; i += 2) {
            const ts = this._rates[i];
            const timeslice = normalizeTs(ts);
            if (timeslice >= upToTimeslice) {
                debug('_aggregateRates early return // i=', i, 'timeslice=', timeslice, new Date(timeslice));
                spliceTo = i;
                break;
            }
            const name = this._rates[i + 1];
            if (!a[timeslice]) {
                a[timeslice] = {};
            }
            if (!a[timeslice][name]) {
                a[timeslice][name] = {
                    first: Number.POSITIVE_INFINITY,
                    last: 0,
                    count: 0
                };
            }
            a[timeslice][name].first = Math.min(a[timeslice][name].first, ts);
            a[timeslice][name].last = Math.max(a[timeslice][name].last, ts);
            a[timeslice][name].count++;
        }
        for (const [ts, rs] of Object.entries(a)) {
            for (const [name, _] of Object.entries(rs)) {
                const { first, last, count } = a[ts][name];
                if (!this._aggregatedRates[ts]) {
                    this._aggregatedRates[ts] = {};
                }
                this._aggregatedRates[ts][name] = round(count / (Math.max(last - first, 1000) / 1000), 0);
            }
        }
        this._rates.splice(0, spliceTo);
    }
    aggregate(forceAll) {
        const currentTimeslice = normalizeTs(Date.now()) + (forceAll ? 30 * 1000 : 0);
        this._aggregateCounters(currentTimeslice);
        this._aggregateHistograms(currentTimeslice);
        this._aggregateRates(currentTimeslice);
        if (forceAll) {
            this._emitPeriods();
        }
        else {
            this._maybeEmitMostRecentPeriod();
        }
    }
    _emitPeriods() {
        const bucketIds = this.getBucketIds();
        const lastPeriod = parseInt(this._lastPeriod ?? '', 10);
        for (let i = 0; i < bucketIds.length; i++) {
            const period = bucketIds[i];
            if (!this._lastPeriod || parseInt(period, 10) > lastPeriod) {
                this.emit('metricData', period, this.getMetrics(period));
            }
        }
    }
    _maybeEmitMostRecentPeriod() {
        const p = this.getBucketIds()[0];
        if (p && p !== this._lastPeriod) {
            this.emit('metricData', p, this.getMetrics(p)); // Measurements in period p have been aggregated
            this._lastPeriod = p;
        }
    }
}
function normalizeTs(epochMs, windowSize = 10) {
    // Reset down to minute
    const m = Math.floor((epochMs - (epochMs % 1000)) / 1000 / 60) * 60 * 1000;
    // Number of seconds past the minute
    const s = ((epochMs - (epochMs % 1000)) / 1000) % 60;
    // Number of seconds to take off
    const d = s % windowSize;
    return m + (s - d) * 1000;
}
// Function hms(epochMs) {
//   return [
//     Math.round((epochMs / 1000 / 60 / 60) % 24),
//     Math.round((epochMs / 1000 / 60) % 60),
//     Math.round(epochMs / 1000) % 60
//   ];
// }
function round(number, decimals) {
    const m = 10 ** decimals;
    return Math.round(number * m) / m;
}
// h is an instance of DDSketch
function summarizeHistogram(h) {
    return {
        min: round(h.min, 1),
        max: round(h.max, 1),
        count: h.count,
        mean: round(h.sum / h.count, 1),
        p50: round(h.getValueAtQuantile(0.5), 1),
        median: round(h.getValueAtQuantile(0.5), 1), // Here for compatibility
        p75: round(h.getValueAtQuantile(0.75), 1),
        p90: round(h.getValueAtQuantile(0.9), 1),
        p95: round(h.getValueAtQuantile(0.95), 1),
        p99: round(h.getValueAtQuantile(0.99), 1),
        p999: round(h.getValueAtQuantile(0.999), 1)
    };
}
/// ///////////////////////////////////////////
function stringify(value, space) {
    return JSON.stringify(value, replacer, space);
}
function parse(text) {
    return JSON.parse(text, reviver);
}
function replacer(_key, value) {
    if (isBufferLike(value) && isArray(value.data)) {
        if (value.data.length > 0) {
            value.data = `base64:${Buffer.from(value.data).toString('base64')}`;
        }
        else {
            value.data = '';
        }
    }
    return value;
}
function reviver(_key, value) {
    if (isBufferLike(value)) {
        if (isArray(value.data)) {
            return Buffer.from(value.data);
        }
        if (isString(value.data)) {
            if (value.data.startsWith('base64:')) {
                return Buffer.from(value.data.slice('base64:'.length), 'base64');
            }
            // Assume that the string is UTF-8 encoded (or empty).
            return Buffer.from(value.data);
        }
    }
    return value;
}
function isBufferLike(x) {
    return (isObject(x) && x.type === 'Buffer' && (isArray(x.data) || isString(x.data)));
}
function isArray(x) {
    return Array.isArray(x);
}
function isString(x) {
    return typeof x === 'string';
}
function isObject(x) {
    return typeof x === 'object' && x !== null;
}
/// /////////////////
// Like Math.min and Math.max but take a list of values, and ignore
// undefined's rather than returning NaN when a value is undefined.
// Returns undefined if all arguments are undefined.
// NOTE: pre-existing quirk preserved: the truthiness filter also drops
// legitimate zero values.
function min(values) {
    const m = Math.min(...values.filter((x) => x));
    return m === Number.POSITIVE_INFINITY ? undefined : m;
}
function max(values) {
    const m = Math.max(...values.filter((x) => x));
    return m === Number.NEGATIVE_INFINITY ? undefined : m;
}
export { SSMS, summarizeHistogram, normalizeTs };