@naturalcycles/nodejs-lib/dist/stream/progressLogger.js

Standard library for Node.js

import { inspect } from 'node:util';
import { _hc, _mb } from '@naturalcycles/js-lib';
import { _since, localTime } from '@naturalcycles/js-lib/datetime';
import { SimpleMovingAverage } from '@naturalcycles/js-lib/math';
import { boldWhite, dimGrey, hasColors, white, yellow } from '../colors/colors.js';
const inspectOpt = {
    colors: hasColors,
    breakLength: 300,
};
export class ProgressLogger {
    constructor(cfg = {}) {
        this.cfg = {
            metric: 'progress',
            rss: true,
            peakRSS: true,
            logRPS: true,
            logEvery: 1000,
            logSizesBuffer: 100_000,
            chunkSize: 1,
            logger: console,
            logProgress: cfg.logProgress !== false && cfg.logEvery !== 0,
            ...cfg,
        };
        this.logEvery10 = this.cfg.logEvery * 10;
        this.start();
        this.logStats(); // initial
    }
    cfg;
    started;
    lastSecondStarted;
    sma;
    logEvery10;
    processedLastSecond;
    progress;
    peakRSS;
    start() {
        this.started = Date.now();
        this.lastSecondStarted = Date.now();
        this.sma = new SimpleMovingAverage(10);
        this.processedLastSecond = 0;
        this.progress = 0;
        this.peakRSS = 0;
    }
    log(chunk) {
        this.progress++;
        this.processedLastSecond++;
        if (this.cfg.logProgress && this.progress % this.cfg.logEvery === 0) {
            this.logStats(chunk, false, this.progress % this.logEvery10 === 0);
        }
    }
    done() {
        this.logStats(undefined, true);
    }
    [Symbol.dispose]() {
        this.done();
    }
    logStats(chunk, final = false, tenx = false) {
        if (!this.cfg.logProgress)
            return;
        const { metric, extra, chunkSize, heapUsed: logHeapUsed, heapTotal: logHeapTotal, rss: logRss, peakRSS: logPeakRss, rssMinusHeap, external, arrayBuffers, logRPS, logger, } = this.cfg;
        const mem = process.memoryUsage();
        const now = Date.now();
        const batchedProgress = this.progress * chunkSize;
        const lastRPS = (this.processedLastSecond * chunkSize) / ((now - this.lastSecondStarted) / 1000) || 0;
        const rpsTotal = Math.round(batchedProgress / ((now - this.started) / 1000)) || 0;
        this.lastSecondStarted = now;
        this.processedLastSecond = 0;
        const rps10 = Math.round(this.sma.pushGetAvg(lastRPS));
        if (mem.rss > this.peakRSS)
            this.peakRSS = mem.rss;
        const o = {
            [final ? `${this.cfg.metric}_final` : this.cfg.metric]: batchedProgress,
        };
        if (extra)
            Object.assign(o, extra(chunk, this.progress));
        if (logHeapUsed)
            o.heapUsed = _mb(mem.heapUsed);
        if (logHeapTotal)
            o.heapTotal = _mb(mem.heapTotal);
        if (logRss)
            o.rss = _mb(mem.rss);
        if (logPeakRss)
            o.peakRSS = _mb(this.peakRSS);
        if (rssMinusHeap)
            o.rssMinusHeap = _mb(mem.rss - mem.heapTotal);
        if (external)
            o.external = _mb(mem.external);
        if (arrayBuffers)
            o.arrayBuffers = _mb(mem.arrayBuffers || 0);
        if (logRPS)
            Object.assign(o, { rps10, rpsTotal });
        logger.log(inspect(o, inspectOpt));
        if (tenx) {
            const perHour = _hc((batchedProgress * 1000 * 60 * 60) / (now - this.started));
            logger.log(`${dimGrey(localTime.now().toPretty())} ${white(metric)} took ${yellow(_since(this.started))} so far to process ${yellow(_hc(batchedProgress))} rows, ~${yellow(perHour)}/hour`);
        }
        else if (final) {
            logger.log(`${boldWhite(metric)} took ${yellow(_since(this.started))} to process ${yellow(batchedProgress)} rows with total RPS of ${yellow(rpsTotal)}`);
            try {
                this.cfg.onProgressDone?.(o);
            }
            catch (err) {
                logger.error(err);
            }
        }
    }
}
/**
 * Create new ProgressLogger.
 */
export function progressLogger(cfg = {}) {
    return new ProgressLogger(cfg);
}
/**
 * Limitation: I don't know how to catch the `final` callback to log final stats.
 *
 * @experimental
 */
export function progressReadableMapper(cfg = {}) {
    const progress = new ProgressLogger(cfg);
    return chunk => {
        progress.log(chunk);
        return chunk;
    };
}