@interopio/gateway-server/src/server/monitoring.ts

## `@glue42/gateway-ent` compatibility

import getLogger from '../logger.js';
import {getHeapStatistics, writeHeapSnapshot, HeapInfo} from 'node:v8';
import {PathLike} from 'node:fs';
import {access, mkdir, rename, unlink} from 'node:fs/promises';

const log = getLogger('monitoring');

export type Options = typeof DEFAULT_OPTIONS;

export type Command = 'run' | 'dump' | 'stop';
export type Channel = (command?: Command) => Promise<boolean>;

const DEFAULT_OPTIONS = {
    memoryLimit: 1024 * 1024 * 1024, // 1GB
    reportInterval: 10 * 60 * 1000, // 10 min
    dumpLocation: '.', // current folder
    maxBackups: 10,
    dumpPrefix: 'Heap'
}

function fetchStats(): HeapInfo {
    return getHeapStatistics();
}

async function dumpHeap(opts: Options) {
    const prefix = opts.dumpPrefix ?? 'Heap';
    const target = `${opts.dumpLocation}/${prefix}.heapsnapshot`;
    if (log.enabledFor('debug')) {
        log.debug(`starting heap dump in ${target}`);
    }

    await fileExists(opts.dumpLocation)
        .catch(async (_) => {
            if (log.enabledFor('debug')) {
                log.debug(`dump location  ${opts.dumpLocation} does not exists. Will try to create it`);
            }
            try {
                await mkdir(opts.dumpLocation, {recursive: true});
                log.info(`dump location dir ${opts.dumpLocation} successfully created`);
            } catch (e) {
                log.error(`failed to create dump location ${opts.dumpLocation}`);
            }
        });
    const dumpFileName = writeHeapSnapshot(target);
    log.info(`heap dumped`);
    try {
        log.debug(`rolling snapshot backups`);
        const lastFileName = `${opts.dumpLocation}/${prefix}.${opts.maxBackups}.heapsnapshot`;
        await fileExists(lastFileName)
            .then(async () => {
                if (log.enabledFor('debug')) {
                    log.debug(`deleting ${lastFileName}`);
                }
                try {
                    await unlink(lastFileName);
                } catch (e) {
                    log.warn(`failed to delete ${lastFileName}`, e);
                }
            })
            .catch(() => {
                /* do nothing*/
            });
        for (let i = opts.maxBackups - 1; i > 0; i--) {
            const currentFileName = `${opts.dumpLocation}/${prefix}.${i}.heapsnapshot`;
            const nextFileName = `${opts.dumpLocation}/${prefix}.${i + 1}.heapsnapshot`;
            await fileExists(currentFileName)
                .then(async () => {
                    try {
                        await rename(currentFileName, nextFileName);
                    } catch (e) {
                        log.warn(`failed to rename ${currentFileName} to ${nextFileName}`, e);
                    }
                })
                .catch(() => {
                    /* do nothing*/
                });
        }
        const firstFileName = `${opts.dumpLocation}/${prefix}.${1}.heapsnapshot`;
        try {
            await rename(dumpFileName, firstFileName);
        } catch (e) {
            log.warn(`failed to rename ${dumpFileName} to ${firstFileName}`, e);
        }
        log.debug('snapshots rolled');
    } catch (e) {
        log.error('error rolling backups', e);
        throw e;
    }
}

async function fileExists(path: PathLike): Promise<void> {
    log.trace(`checking file ${path}`);
    await access(path);
}

async function processStats(stats: HeapInfo, state: {
    memoryLimitExceeded: boolean,
    snapshot?: boolean
}, opts: Options) {
    if (log.enabledFor('debug')) {
        log.debug(`processing heap stats ${JSON.stringify(stats)}`);
    }
    const limit = Math.min(opts.memoryLimit, (0.95 * stats.heap_size_limit));
    const used = stats.used_heap_size;
    log.info(`heap stats ${JSON.stringify(stats)}`);
    if (used >= limit) {
        log.warn(`used heap ${used} bytes exceeds memory limit ${limit} bytes`);
        if (state.memoryLimitExceeded) {
            delete state.snapshot;
        } else {
            state.memoryLimitExceeded = true;
            state.snapshot = true;
        }
        await dumpHeap(opts);
    } else {
        state.memoryLimitExceeded = false;
        delete state.snapshot;
    }
}

export function start(opts?: Partial<Options>): Options & { channel: Channel } {
    const merged: Options = {...DEFAULT_OPTIONS, ...opts};

    let stopped = false;
    const state = {memoryLimitExceeded: false};
    const report = async () => {
        const stats = fetchStats();
        await processStats(stats, state, merged);
    }
    const interval = setInterval(report, merged.reportInterval);
    const channel = async (command?: Command) => {
        if (!stopped) {
            command ??= 'run';
            switch (command) {
                case 'run': {
                    await report();
                    break;
                }
                case 'dump': {
                    await dumpHeap(merged);
                    break;
                }
                case 'stop': {
                    stopped = true;
                    clearInterval(interval);
                    log.info('exit memory diagnostic');
                    break;
                }
            }

        }
        return stopped;
    }

    return {...merged, channel};
}

async function run({channel}: { channel: Channel }, command?: Command) {
    if (!await channel(command)) {
        log.warn(`cannot execute command "${command}" already closed`)
    }
}


export async function stop(m: { channel: Channel }) {
    return await run(m, 'stop');
}