@naturalcycles/db-lib/dist/pipeline/dbPipelineBackup.js

Lowest Common Denominator API to supported Databases

import { _passthroughMapper, AppError, ErrorMode, localTime, pMap } from '@naturalcycles/js-lib';
import { _pipeline, boldWhite, dimWhite, fs2, grey, NDJsonStats, transformLogProgress, transformMap, transformTap, yellow, } from '@naturalcycles/nodejs-lib';
import { DBQuery } from '../query/dbQuery.js';
/**
 * Pipeline from input stream(s) to a NDJSON file (optionally gzipped).
 * File is overwritten (by default).
 * Input stream can be a stream from CommonDB.streamQuery()
 * Allows to define a mapper and a predicate to map/filter objects between input and output.
 * Handles backpressure.
 *
 * Optionally you can provide mapperPerTable and @param transformMapOptions (one for all mappers) - it will run for each table.
 */
export async function dbPipelineBackup(opt) {
    const { db, concurrency = 16, limit = 0, outputDirPath, protectFromOverwrite = false, mapperPerTable = {}, queryPerTable = {}, logEveryPerTable = {}, transformMapOptions, errorMode = ErrorMode.SUPPRESS, emitSchemaFromDB = false, } = opt;
    const gzip = opt.gzip !== false; // default to true
    let { tables } = opt;
    console.log(`>> ${dimWhite('dbPipelineBackup')} started in ${grey(outputDirPath)}...`);
    fs2.ensureDir(outputDirPath);
    tables ||= await db.getTables();
    console.log(`${yellow(tables.length)} ${boldWhite('table(s)')}:\n` + tables.join('\n'));
    const statsPerTable = {};
    await pMap(tables, async (table) => {
        let q = DBQuery.create(table).limit(limit);
        const sinceUpdated = opt.sinceUpdatedPerTable?.[table] ?? opt.sinceUpdated;
        if (sinceUpdated) {
            q = q.filter('updated', '>=', sinceUpdated);
        }
        if (queryPerTable[table]) {
            // Override the Query with this Query, completely ingoring any of the other query-related options
            q = queryPerTable[table];
            console.log(`>> ${grey(table)} ${q.pretty()}`);
        }
        else {
            const sinceUpdatedStr = sinceUpdated
                ? ' since ' + grey(localTime(sinceUpdated).toPretty())
                : '';
            console.log(`>> ${grey(table)}${sinceUpdatedStr}`);
        }
        const filePath = `${outputDirPath}/${table}.ndjson` + (gzip ? '.gz' : '');
        const schemaFilePath = `${outputDirPath}/${table}.schema.json`;
        if (protectFromOverwrite && fs2.pathExists(filePath)) {
            throw new AppError(`dbPipelineBackup: output file exists: ${filePath}`);
        }
        const started = Date.now();
        let rows = 0;
        fs2.ensureFile(filePath);
        // console.log(`>> ${grey(filePath)} started...`)
        if (emitSchemaFromDB) {
            const schema = await db.getTableSchema(table);
            await fs2.writeJsonAsync(schemaFilePath, schema, { spaces: 2 });
            console.log(`>> ${grey(schemaFilePath)} saved (generated from DB)`);
        }
        await _pipeline([
            db.streamQuery(q),
            transformLogProgress({
                ...opt,
                logEvery: logEveryPerTable[table] ?? opt.logEvery ?? 1000,
                metric: table,
            }),
            transformMap(mapperPerTable[table] || _passthroughMapper, {
                errorMode,
                flattenArrayOutput: true,
                ...transformMapOptions,
                metric: table,
            }),
            transformTap(() => {
                rows++;
            }),
            ...fs2.createWriteStreamAsNDJSON(filePath),
        ]);
        const { size: sizeBytes } = await fs2.statAsync(filePath);
        const stats = NDJsonStats.create({
            tookMillis: Date.now() - started,
            rows,
            sizeBytes,
        });
        console.log(`>> ${grey(filePath)}\n` + stats.toPretty());
        statsPerTable[table] = stats;
    }, { concurrency, errorMode });
    const statsTotal = NDJsonStats.createCombined(Object.values(statsPerTable));
    console.log(statsTotal.toPretty('total'));
    return statsTotal;
}