@naturalcycles/db-lib/dist/pipeline/dbPipelineCopy.js

Lowest Common Denominator API to supported Databases

import { _passthroughMapper, ErrorMode, localTime, pMap } from '@naturalcycles/js-lib';
import { _pipeline, boldWhite, dimWhite, grey, NDJsonStats, transformChunk, transformLogProgress, transformMap, transformTap, writableForEach, yellow, } from '@naturalcycles/nodejs-lib';
import { DBQuery } from '../query/dbQuery.js';
/**
 * Pipeline from input stream(s) to CommonDB .saveBatch().
 * Input stream can be a stream from CommonDB.streamQuery()
 * Allows to define a mapper and a predicate to map/filter objects between input and output.
 * Handles backpressure.
 */
export async function dbPipelineCopy(opt) {
    const { chunkSize = 100, dbInput, dbOutput, concurrency = 16, limit = 0, sinceUpdated, mapperPerTable = {}, saveOptionsPerTable = {}, transformMapOptions, errorMode = ErrorMode.SUPPRESS, } = opt;
    let { tables } = opt;
    const sinceUpdatedStr = sinceUpdated ? ' since ' + grey(localTime(sinceUpdated).toPretty()) : '';
    console.log(`>> ${dimWhite('dbPipelineCopy')} started...${sinceUpdatedStr}`);
    tables ||= await dbInput.getTables();
    console.log(`${yellow(tables.length)} ${boldWhite('table(s)')}:\n` + tables.join('\n'));
    const statsPerTable = {};
    await pMap(tables, async (table) => {
        let q = DBQuery.create(table).limit(limit);
        if (sinceUpdated) {
            q = q.filter('updated', '>=', sinceUpdated);
        }
        const saveOptions = saveOptionsPerTable[table] || {};
        const mapper = mapperPerTable[table] || _passthroughMapper;
        const stream = dbInput.streamQuery(q);
        const started = Date.now();
        let rows = 0;
        await _pipeline([
            stream,
            transformLogProgress({
                logEvery: 1000,
                ...opt,
                metric: table,
            }),
            transformMap(mapper, {
                errorMode,
                flattenArrayOutput: true,
                ...transformMapOptions,
                metric: table,
            }),
            transformTap(() => rows++),
            transformChunk({ chunkSize }),
            writableForEach(async (dbms) => {
                await dbOutput.saveBatch(table, dbms, saveOptions);
            }),
        ]);
        const stats = NDJsonStats.create({
            tookMillis: Date.now() - started,
            rows,
            sizeBytes: 0, // n/a
        });
        console.log(`>> ${grey(table)}\n` + stats.toPretty());
        statsPerTable[table] = stats;
    }, { concurrency, errorMode });
    const statsTotal = NDJsonStats.createCombined(Object.values(statsPerTable));
    console.log(statsTotal.toPretty('total'));
    return statsTotal;
}