@naturalcycles/db-lib/dist/pipeline/dbPipelineCopy.js

Lowest Common Denominator API to supported Databases

import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js';
import { ErrorMode } from '@naturalcycles/js-lib/error/errorMode.js';
import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
import { _passthroughMapper } from '@naturalcycles/js-lib/types';
import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors';
import { NDJsonStats } from '@naturalcycles/nodejs-lib/stream';
import { DBQuery } from '../query/dbQuery.js';
/**
 * Pipeline from input stream(s) to CommonDB .saveBatch().
 * Input stream can be a stream from CommonDB.streamQuery()
 * Allows to define a mapper and a predicate to map/filter objects between input and output.
 * Handles backpressure.
 */
export async function dbPipelineCopy(opt) {
    const { chunkSize = 100, dbInput, dbOutput, concurrency = 16, limit = 0, sinceUpdated, mapperPerTable = {}, saveOptionsPerTable = {}, transformMapOptions, errorMode = ErrorMode.SUPPRESS, } = opt;
    let { tables } = opt;
    const sinceUpdatedStr = sinceUpdated ? ' since ' + grey(localTime(sinceUpdated).toPretty()) : '';
    console.log(`>> ${dimWhite('dbPipelineCopy')} started...${sinceUpdatedStr}`);
    tables ||= await dbInput.getTables();
    console.log(`${yellow(tables.length)} ${boldWhite('table(s)')}:\n` + tables.join('\n'));
    const statsPerTable = {};
    await pMap(tables, async (table) => {
        let q = DBQuery.create(table).limit(limit);
        if (sinceUpdated) {
            q = q.filter('updated', '>=', sinceUpdated);
        }
        const saveOptions = saveOptionsPerTable[table] || {};
        const mapper = mapperPerTable[table] || _passthroughMapper;
        const started = Date.now();
        let rows = 0;
        await dbInput
            .streamQuery(q)
            .logProgress({
            logEvery: 1000,
            ...opt,
            metric: table,
        })
            .map(mapper, {
            errorMode,
            ...transformMapOptions,
            metric: table,
        })
            .flattenIfNeeded()
            .tapSync(() => rows++)
            .chunk(chunkSize)
            .forEach(async (dbms) => {
            await dbOutput.saveBatch(table, dbms, saveOptions);
        });
        const stats = NDJsonStats.create({
            tookMillis: Date.now() - started,
            rows,
            sizeBytes: 0, // n/a
        });
        console.log(`>> ${grey(table)}\n` + stats.toPretty());
        statsPerTable[table] = stats;
    }, { concurrency, errorMode });
    const statsTotal = NDJsonStats.createCombined(Object.values(statsPerTable));
    console.log(statsTotal.toPretty('total'));
    return statsTotal;
}