@naturalcycles/db-lib
Version:
Lowest Common Denominator API to supported Databases
62 lines (61 loc) • 2.88 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.dbPipelineCopy = dbPipelineCopy;
const js_lib_1 = require("@naturalcycles/js-lib");
const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
const dbQuery_1 = require("../query/dbQuery");
/**
* Pipeline from input stream(s) to CommonDB .saveBatch().
* Input stream can be a stream from CommonDB.streamQuery()
* Allows to define a mapper and a predicate to map/filter objects between input and output.
* Handles backpressure.
*/
async function dbPipelineCopy(opt) {
const { chunkSize = 100, dbInput, dbOutput, concurrency = 16, limit = 0, sinceUpdated, mapperPerTable = {}, saveOptionsPerTable = {}, transformMapOptions, errorMode = js_lib_1.ErrorMode.SUPPRESS, } = opt;
let { tables } = opt;
const sinceUpdatedStr = sinceUpdated ? ' since ' + (0, nodejs_lib_1.grey)((0, js_lib_1.localTime)(sinceUpdated).toPretty()) : '';
console.log(`>> ${(0, nodejs_lib_1.dimWhite)('dbPipelineCopy')} started...${sinceUpdatedStr}`);
tables ||= await dbInput.getTables();
console.log(`${(0, nodejs_lib_1.yellow)(tables.length)} ${(0, nodejs_lib_1.boldWhite)('table(s)')}:\n` + tables.join('\n'));
const statsPerTable = {};
await (0, js_lib_1.pMap)(tables, async (table) => {
let q = dbQuery_1.DBQuery.create(table).limit(limit);
if (sinceUpdated) {
q = q.filter('updated', '>=', sinceUpdated);
}
const saveOptions = saveOptionsPerTable[table] || {};
const mapper = mapperPerTable[table] || js_lib_1._passthroughMapper;
const stream = dbInput.streamQuery(q);
const started = Date.now();
let rows = 0;
await (0, nodejs_lib_1._pipeline)([
stream,
(0, nodejs_lib_1.transformLogProgress)({
logEvery: 1000,
...opt,
metric: table,
}),
(0, nodejs_lib_1.transformMap)(mapper, {
errorMode,
flattenArrayOutput: true,
...transformMapOptions,
metric: table,
}),
(0, nodejs_lib_1.transformTap)(() => rows++),
(0, nodejs_lib_1.transformChunk)({ chunkSize }),
(0, nodejs_lib_1.writableForEach)(async (dbms) => {
await dbOutput.saveBatch(table, dbms, saveOptions);
}),
]);
const stats = nodejs_lib_1.NDJsonStats.create({
tookMillis: Date.now() - started,
rows,
sizeBytes: 0, // n/a
});
console.log(`>> ${(0, nodejs_lib_1.grey)(table)}\n` + stats.toPretty());
statsPerTable[table] = stats;
}, { concurrency, errorMode });
const statsTotal = nodejs_lib_1.NDJsonStats.createCombined(Object.values(statsPerTable));
console.log(statsTotal.toPretty('total'));
return statsTotal;
}