@naturalcycles/db-lib
Version:
Lowest Common Denominator API to supported Databases
87 lines (86 loc) • 4.33 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.dbPipelineBackup = dbPipelineBackup;
const js_lib_1 = require("@naturalcycles/js-lib");
const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
const index_1 = require("../index");
/**
* Pipeline from input stream(s) to a NDJSON file (optionally gzipped).
* File is overwritten (by default).
* Input stream can be a stream from CommonDB.streamQuery()
* Allows to define a mapper and a predicate to map/filter objects between input and output.
* Handles backpressure.
*
* Optionally you can provide mapperPerTable and @param transformMapOptions (one for all mappers) - it will run for each table.
*/
async function dbPipelineBackup(opt) {
const { db, concurrency = 16, limit = 0, outputDirPath, protectFromOverwrite = false, mapperPerTable = {}, queryPerTable = {}, logEveryPerTable = {}, transformMapOptions, errorMode = js_lib_1.ErrorMode.SUPPRESS, emitSchemaFromDB = false, } = opt;
const gzip = opt.gzip !== false; // default to true
let { tables } = opt;
console.log(`>> ${(0, nodejs_lib_1.dimWhite)('dbPipelineBackup')} started in ${(0, nodejs_lib_1.grey)(outputDirPath)}...`);
nodejs_lib_1.fs2.ensureDir(outputDirPath);
tables ||= await db.getTables();
console.log(`${(0, nodejs_lib_1.yellow)(tables.length)} ${(0, nodejs_lib_1.boldWhite)('table(s)')}:\n` + tables.join('\n'));
const statsPerTable = {};
await (0, js_lib_1.pMap)(tables, async (table) => {
let q = index_1.DBQuery.create(table).limit(limit);
const sinceUpdated = opt.sinceUpdatedPerTable?.[table] ?? opt.sinceUpdated;
if (sinceUpdated) {
q = q.filter('updated', '>=', sinceUpdated);
}
if (queryPerTable[table]) {
// Override the Query with this Query, completely ingoring any of the other query-related options
q = queryPerTable[table];
console.log(`>> ${(0, nodejs_lib_1.grey)(table)} ${q.pretty()}`);
}
else {
const sinceUpdatedStr = sinceUpdated
? ' since ' + (0, nodejs_lib_1.grey)((0, js_lib_1.localTime)(sinceUpdated).toPretty())
: '';
console.log(`>> ${(0, nodejs_lib_1.grey)(table)}${sinceUpdatedStr}`);
}
const filePath = `${outputDirPath}/${table}.ndjson` + (gzip ? '.gz' : '');
const schemaFilePath = `${outputDirPath}/${table}.schema.json`;
if (protectFromOverwrite && nodejs_lib_1.fs2.pathExists(filePath)) {
throw new js_lib_1.AppError(`dbPipelineBackup: output file exists: ${filePath}`);
}
const started = Date.now();
let rows = 0;
nodejs_lib_1.fs2.ensureFile(filePath);
// console.log(`>> ${grey(filePath)} started...`)
if (emitSchemaFromDB) {
const schema = await db.getTableSchema(table);
await nodejs_lib_1.fs2.writeJsonAsync(schemaFilePath, schema, { spaces: 2 });
console.log(`>> ${(0, nodejs_lib_1.grey)(schemaFilePath)} saved (generated from DB)`);
}
await (0, nodejs_lib_1._pipeline)([
db.streamQuery(q),
(0, nodejs_lib_1.transformLogProgress)({
...opt,
logEvery: logEveryPerTable[table] ?? opt.logEvery ?? 1000,
metric: table,
}),
(0, nodejs_lib_1.transformMap)(mapperPerTable[table] || js_lib_1._passthroughMapper, {
errorMode,
flattenArrayOutput: true,
...transformMapOptions,
metric: table,
}),
(0, nodejs_lib_1.transformTap)(() => {
rows++;
}),
...nodejs_lib_1.fs2.createWriteStreamAsNDJSON(filePath),
]);
const { size: sizeBytes } = await nodejs_lib_1.fs2.statAsync(filePath);
const stats = nodejs_lib_1.NDJsonStats.create({
tookMillis: Date.now() - started,
rows,
sizeBytes,
});
console.log(`>> ${(0, nodejs_lib_1.grey)(filePath)}\n` + stats.toPretty());
statsPerTable[table] = stats;
}, { concurrency, errorMode });
const statsTotal = nodejs_lib_1.NDJsonStats.createCombined(Object.values(statsPerTable));
console.log(statsTotal.toPretty('total'));
return statsTotal;
}