@naturalcycles/nodejs-lib
Version:
Standard library for Node.js
41 lines • 1.66 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
const js_lib_1 = require("@naturalcycles/js-lib");
const fs = require("fs-extra");
const zlib_1 = require("zlib");
const __1 = require("../..");
const colors_1 = require("../../colors");
const ndjson_model_1 = require("./ndjson.model");
const transformToNDJson_1 = require("./transformToNDJson");
/**
* Convenience pipeline to transform stream of objects into a file in NDJSON format.
*
* Does fs.ensureFile() before starting, which will create all needed directories and truncate the file if it existed.
*/
async function pipelineToNDJsonFile(streams, opt) {
const { filePath, gzip, protectFromOverwrite = false } = opt;
if (protectFromOverwrite && (await fs.pathExists(filePath))) {
throw new js_lib_1.AppError(`pipelineToNDJsonFile: output file exists: ${filePath}`);
}
const started = Date.now();
let rows = 0;
await fs.ensureFile(filePath);
console.log(`>> ${colors_1.grey(filePath)} started...`);
await __1._pipeline([
...streams,
__1.transformTap(() => rows++),
transformToNDJson_1.transformToNDJson(opt),
...(gzip ? [zlib_1.createGzip(opt.zlibOptions)] : []),
fs.createWriteStream(filePath),
]);
const { size: sizeBytes } = await fs.stat(filePath);
const stats = ndjson_model_1.NDJsonStats.create({
tookMillis: Date.now() - started,
rows,
sizeBytes,
});
console.log(`>> ${colors_1.grey(filePath)}\n` + stats.toPretty());
return stats;
}
exports.pipelineToNDJsonFile = pipelineToNDJsonFile;
//# sourceMappingURL=pipelineToNDJsonFile.js.map