@naturalcycles/nodejs-lib/dist/stream/ndjson/pipelineToNDJsonFile.js

Standard library for Node.js

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.pipelineToNDJsonFile = void 0;
const zlib_1 = require("zlib");
const js_lib_1 = require("@naturalcycles/js-lib");
const fs = require("fs-extra");
const __1 = require("../..");
const colors_1 = require("../../colors");
const ndjson_model_1 = require("./ndjson.model");
const transformToNDJson_1 = require("./transformToNDJson");
/**
 * Convenience pipeline to transform stream of objects into a file in NDJSON format.
 *
 * Does fs.ensureFile() before starting, which will create all needed directories and truncate the file if it existed.
 */
async function pipelineToNDJsonFile(streams, opt) {
    const { filePath, gzip, protectFromOverwrite = false } = opt;
    if (protectFromOverwrite && fs.pathExistsSync(filePath)) {
        throw new js_lib_1.AppError(`pipelineToNDJsonFile: output file exists: ${filePath}`);
    }
    const started = Date.now();
    let rows = 0;
    fs.ensureFileSync(filePath);
    console.log(`>> ${(0, colors_1.grey)(filePath)} started...`);
    await (0, __1._pipeline)([
        ...streams,
        (0, __1.transformTap)(() => rows++),
        (0, transformToNDJson_1.transformToNDJson)(opt),
        ...(gzip ? [(0, zlib_1.createGzip)(opt.zlibOptions)] : []),
        fs.createWriteStream(filePath),
    ]);
    const { size: sizeBytes } = fs.statSync(filePath);
    const stats = ndjson_model_1.NDJsonStats.create({
        tookMillis: Date.now() - started,
        rows,
        sizeBytes,
    });
    console.log(`>> ${(0, colors_1.grey)(filePath)}\n` + stats.toPretty());
    return stats;
}
exports.pipelineToNDJsonFile = pipelineToNDJsonFile;