@naturalcycles/nodejs-lib/dist/stream/ndjson/pipelineFromNDJsonFile.js

Standard library for Node.js

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.pipelineFromNDJsonFile = void 0;
const zlib_1 = require("zlib");
const js_lib_1 = require("@naturalcycles/js-lib");
const fs = require("fs-extra");
const __1 = require("../..");
const colors_1 = require("../../colors");
const ndjson_model_1 = require("./ndjson.model");
const transformJsonParse_1 = require("./transformJsonParse");
/**
 * Convenience pipeline that starts from reading NDJSON file.
 */
async function pipelineFromNDJsonFile(streams, opt) {
    const { filePath, gzip, separator } = opt;
    const started = Date.now();
    let rows = 0;
    const { size: sizeBytes } = fs.statSync(filePath);
    console.log(`<< ${(0, colors_1.grey)(filePath)} ${(0, colors_1.dimWhite)((0, js_lib_1._hb)(sizeBytes))} started...`);
    await (0, __1._pipeline)([
        fs.createReadStream(filePath),
        ...(gzip ? [(0, zlib_1.createUnzip)(opt.zlibOptions)] : []),
        (0, __1.transformSplit)(separator),
        (0, transformJsonParse_1.transformJsonParse)(opt),
        (0, __1.transformTap)(() => rows++),
        ...streams,
    ]);
    const stats = ndjson_model_1.NDJsonStats.create({
        tookMillis: Date.now() - started,
        rows,
        sizeBytes,
    });
    console.log(`<< ${(0, colors_1.grey)(filePath)}\n` + stats.toPretty());
    return stats;
}
exports.pipelineFromNDJsonFile = pipelineFromNDJsonFile;