@naturalcycles/nodejs-lib/dist/stream/ndjson/pipelineFromNDJsonFile.js

Standard library for Node.js

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const js_lib_1 = require("@naturalcycles/js-lib");
const fs = require("fs-extra");
const zlib_1 = require("zlib");
const __1 = require("../..");
const __2 = require("../..");
const colors_1 = require("../../colors");
const ndjson_model_1 = require("./ndjson.model");
const transformJsonParse_1 = require("./transformJsonParse");
/**
 * Convenience pipeline that starts from reading NDJSON file.
 */
async function pipelineFromNDJsonFile(streams, opt) {
    const { filePath, gzip, separator } = opt;
    const started = Date.now();
    let rows = 0;
    const { size: sizeBytes } = await fs.stat(filePath);
    console.log(`<< ${colors_1.grey(filePath)} ${colors_1.dimWhite(js_lib_1._hb(sizeBytes))} started...`);
    await __1._pipeline([
        fs.createReadStream(filePath),
        ...(gzip ? [zlib_1.createUnzip(opt.zlibOptions)] : []),
        __2.transformSplit(separator),
        transformJsonParse_1.transformJsonParse(opt),
        __1.transformTap(() => rows++),
        ...streams,
    ]);
    const stats = ndjson_model_1.NDJsonStats.create({
        tookMillis: Date.now() - started,
        rows,
        sizeBytes,
    });
    console.log(`<< ${colors_1.grey(filePath)}\n` + stats.toPretty());
    return stats;
}
exports.pipelineFromNDJsonFile = pipelineFromNDJsonFile;
//# sourceMappingURL=pipelineFromNDJsonFile.js.map