@naturalcycles/nodejs-lib/dist/stream/ndjson/ndjsonMap.js

Standard library for Node.js

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.ndjsonMap = void 0;
const fs_1 = require("fs");
const zlib_1 = require("zlib");
const js_lib_1 = require("@naturalcycles/js-lib");
const __1 = require("../..");
/**
 * Unzips input file automatically, if it ends with `.gz`.
 * Zips output file automatically, if it ends with `.gz`.
 */
async function ndjsonMap(mapper, opt) {
    const { inputFilePath, outputFilePath, logEveryOutput = 100000, limitInput, limitOutput } = opt;
    (0, __1.requireFileToExist)(inputFilePath);
    console.log({
        inputFilePath,
        outputFilePath,
    });
    const transformUnzip = inputFilePath.endsWith('.gz') ? [(0, zlib_1.createUnzip)()] : [];
    const transformZip = outputFilePath.endsWith('.gz') ? [(0, zlib_1.createGzip)()] : [];
    const readable = (0, fs_1.createReadStream)(inputFilePath);
    await (0, __1._pipeline)([
        readable,
        ...transformUnzip,
        (0, __1.transformSplit)(),
        (0, __1.transformJsonParse)(),
        (0, __1.transformLimit)({ limit: limitInput, sourceReadable: readable }),
        (0, __1.transformLogProgress)({ metric: 'read', ...opt }),
        (0, __1.transformMap)(mapper, {
            flattenArrayOutput: true,
            errorMode: js_lib_1.ErrorMode.SUPPRESS,
            ...opt,
        }),
        (0, __1.transformLimit)({ limit: limitOutput, sourceReadable: readable }),
        (0, __1.transformLogProgress)({ metric: 'saved', logEvery: logEveryOutput }),
        (0, __1.transformToNDJson)(),
        ...transformZip,
        (0, fs_1.createWriteStream)(outputFilePath),
    ]);
}
exports.ndjsonMap = ndjsonMap;