@naturalcycles/nodejs-lib
Version:
Standard library for Node.js
51 lines • 1.77 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
const fs_1 = require("fs");
const path = require("path");
const zlib_1 = require("zlib");
const __1 = require("../..");
/**
* Unzips input file automatically, if it ends with `.gz`.
* Zips output file automatically, if it ends with `.gz`.
*/
async function ndjsonMap(opt) {
const { inputFilePath, outputFilePath, mapperFilePath, logEvery, limit } = opt;
await __1.requireFileToExist(inputFilePath);
await __1.requireFileToExist(mapperFilePath);
const resolvedMapperPath = path.resolve(mapperFilePath);
console.log({
inputFilePath,
outputFilePath,
mapperFilePath,
resolvedMapperPath,
});
// This is to allow importing *.ts mappers
try {
require('ts-node/register/transpile-only');
require('tsconfig-paths/register');
}
catch { } // require if exists
const { mapper } = require(resolvedMapperPath);
if (!mapper) {
throw new Error(`Mapper file should export "mapper" function`);
}
const transformUnzip = inputFilePath.endsWith('.gz') ? [zlib_1.createUnzip()] : [];
const transformZip = outputFilePath.endsWith('.gz') ? [zlib_1.createGzip()] : [];
await __1._pipeline([
fs_1.createReadStream(inputFilePath),
...transformUnzip,
__1.transformSplit(),
__1.transformJsonParse(),
__1.transformLimit(limit),
__1.transformMap(mapper, {
flattenArrayOutput: true,
...opt,
}),
__1.transformLogProgress({ logEvery }),
__1.transformToNDJson(),
...transformZip,
fs_1.createWriteStream(outputFilePath),
]);
}
exports.ndjsonMap = ndjsonMap;
//# sourceMappingURL=ndjsonMap.js.map