@pujansrt/data-genie
Version:
High performant ETL engine written in TypeScript
85 lines (84 loc) • 3.58 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.NDJsonReader = void 0;
const fs_1 = require("fs");
const readline = __importStar(require("readline"));
/**
* NDJsonReader class for reading data records from a file in NDJSON (Newline Delimited JSON) format.
* Each line in the file is expected to be a valid JSON object.
* This class reads records incrementally, suitable for very large files.
*/
class NDJsonReader {
/**
* Constructs a new NDJsonReader.
* @param filePath The path to the NDJSON file.
*/
constructor(filePath) {
this.filePath = filePath;
}
/**
* Reads data records from the NDJSON file asynchronously, yielding each record as it's parsed.
* @returns An AsyncIterableIterator of DataRecord objects.
*/
async *read() {
const stream = (0, fs_1.createReadStream)(this.filePath, { encoding: 'utf8' });
const rl = readline.createInterface({
input: stream,
crlfDelay: Infinity // Recognizes both CRLF and LF as line endings
});
for await (const line of rl) {
// Skip empty lines that might occur (e.g., at the end of the file or accidental blank lines)
if (!line.trim()) {
continue;
}
try {
// Parse each non-empty line as a JSON object
const record = JSON.parse(line);
yield record;
}
catch (error) {
// Log an error or throw if a line cannot be parsed as valid JSON
console.error(`NDJsonReader: Error parsing JSON line from ${this.filePath}: "${line.substring(0, 100)}..."`, error);
// Depending on your requirements, you might choose to:
// 1. continue; // Skip the malformed line and proceed
// 2. throw error; // Stop processing and propagate the error
// For a reader, skipping might be preferred to allow reading remaining valid data.
throw new Error(`Invalid JSON format in file ${this.filePath} on line: ${line.substring(0, 50)}...`);
}
}
}
}
exports.NDJsonReader = NDJsonReader;