json-file-merger
Version:
A Node.js package for merging large JSON files using streams
123 lines • 3.55 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.JsonStreamMerger = void 0;
const stream_1 = require("stream");
/**
* A Transform stream that merges multiple JSON files into a single JSON array
*/
class JsonStreamMerger extends stream_1.Transform {
constructor(options = {}) {
super();
this.isFirstFile = true;
this.processedBytes = 0;
this.totalBytes = 0;
this.arrayStartWritten = false;
this.isInterrupted = false;
this.lastChar = '';
this.buffer = '';
this.startTime = Date.now();
this.onProgress = options.onProgress || (() => { });
}
/**
* Process a chunk of data
*/
_transform(chunk, encoding, callback) {
try {
this.processedBytes += chunk.length;
this._writeArrayStart();
this._processChunk(chunk.toString());
this._reportProgress();
callback();
}
catch (error) {
callback(error instanceof Error ? error : new Error(String(error)));
}
}
/**
* Write the opening array bracket if not already written
*/
_writeArrayStart() {
if (!this.arrayStartWritten) {
this.push('[\n');
this.arrayStartWritten = true;
}
}
/**
* Process a chunk of text data
*/
_processChunk(chunkStr) {
const fullChunk = this.buffer + chunkStr;
this.buffer = '';
const lines = fullChunk.split('\n');
for (const line of lines) {
const trimmedLine = line.trim();
if (!trimmedLine)
continue;
if (this._shouldSkipLine(trimmedLine))
continue;
this._writeLine(trimmedLine);
}
}
/**
* Check if a line should be skipped
*/
_shouldSkipLine(line) {
return line === '[' || line === ']';
}
/**
* Write a line with proper formatting
*/
_writeLine(line) {
if (this.lastChar === '}' && line.startsWith('{')) {
this.push(',\n');
}
this.push(line + '\n');
this.lastChar = line[line.length - 1];
}
/**
* Report progress information
*/
_reportProgress() {
if (this.totalBytes > 0) {
const progress = (this.processedBytes / this.totalBytes) * 100;
const elapsed = (Date.now() - this.startTime) / 1000;
const speed = this.processedBytes / elapsed / 1024 / 1024;
const progressInfo = {
progress,
processedBytes: this.processedBytes,
speed,
bufferInfo: {
bufferSize: this.buffer.length,
bufferGrowth: 0
}
};
this.onProgress(progressInfo);
}
}
/**
* Flush any remaining data
*/
_flush(callback) {
try {
if (this.buffer) {
this._processChunk('');
}
if (this.arrayStartWritten && !this.isInterrupted) {
this.push(']');
}
callback();
}
catch (error) {
callback(error instanceof Error ? error : new Error(String(error)));
}
}
/**
* Handle stream destruction
*/
_destroy(err, callback) {
this.isInterrupted = true;
callback(err);
}
}
exports.JsonStreamMerger = JsonStreamMerger;
//# sourceMappingURL=json-stream-merger.js.map