json-file-merger
Version:
A Node.js package for merging large JSON files using streams
82 lines • 3.56 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.getJsonFilesFromDirectory = exports.formatBytes = exports.calculateOptimalChunkSize = exports.getFileSize = void 0;
exports.mergeJsonFiles = mergeJsonFiles;
const fs_1 = require("fs");
const stream_1 = require("stream");
const json_stream_merger_1 = require("./streams/json-stream-merger");
const file_utils_1 = require("./utils/file-utils");
Object.defineProperty(exports, "getFileSize", { enumerable: true, get: function () { return file_utils_1.getFileSize; } });
Object.defineProperty(exports, "calculateOptimalChunkSize", { enumerable: true, get: function () { return file_utils_1.calculateOptimalChunkSize; } });
Object.defineProperty(exports, "formatBytes", { enumerable: true, get: function () { return file_utils_1.formatBytes; } });
Object.defineProperty(exports, "getJsonFilesFromDirectory", { enumerable: true, get: function () { return file_utils_1.getJsonFilesFromDirectory; } });
/**
* Merges multiple JSON files into a single JSON array
* @param inputFiles Array of input file paths
* @param outputFile Path to the output file
* @param options Optional configuration
*/
async function mergeJsonFiles(inputFiles, outputFile, options = {}) {
if (!inputFiles.length) {
throw new Error('No input files provided');
}
// Create writable stream with a large buffer
const outputStream = (0, fs_1.createWriteStream)(outputFile, {
highWaterMark: 16 * 1024 * 1024 // 16MB buffer
});
// Calculate total size of all files
let totalBytes = 0;
for (const file of inputFiles) {
totalBytes += await (0, file_utils_1.getFileSize)(file);
}
// Create the merger stream
const merger = new json_stream_merger_1.JsonStreamMerger({ onProgress: options.onProgress });
merger.totalBytes = totalBytes;
// Debug information
const debugInfo = {
firstLines: [],
lastLines: [],
lineCount: 0
};
// Create a transform stream to capture debug info
const debugStream = new stream_1.Transform({
transform(chunk, encoding, callback) {
const lines = chunk.toString().split('\n');
debugInfo.lineCount += lines.length;
if (debugInfo.firstLines.length < 5) {
debugInfo.firstLines.push(...lines.slice(0, 5 - debugInfo.firstLines.length));
}
debugInfo.lastLines = lines.slice(-5);
callback(null, chunk);
}
});
// Calculate optimal chunk size
const chunkSize = (0, file_utils_1.calculateOptimalChunkSize)(totalBytes);
// Pipe the streams together
merger
.pipe(debugStream)
.pipe(outputStream);
// Process each input file
for (const file of inputFiles) {
const readStream = (0, fs_1.createReadStream)(file, {
highWaterMark: chunkSize
});
await new Promise((resolve, reject) => {
readStream.on('error', reject);
readStream.pipe(merger, { end: false });
readStream.on('end', () => resolve());
});
}
// Finalize the merge
await new Promise((resolve, reject) => {
outputStream.on('error', reject);
outputStream.on('finish', resolve);
merger.end();
});
// Log debug information
console.log('\nDebug Information:');
console.log('First 5 lines:', debugInfo.firstLines);
console.log('Last 5 lines:', debugInfo.lastLines);
console.log('Total lines:', debugInfo.lineCount);
}
//# sourceMappingURL=index.js.map