rdf-dataset-fragmenter
Version:
Fragments an RDF dataset into multiple parts
125 lines • 5.35 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.ParallelFileWriter = void 0;
const fs = __importStar(require("node:fs"));
const node_path_1 = require("node:path");
const node_stream_1 = require("node:stream");
// eslint-disable-next-line ts/no-require-imports
const AsyncLock = require("async-lock");
const lru_cache_1 = require("lru-cache");
const mkdirp_1 = require("mkdirp");
const rdf_serialize_1 = require("rdf-serialize");
/**
* A parallel file writer enables the writing to an infinite number of files in parallel.
*
* It works around system I/O limitations regarding the maximum number of open file descriptors
* by internally only having a certain number of open file streams,
* and intelligently closing/re-opening streams when needed using an LRU strategy.
*/
class ParallelFileWriter {
cache;
lock;
fileClosingPromises;
constructor(options) {
this.cache = new lru_cache_1.LRUCache({
max: options.streams,
dispose: (value, key) => this.closeWriteEntry(key, value),
noDisposeOnSet: true,
});
this.lock = new AsyncLock();
this.fileClosingPromises = [];
}
/**
* Get a write stream that accepts RDF/JS quads for the given file path.
* It will automatically be serialized to the RDF format of the given content type.
*
* The returned stream is only safe to use until another call to this method.
*
* This is safe with regards to non-existing folders.
* If any of the parent folders do not exist, they will be created.
*
* @param path Path to the file to write to.
* @param contentType The content type to serialize for.
* Note that this only should be content types that enable appending
*/
async getWriteStream(path, contentType) {
return this.lock.acquire('getWriteStream', () => this.getWriteStreamUnsafe(path, contentType));
}
async getWriteStreamUnsafe(path, contentType) {
// Try to get the stream from cache, or open a new one if not yet open.
let writeEntry = this.cache.get(path);
if (!writeEntry) {
// Before opening new streams, wait for previous file closings to end
await Promise.all(this.fileClosingPromises);
this.fileClosingPromises = [];
// Open the file stream, and prepare the RDF serializer
const writeStream = new node_stream_1.PassThrough({ objectMode: true });
const folder = (0, node_path_1.dirname)(path);
await (0, mkdirp_1.mkdirp)(folder);
const fileStream = fs.createWriteStream(path, { flags: 'a' });
rdf_serialize_1.rdfSerializer.serialize(writeStream, { contentType }).pipe(fileStream);
writeEntry = { writeStream, fileStream };
this.cache.set(path, writeEntry);
}
return writeEntry.writeStream;
}
/**
* Close all open streams.
*/
async close() {
const outputStreamPromises = [];
// eslint-disable-next-line unicorn/no-array-for-each
this.cache.forEach((entry) => {
// Wait asynchronously for the file stream associated with the current write stream to be close
outputStreamPromises.push(new Promise((resolve, reject) => {
entry.fileStream.on('finish', resolve);
entry.fileStream.on('error', reject);
}));
// Close the current write stream
entry.writeStream.end();
});
await Promise.all(outputStreamPromises);
}
closeWriteEntry(path, writeEntry) {
this.fileClosingPromises.push(new Promise((resolve, reject) => {
writeEntry.fileStream.on('finish', resolve);
writeEntry.fileStream.on('error', reject);
}));
writeEntry.writeStream.end();
}
}
exports.ParallelFileWriter = ParallelFileWriter;
//# sourceMappingURL=ParallelFileWriter.js.map