rdf-dataset-fragmenter
Version:
Fragments an RDF dataset into multiple parts
83 lines • 2.94 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.QuadSinkFile = void 0;
const node_readline_1 = require("node:readline");
const ParallelFileWriter_1 = require("./ParallelFileWriter");
/**
* A quad sink that writes to files using an IRI to local file system path mapping.
*/
class QuadSinkFile {
outputFormat;
iriToPath;
fileWriter;
log;
fileExtension;
counter = 0;
constructor(options) {
this.outputFormat = options.outputFormat;
this.iriToPath = new Map(Object.entries(options.iriToPath).map(([exp, sub]) => [
new RegExp(exp, 'u'),
sub,
]));
this.log = Boolean(options.log);
this.fileExtension = options.fileExtension;
this.fileWriter = new ParallelFileWriter_1.ParallelFileWriter({ streams: 128 });
this.attemptLog();
}
attemptLog(newLine = false) {
if (this.log && (this.counter % 1_000 === 0 || newLine)) {
(0, node_readline_1.clearLine)(process.stdout, 0);
(0, node_readline_1.cursorTo)(process.stdout, 0);
process.stdout.write(`\rHandled quads: ${this.counter / 1_000}K`);
if (newLine) {
process.stdout.write(`\n`);
}
}
}
getFilePath(iri) {
// Remove hash fragment
const posHash = iri.indexOf('#');
if (posHash >= 0) {
iri = iri.slice(0, posHash);
}
// Find base path from the first matching baseIRI
let bestMatch;
let bestRegex;
for (const exp of this.iriToPath.keys()) {
const match = exp.exec(iri);
if (match && (bestMatch === undefined || match[0].length > bestMatch[0].length)) {
bestMatch = match;
bestRegex = exp;
}
}
// Crash if we did not find a matching baseIRI
if (!bestRegex) {
throw new Error(`No IRI mapping found for ${iri}`);
}
// Perform substitution and replace illegal directory names
let path = iri.replace(bestRegex, this.iriToPath.get(bestRegex));
// Replace illegal directory names
path = path.replaceAll(/[*|"<>?:]/ug, '_');
// Add file extension if we don't have one yet
if (this.fileExtension && !/\.[a-z]$/iu.test(this.fileExtension)) {
path = `${path}${this.fileExtension}`;
}
return path;
}
async getFileStream(path) {
return this.fileWriter.getWriteStream(path, this.outputFormat);
}
async push(iri, quad) {
this.counter++;
this.attemptLog();
const path = this.getFilePath(iri);
const os = await this.getFileStream(path);
os.write(quad);
}
async close() {
await this.fileWriter.close();
this.attemptLog(true);
}
}
exports.QuadSinkFile = QuadSinkFile;
//# sourceMappingURL=QuadSinkFile.js.map