rdf-dataset-fragmenter/lib/io/QuadSinkHdt.js

Fragments an RDF dataset into multiple parts

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.QuadSinkHdt = void 0;
const promises_1 = require("node:fs/promises");
const node_path_1 = require("node:path");
const node_stream_1 = require("node:stream");
const Docker = require("dockerode");
const streamToString = require("stream-to-string");
const QuadSinkFile_1 = require("./QuadSinkFile");
const HDTCPP_MOUNT_PATH = '/tmp/convert';
const HDTCPP_DOCKER_IMAGE = 'rdfhdt/hdt-cpp:latest';
const HDTCPP_FORMATS = new Map([
    ['application/n-quads', 'nquad'],
    ['application/n-triples', 'ntriples'],
    ['text/turtle', 'turtle'],
    ['application/rdf+xml', 'rdfxml'],
    ['text/n3', 'n3'],
]);
class QuadSinkHdt extends QuadSinkFile_1.QuadSinkFile {
    constructor(options) {
        super(options);
        if (!HDTCPP_FORMATS.has(options.outputFormat)) {
            throw new Error(`Unsupported HDT output format ${options.outputFormat}`);
        }
        this.generateIndexes = options.generateIndexes ?? true;
        this.removeSourceFiles = options.removeSourceFiles ?? true;
        this.conversionConcurrency = options.conversionConcurrency ?? 1;
        this.docker = new Docker();
        this.filesToConvert = new Set();
    }
    getFilePath(iri) {
        const path = super.getFilePath(iri);
        this.filesToConvert.add((0, node_path_1.resolve)(path));
        return path;
    }
    /**
     * Runs the equivalent of `docker pull` for the rdf2hdt Docker image
     */
    async pullDockerImage() {
        await new Promise((resolve, reject) => this.docker.pull(HDTCPP_DOCKER_IMAGE, {}, (error, result) => {
            if (error) {
                reject(error);
            }
            this.docker.modem.followProgress(result, error => error ? reject(error) : resolve());
        }));
    }
    /**
     * Runs the equivalent of `docker run` to convert `rdfFilePath` into a HDT file
     * @param rdfFilePath The path to the RDF file to convert
     */
    async convertSingleFile(rdfFilePath) {
        const cmd = [
            'rdf2hdt',
            '-p',
            ...this.generateIndexes ? ['-i'] : [],
            `${HDTCPP_MOUNT_PATH}/${(0, node_path_1.basename)(rdfFilePath)}`,
            `${HDTCPP_MOUNT_PATH}/${(0, node_path_1.basename)(rdfFilePath).replace(this.fileExtension ?? '', '')}.hdt`,
        ];
        const createOptions = {
            // eslint-disable-next-line ts/naming-convention
            HostConfig: {
                // eslint-disable-next-line ts/naming-convention
                AutoRemove: true,
                // eslint-disable-next-line ts/naming-convention
                Binds: [`${(0, node_path_1.dirname)(rdfFilePath)}:${HDTCPP_MOUNT_PATH}:rw`],
            },
        };
        const passThrough = new node_stream_1.PassThrough();
        const passThroughToString = streamToString(passThrough);
        // eslint-disable-next-line ts/naming-convention
        const resultStatus = (await this.docker.run(HDTCPP_DOCKER_IMAGE, cmd, passThrough, createOptions))[0].StatusCode;
        if (resultStatus) {
            throw new Error(`Failed to convert ${rdfFilePath}:\n${await passThroughToString}`);
        }
        if (this.removeSourceFiles) {
            await (0, promises_1.unlink)(rdfFilePath);
        }
        return rdfFilePath;
    }
    /**
     * Converts all files stored in `filesToConvert` into HDT format using the Docker image,
     * with a maximum concurrency of `conversionConcurrency`
     */
    async convertToHdt() {
        await this.pullDockerImage();
        const runningConversions = new Map();
        let convertedFiles = 0;
        for (const rdfFilePath of this.filesToConvert) {
            runningConversions.set(rdfFilePath, this.convertSingleFile(rdfFilePath));
            if (runningConversions.size === this.conversionConcurrency) {
                runningConversions.delete(await Promise.race(runningConversions.values()));
                this.attemptConversionLog(convertedFiles++);
            }
        }
        await Promise.all(runningConversions.values());
        this.attemptConversionLog(convertedFiles + runningConversions.size);
    }
    attemptConversionLog(count) {
        if (this.log) {
            const lineTerminator = count === this.filesToConvert.size ? '\n' : '';
            process.stdout.write(`\rConverted files: ${count}/${this.filesToConvert.size}${lineTerminator}`);
        }
    }
    async close() {
        await super.close();
        await this.convertToHdt();
    }
}
exports.QuadSinkHdt = QuadSinkHdt;
//# sourceMappingURL=QuadSinkHdt.js.map