pg-rdf-to-json/dist/cli.js

Transforms RDF files from rdf-files.tar.bz2 provided by Project Gutenberg into JS/JSON objects.

#!/usr/bin/env node
import fs from 'node:fs/promises';
import path from 'node:path';
import { booksFromStream, booksFromArchive } from './index.js';
import { bookSchema } from './types.js';
import { parseArgs, inspect } from 'node:util';
import { Ajv } from 'ajv/dist/jtd.js';
import { extractProp } from './util.js';
function log(contents) {
    process.stdout.moveCursor(0, -1);
    process.stdout.cursorTo(0);
    process.stdout.clearLine(0);
    process.stdout.write(contents);
    process.stdout.write('\n');
}
function logError(contents) {
    return new Promise((resolve, reject) => {
        process.stderr.write(`${contents}\n`, err => {
            if (err) {
                reject(err);
            }
            else {
                resolve();
            }
        });
    });
}
async function booksToFiles(books, { options, output }) {
    let processed = 0;
    const createdDirectories = new Set();
    const ajv = new Ajv();
    const validate = ajv.compile(bookSchema);
    for await (const book of books) {
        if (options.validate) {
            validate(book);
            if (Array.isArray(validate.errors)) {
                const detailedErrors = validate.errors.map(error => ({
                    ...error,
                    value: extractProp(book, error.instancePath)
                }));
                await logError(`Validation error for book: ${book.about}`);
                for (const error of detailedErrors) {
                    await logError(inspect(error, { colors: true, depth: Infinity }));
                }
                process.exit(1);
            }
        }
        const bookId = book.about;
        const newFile = path.join(output, `${bookId}.json`);
        const newDir = path.dirname(newFile);
        if (!createdDirectories.has(newDir)) {
            await fs.mkdir(newDir, { recursive: true });
            createdDirectories.add(newDir);
        }
        await fs.writeFile(newFile, JSON.stringify(book, null, 2), 'utf8');
        const count = (++processed).toString().padStart(5, '0');
        log(`processed ${count}: ${bookId}`);
    }
    console.log('done');
}
async function booksToStdout(books) {
    const recordSeparator = String.fromCharCode(30);
    const lineFeed = String.fromCharCode(10);
    for await (const book of books) {
        process.stdout.write(recordSeparator);
        process.stdout.write(JSON.stringify(book));
        process.stdout.write(lineFeed);
    }
}
const { values: { input, output, ...options }, } = parseArgs({
    options: {
        validate: {
            type: 'boolean',
            short: 'v',
        },
        input: {
            type: 'string',
            short: 'i',
            default: '-',
        },
        output: {
            type: 'string',
            short: 'o',
        },
    }
});
const books = input !== '-' ?
    booksFromArchive(input) :
    booksFromStream(process.stdin);
if (output) {
    await booksToFiles(books, { options, output });
}
else {
    await booksToStdout(books);
}
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY2xpLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vc3JjL2NsaS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiO0FBRUEsT0FBTyxFQUFFLE1BQU0sa0JBQWtCLENBQUM7QUFDbEMsT0FBTyxJQUFJLE1BQU0sV0FBVyxDQUFDO0FBQzdCLE9BQU8sRUFBRSxlQUFlLEVBQUUsZ0JBQWdCLEVBQUUsTUFBTSxZQUFZLENBQUM7QUFDL0QsT0FBTyxFQUFhLFVBQVUsRUFBRSxNQUFNLFlBQVksQ0FBQztBQUNuRCxPQUFPLEVBQUUsU0FBUyxFQUFFLE9BQU8sRUFBRSxNQUFNLFdBQVcsQ0FBQztBQUMvQyxPQUFPLEVBQUUsR0FBRyxFQUFFLE1BQU0saUJBQWlCLENBQUM7QUFDdEMsT0FBTyxFQUFFLFdBQVcsRUFBRSxNQUFNLFdBQVcsQ0FBQztBQUV4QyxTQUFTLEdBQUcsQ0FBQyxRQUFnQjtJQUMzQixPQUFPLENBQUMsTUFBTSxDQUFDLFVBQVUsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxDQUFDLENBQUMsQ0FBQztJQUNqQyxPQUFPLENBQUMsTUFBTSxDQUFDLFFBQVEsQ0FBQyxDQUFDLENBQUMsQ0FBQztJQUMzQixPQUFPLENBQUMsTUFBTSxDQUFDLFNBQVMsQ0FBQyxDQUFDLENBQUMsQ0FBQztJQUM1QixPQUFPLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyxRQUFRLENBQUMsQ0FBQztJQUMvQixPQUFPLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyxJQUFJLENBQUMsQ0FBQztBQUM3QixDQUFDO0FBRUQsU0FBUyxRQUFRLENBQUMsUUFBZ0I7SUFDaEMsT0FBTyxJQUFJLE9BQU8sQ0FBQyxDQUFDLE9BQU8sRUFBRSxNQUFNLEVBQUUsRUFBRTtRQUNyQyxPQUFPLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyxHQUFHLFFBQVEsSUFBSSxFQUFFLEdBQUcsQ0FBQyxFQUFFO1lBQzFDLElBQUksR0FBRyxFQUFFLENBQUM7Z0JBQ1IsTUFBTSxDQUFDLEdBQUcsQ0FBQyxDQUFDO1lBQ2QsQ0FBQztpQkFBTSxDQUFDO2dCQUNOLE9BQU8sRUFBRSxDQUFDO1lBQ1osQ0FBQztRQUNILENBQUMsQ0FBQyxDQUFBO0lBQ0osQ0FBQyxDQUFDLENBQUM7QUFDTCxDQUFDO0FBRUQsS0FBSyxVQUFVLFlBQVksQ0FDekIsS0FBMkIsRUFDM0IsRUFDRSxPQUFPLEVBQ1AsTUFBTSxFQU1QO0lBRUQsSUFBSSxTQUFTLEdBQUcsQ0FBQyxDQUFDO0lBRWxCLE1BQU0sa0JBQWtCLEdBQUcsSUFBSSxHQUFHLEVBQUUsQ0FBQztJQUVyQyxNQUFNLEdBQUcsR0FBRyxJQUFJLEdBQUcsRUFBRSxDQUFDO0lBQ3RCLE1BQU0sUUFBUSxHQUFHLEdBQUcsQ0FBQyxPQUFPLENBQUMsVUFBVSxDQUFDLENBQUM7SUFFekMsSUFBSSxLQUFLLEVBQUUsTUFBTSxJQUFJLElBQUksS0FBSyxFQUFFLENBQUM7UUFDL0IsSUFBSSxPQUFPLENBQUMsUUFBUSxFQUFFLENBQUM7WUFDckIsUUFBUSxDQUFDLElBQUksQ0FBQyxDQUFDO1lBRWYsSUFBSSxLQUFLLENBQUMsT0FBTyxDQUFDLFFBQVEsQ0FBQyxNQUFNLENBQUMsRUFBRSxDQUFDO2dCQUNuQyxNQUFNLGNBQWMsR0FBRyxRQUFRLENBQUMsTUFBTSxDQUFDLEdBQUcsQ0FBQyxLQUFLLENBQUMsRUFBRSxDQUFDLENBQUM7b0JBQ25ELEdBQUcsS0FBSztvQkFDUixLQUFLLEVBQUUsV0FBVyxDQUFDLElBQUksRUFBRSxLQUFLLENBQUMsWUFBWSxDQUFDO2lCQUM3QyxDQUFDLENBQUMsQ0FBQztnQkFFSixNQUFNLFFBQVEsQ0FBQyw4QkFBOEIsSUFBSSxDQUFDLEtBQUssRUFBRSxDQUFDLENBQUM7Z0JBRTNELEtBQUssTUFBTSxLQUFLLElBQUksY0FBYyxFQUFFLENBQUM7b0JBQ25DLE1BQU0sUUFBUSxDQUFDLE9BQU8sQ0FBQyxLQUFLLEVBQUUsRUFBQyxNQUFNLEVBQUUsSUFBSSxFQUFFLEtBQUssRUFBRSxRQUFRLEVBQUMsQ0FBQyxDQUFDLENBQUM7Z0JBQ2xFLENBQUM7Z0JBRUQsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUMsQ0FBQztZQUNsQixDQUFDO1FBQ0gsQ0FBQztRQUVELE1BQU0sTUFBTSxHQUFHLElBQUksQ0FBQyxLQUFLLENBQUM7UUFDMUIsTUFBTSxPQUFPLEdBQUcsSUFBSSxDQUFDLElBQUksQ0FBQyxNQUFNLEVBQUUsR0FBRyxNQUFNLE9BQU8sQ0FBQyxDQUFDO1FBQ3BELE1BQU0sTUFBTSxHQUFHLElBQUksQ0FBQyxPQUFPLENBQUMsT0FBTyxDQUFDLENBQUM7UUFFckMsSUFBSSxDQUFDLGtCQUFrQixDQUFDLEdBQUcsQ0FBQyxNQUFNLENBQUMsRUFBRSxDQUFDO1lBQ3BDLE1BQU0sRUFBRSxDQUFDLEtBQUssQ0FBQyxNQUFNLEVBQUUsRUFBQyxTQUFTLEVBQUUsSUFBSSxFQUFDLENBQUMsQ0FBQztZQUMxQyxrQkFBa0IsQ0FBQyxHQUFHLENBQUMsTUFBTSxDQUFDLENBQUM7UUFDakMsQ0FBQztRQUVELE1BQU0sRUFBRSxDQUFDLFNBQVMsQ0FBQyxPQUFPLEVBQUUsSUFBSSxDQUFDLFNBQVMsQ0FBQyxJQUFJLEVBQUUsSUFBSSxFQUFFLENBQUMsQ0FBQyxFQUFFLE1BQU0sQ0FBQyxDQUFDO1FBRW5FLE1BQU0sS0FBSyxHQUFHLENBQUMsRUFBRSxTQUFTLENBQUMsQ0FBQyxRQUFRLEVBQUUsQ0FBQyxRQUFRLENBQUMsQ0FBQyxFQUFFLEdBQUcsQ0FBQyxDQUFDO1FBRXhELEdBQUcsQ0FBQyxhQUFhLEtBQUssS0FBSyxNQUFNLEVBQUUsQ0FBQyxDQUFDO0lBQ3ZDLENBQUM7SUFFRCxPQUFPLENBQUMsR0FBRyxDQUFDLE1BQU0sQ0FBQyxDQUFDO0FBQ3RCLENBQUM7QUFFRCxLQUFLLFVBQVUsYUFBYSxDQUFDLEtBQTJCO0lBQ3RELE1BQU0sZUFBZSxHQUFHLE1BQU0sQ0FBQyxZQUFZLENBQUMsRUFBRSxDQUFDLENBQUM7SUFDaEQsTUFBTSxRQUFRLEdBQUcsTUFBTSxDQUFDLFlBQVksQ0FBQyxFQUFFLENBQUMsQ0FBQztJQUV6QyxJQUFJLEtBQUssRUFBRSxNQUFNLElBQUksSUFBSSxLQUFLLEVBQUUsQ0FBQztRQUMvQixPQUFPLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyxlQUFlLENBQUMsQ0FBQztRQUN0QyxPQUFPLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyxJQUFJLENBQUMsU0FBUyxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUM7UUFDM0MsT0FBTyxDQUFDLE1BQU0sQ0FBQyxLQUFLLENBQUMsUUFBUSxDQUFDLENBQUM7SUFDakMsQ0FBQztBQUNILENBQUM7QUFFRCxNQUFNLEVBQ0osTUFBTSxFQUFFLEVBQ04sS0FBSyxFQUNMLE1BQU0sRUFDTixHQUFHLE9BQU8sRUFDWCxHQUNGLEdBQUcsU0FBUyxDQUFDO0lBQ1osT0FBTyxFQUFFO1FBQ1AsUUFBUSxFQUFFO1lBQ1IsSUFBSSxFQUFFLFNBQVM7WUFDZixLQUFLLEVBQUUsR0FBRztTQUNYO1FBQ0QsS0FBSyxFQUFFO1lBQ0wsSUFBSSxFQUFFLFFBQVE7WUFDZCxLQUFLLEVBQUUsR0FBRztZQUNWLE9BQU8sRUFBRSxHQUFHO1NBQ2I7UUFDRCxNQUFNLEVBQUU7WUFDTixJQUFJLEVBQUUsUUFBUTtZQUNkLEtBQUssRUFBRSxHQUFHO1NBQ1g7S0FDRjtDQUNGLENBQUMsQ0FBQztBQUVILE1BQU0sS0FBSyxHQUFHLEtBQUssS0FBSyxHQUFHLENBQUMsQ0FBQztJQUMzQixnQkFBZ0IsQ0FBQyxLQUFLLENBQUMsQ0FBQyxDQUFDO0lBQ3pCLGVBQWUsQ0FBQyxPQUFPLENBQUMsS0FBSyxDQUFDLENBQUM7QUFFakMsSUFBSSxNQUFNLEVBQUUsQ0FBQztJQUNYLE1BQU0sWUFBWSxDQUFDLEtBQUssRUFBRSxFQUFDLE9BQU8sRUFBRSxNQUFNLEVBQUMsQ0FBQyxDQUFDO0FBQy9DLENBQUM7S0FBTSxDQUFDO0lBQ04sTUFBTSxhQUFhLENBQUMsS0FBSyxDQUFDLENBQUM7QUFDN0IsQ0FBQyJ9