pg-rdf-to-json/dist/index.js

Transforms RDF files from rdf-files.tar.bz2 provided by Project Gutenberg into JS/JSON objects.

import readline from 'node:readline/promises';
import { spawn } from 'node:child_process';
import { formatRDFFile } from './formatters/formatters.js';
import { getParser } from './parser.js';
import { bookSchema, } from './types.js';
/** Re-export types for use downstream. */
export { bookSchema, };
/**
 * Accepts a {@link internal.Readable} stream of XML text. This generator will
 * yield the complete text of each XML document it encounters in the passed
 * stream.
 *
 * @param archiveStream A Readable stream of text.
 */
async function* processArchive(archiveStream) {
    let file = '';
    const stream = readline.createInterface(archiveStream);
    for await (const line of stream) {
        if (line === '<?xml version="1.0" encoding="utf-8"?>') {
            if (file !== '') {
                yield file;
            }
            file = line;
        }
        else {
            file += '\n' + line;
        }
    }
    if (file !== '') {
        yield file;
    }
}
/**
 * Accepts a {@link internal.Readable} stream of XML text. It will parse each
 * XML document yielded by {@link processArchive} using the parser produced by
 * {@link getParser}, yielding the resulting object.
 *
 * @param stream A Readable stream of XML text.
 */
async function* rdfFileStream(stream) {
    const parser = getParser();
    for await (const file of processArchive(stream)) {
        yield parser.parse(file);
    }
}
/**
 * Accepts a {@link internal.Readable} stream of XML text. This function will
 * parse each XML file and yield each parsed document as a
 * {@link Book}.
 *
 * @param stream A Readable stream of XML text.
 */
export async function* booksFromStream(stream) {
    for await (const file of rdfFileStream(stream)) {
        const eBook = file.rdf.ebook;
        try {
            if (!Object.hasOwn(eBook, 'title')) {
                continue;
            }
            if (!Object.hasOwn(eBook, 'type')) {
                continue;
            }
            yield formatRDFFile(file).rdf.ebook;
        }
        catch (cause) {
            const bookId = eBook.about;
            throw new Error(`Problem parsing "${bookId}"`, { cause });
        }
    }
}
/**
 * Spawns `tar -lxOf $pathToArchive`, passes the resulting stream to
 * {@link booksFromStream} and yields each resulting {@link Book}.
 *
 * @throws Will throw the contents of `tar`'s `stderr` if it does not exit
 * cleanly.
 *
 * @param pathToArchive The path to a TAR archive.
 */
export async function* booksFromArchive(pathToArchive) {
    const inputStream = spawn('tar', ['-lxOf', pathToArchive]);
    yield* booksFromStream(inputStream.stdout);
    if (inputStream.exitCode !== 0) {
        for await (const chunk of inputStream.stderr) {
            throw new Error(chunk.toString());
        }
    }
}
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvaW5kZXgudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxRQUFRLE1BQU0sd0JBQXdCLENBQUM7QUFDOUMsT0FBTyxFQUFFLEtBQUssRUFBRSxNQUFNLG9CQUFvQixDQUFDO0FBRTNDLE9BQU8sRUFBRSxhQUFhLEVBQUUsTUFBTSw0QkFBNEIsQ0FBQztBQUMzRCxPQUFPLEVBQUUsU0FBUyxFQUFFLE1BQU0sYUFBYSxDQUFDO0FBQ3hDLE9BQU8sRUFDTCxVQUFVLEdBTVgsTUFBTSxZQUFZLENBQUM7QUFFcEIsMENBQTBDO0FBQzFDLE9BQU8sRUFDTCxVQUFVLEdBS1gsQ0FBQztBQUVGOzs7Ozs7R0FNRztBQUNILEtBQUssU0FBUyxDQUFDLENBQUMsY0FBYyxDQUFDLGFBQWdDO0lBQzdELElBQUksSUFBSSxHQUFHLEVBQUUsQ0FBQztJQUVkLE1BQU0sTUFBTSxHQUFHLFFBQVEsQ0FBQyxlQUFlLENBQUMsYUFBYSxDQUFDLENBQUM7SUFFdkQsSUFBSSxLQUFLLEVBQUUsTUFBTSxJQUFJLElBQUksTUFBTSxFQUFFLENBQUM7UUFDaEMsSUFBSSxJQUFJLEtBQUssd0NBQXdDLEVBQUUsQ0FBQztZQUN0RCxJQUFJLElBQUksS0FBSyxFQUFFLEVBQUUsQ0FBQztnQkFDaEIsTUFBTSxJQUFJLENBQUM7WUFDYixDQUFDO1lBQ0QsSUFBSSxHQUFHLElBQUksQ0FBQztRQUNkLENBQUM7YUFBTSxDQUFDO1lBQ04sSUFBSSxJQUFJLElBQUksR0FBRyxJQUFJLENBQUM7UUFDdEIsQ0FBQztJQUNILENBQUM7SUFFRCxJQUFJLElBQUksS0FBSyxFQUFFLEVBQUUsQ0FBQztRQUNoQixNQUFNLElBQUksQ0FBQztJQUNiLENBQUM7QUFDSCxDQUFDO0FBRUQ7Ozs7OztHQU1HO0FBQ0gsS0FBSyxTQUFTLENBQUMsQ0FBQyxhQUFhLENBQUMsTUFBeUI7SUFDckQsTUFBTSxNQUFNLEdBQUcsU0FBUyxFQUFFLENBQUM7SUFFM0IsSUFBSSxLQUFLLEVBQUUsTUFBTSxJQUFJLElBQUksY0FBYyxDQUFDLE1BQU0sQ0FBQyxFQUFFLENBQUM7UUFDaEQsTUFBTSxNQUFNLENBQUMsS0FBSyxDQUFDLElBQUksQ0FBdUIsQ0FBQztJQUNqRCxDQUFDO0FBQ0gsQ0FBQztBQUVEOzs7Ozs7R0FNRztBQUNILE1BQU0sQ0FBQyxLQUFLLFNBQVMsQ0FBQyxDQUFDLGVBQWUsQ0FBQyxNQUF5QjtJQUM5RCxJQUFJLEtBQUssRUFBRSxNQUFNLElBQUksSUFBSSxhQUFhLENBQUMsTUFBTSxDQUFDLEVBQUUsQ0FBQztRQUMvQyxNQUFNLEtBQUssR0FBRyxJQUFJLENBQUMsR0FBRyxDQUFDLEtBQUssQ0FBQztRQUU3QixJQUFJLENBQUM7WUFDSCxJQUFJLENBQUMsTUFBTSxDQUFDLE1BQU0sQ0FBQyxLQUFLLEVBQUUsT0FBTyxDQUFDLEVBQUUsQ0FBQztnQkFDbkMsU0FBUztZQUNYLENBQUM7WUFDRCxJQUFJLENBQUMsTUFBTSxDQUFDLE1BQU0sQ0FBQyxLQUFLLEVBQUUsTUFBTSxDQUFDLEVBQUUsQ0FBQztnQkFDbEMsU0FBUztZQUNYLENBQUM7WUFDRCxNQUFNLGFBQWEsQ0FBQyxJQUFJLENBQUMsQ0FBQyxHQUFHLENBQUMsS0FBSyxDQUFDO1FBQ3RDLENBQUM7UUFBQyxPQUFPLEtBQUssRUFBRSxDQUFDO1lBQ2YsTUFBTSxNQUFNLEdBQUcsS0FBSyxDQUFDLEtBQUssQ0FBQztZQUMzQixNQUFNLElBQUksS0FBSyxDQUFDLG9CQUFvQixNQUFNLEdBQUcsRUFBRSxFQUFDLEtBQUssRUFBQyxDQUFDLENBQUM7UUFDMUQsQ0FBQztJQUNILENBQUM7QUFDSCxDQUFDO0FBRUQ7Ozs7Ozs7O0dBUUc7QUFDSCxNQUFNLENBQUMsS0FBSyxTQUFTLENBQUMsQ0FBQyxnQkFBZ0IsQ0FBQyxhQUFxQjtJQUMzRCxNQUFNLFdBQVcsR0FBRyxLQUFLLENBQUMsS0FBSyxFQUFFLENBQUMsT0FBTyxFQUFFLGFBQWEsQ0FBQyxDQUFDLENBQUM7SUFFM0QsS0FBSyxDQUFDLENBQUMsZUFBZSxDQUFDLFdBQVcsQ0FBQyxNQUFNLENBQUMsQ0FBQztJQUUzQyxJQUFJLFdBQVcsQ0FBQyxRQUFRLEtBQUssQ0FBQyxFQUFFLENBQUM7UUFDL0IsSUFBSSxLQUFLLEVBQUUsTUFBTSxLQUFLLElBQUksV0FBVyxDQUFDLE1BQU0sRUFBRSxDQUFDO1lBQzdDLE1BQU0sSUFBSSxLQUFLLENBQUMsS0FBSyxDQUFDLFFBQVEsRUFBRSxDQUFDLENBQUM7UUFDcEMsQ0FBQztJQUNILENBQUM7QUFDSCxDQUFDIn0=