@fin.cx/opendata
Version:
A comprehensive TypeScript library that manages open business data for German companies by integrating MongoDB, processing JSONL bulk data, and automating browser interactions for Handelsregister data retrieval.
65 lines • 6.24 kB
JavaScript
import * as plugins from './plugins.js';
import * as paths from './paths.js';
export class JsonlDataProcessor {
constructor(forEachFunctionArg) {
this.forEachFunction = forEachFunctionArg;
}
// TODO: define a mapper as argument instead of hard-coding it
async processDataFromUrl(dataUrlArg = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2') {
const done = plugins.smartpromise.defer();
const dataExists = await plugins.smartfile.fs.isDirectory(paths.germanBusinessDataDir);
if (!dataExists) {
await plugins.smartfile.fs.ensureDir(paths.germanBusinessDataDir);
}
else {
}
const smartarchive = await plugins.smartarchive.SmartArchive.fromArchiveUrl(dataUrlArg);
const jsonlDataStream = await smartarchive.exportToStreamOfStreamFiles();
let totalRecordsCounter = 0;
let nextRest = '';
jsonlDataStream.pipe(new plugins.smartstream.SmartDuplex({
objectMode: true,
writeFunction: async (chunkArg, streamToolsArg) => {
const readStream = await chunkArg.createReadStream();
readStream.pipe(new plugins.smartstream.SmartDuplex({
objectMode: true,
writeFunction: async (chunkArg, streamToolsArg) => {
const currentString = nextRest + chunkArg.toString();
const lines = currentString.split('\n');
nextRest = lines.pop();
console.log(`Got another ${lines.length} records.`);
const concurrentProcessor = new plugins.smartarray.ConcurrentProcessor(async (line) => {
let entry;
if (!line)
return;
try {
entry = JSON.parse(line);
}
catch (err) {
console.log(line);
await plugins.smartdelay.delayFor(10000);
}
if (!entry)
return;
totalRecordsCounter++;
if (totalRecordsCounter % 10000 === 0)
console.log(`${totalRecordsCounter} total records.`);
await this.forEachFunction(entry);
}, 1000);
await concurrentProcessor.process(lines);
},
finalFunction: async (streamToolsArg) => {
console.log(`finished processing ${totalRecordsCounter} records.`);
if (nextRest) {
JSON.parse(nextRest);
}
;
done.resolve();
},
}));
},
}));
await done.promise;
}
}
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY2xhc3Nlcy5qc29ubGRhdGEuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi90cy9jbGFzc2VzLmpzb25sZGF0YS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEtBQUssT0FBTyxNQUFNLGNBQWMsQ0FBQztBQUN4QyxPQUFPLEtBQUssS0FBSyxNQUFNLFlBQVksQ0FBQztBQXlDcEMsTUFBTSxPQUFPLGtCQUFrQjtJQUU3QixZQUFZLGtCQUErQztRQUN6RCxJQUFJLENBQUMsZUFBZSxHQUFHLGtCQUFrQixDQUFDO0lBQzVDLENBQUM7SUFFRCw4REFBOEQ7SUFDdkQsS0FBSyxDQUFDLGtCQUFrQixDQUM3QixVQUFVLEdBQUcsK0RBQStEO1FBRTVFLE1BQU0sSUFBSSxHQUFHLE9BQU8sQ0FBQyxZQUFZLENBQUMsS0FBSyxFQUFFLENBQUM7UUFDMUMsTUFBTSxVQUFVLEdBQUcsTUFBTSxPQUFPLENBQUMsU0FBUyxDQUFDLEVBQUUsQ0FBQyxXQUFXLENBQUMsS0FBSyxDQUFDLHFCQUFxQixDQUFDLENBQUM7UUFDdkYsSUFBSSxDQUFDLFVBQVUsRUFBRSxDQUFDO1lBQ2hCLE1BQU0sT0FBTyxDQUFDLFNBQVMsQ0FBQyxFQUFFLENBQUMsU0FBUyxDQUFDLEtBQUssQ0FBQyxxQkFBcUIsQ0FBQyxDQUFDO1FBQ3BFLENBQUM7YUFBTSxDQUFDO1FBQ1IsQ0FBQztRQUVELE1BQU0sWUFBWSxHQUFHLE1BQU0sT0FBTyxDQUFDLFlBQVksQ0FBQyxZQUFZLENBQUMsY0FBYyxDQUFDLFVBQVUsQ0FBQyxDQUFDO1FBQ3hGLE1BQU0sZUFBZSxHQUFHLE1BQU0sWUFBWSxDQUFDLDJCQUEyQixFQUFFLENBQUM7UUFDekUsSUFBSSxtQkFBbUIsR0FBRyxDQUFDLENBQUM7UUFDNUIsSUFBSSxRQUFRLEdBQVcsRUFBRSxDQUFDO1FBQzFCLGVBQWUsQ0FBQyxJQUFJLENBQ2xCLElBQUksT0FBTyxDQUFDLFdBQVcsQ0FBQyxXQUFXLENBQUM7WUFDbEMsVUFBVSxFQUFFLElBQUk7WUFDaEIsYUFBYSxFQUFFLEtBQUssRUFBRSxRQUFzQyxFQUFFLGNBQWMsRUFBRSxFQUFFO2dCQUM5RSxNQUFNLFVBQVUsR0FBRyxNQUFNLFFBQVEsQ0FBQyxnQkFBZ0IsRUFBRSxDQUFDO2dCQUNyRCxVQUFVLENBQUMsSUFBSSxDQUNiLElBQUksT0FBTyxDQUFDLFdBQVcsQ0FBQyxXQUFXLENBQUM7b0JBQ2xDLFVBQVUsRUFBRSxJQUFJO29CQUNoQixhQUFhLEVBQUUsS0FBSyxFQUFFLFFBQWdCLEVBQUUsY0FBYyxFQUFFLEVBQUU7d0JBQ3hELE1BQU0sYUFBYSxHQUFHLFFBQVEsR0FBRyxRQUFRLENBQUMsUUFBUSxFQUFFLENBQUM7d0JBQ3JELE1BQU0sS0FBSyxHQUFHLGFBQWEsQ0FBQyxLQUFLLENBQUMsSUFBSSxDQUFDLENBQUM7d0JBQ3hDLFFBQVEsR0FBRyxLQUFLLENBQUMsR0FBRyxFQUFFLENBQUM7d0JBQ3ZCLE9BQU8sQ0FBQyxHQUFHLENBQUMsZUFBZSxLQUFLLENBQUMsTUFBTSxXQUFXLENBQUMsQ0FBQzt3QkFDcEQsTUFBTSxtQkFBbUIsR0FBRyxJQUFJLE9BQU8sQ0FBQyxVQUFVLENBQUMsbUJBQW1CLENBQ3BFLEtBQUssRUFBRSxJQUFJLEVBQUUsRUFBRTs0QkFDYixJQUFJLEtBQVEsQ0FBQzs0QkFDYixJQUFJLENBQUMsSUFBSTtnQ0FBRSxPQUFPOzRCQUNsQixJQUFJLENBQUM7Z0NBQ0gsS0FBSyxHQUFHLElBQUksQ0FBQyxLQUFLLENBQUMsSUFBSSxDQUFDLENBQUM7NEJBQzNCLENBQUM7NEJBQUMsT0FBTyxHQUFHLEVBQUUsQ0FBQztnQ0FDYixPQUFPLENBQUMsR0FBRyxDQUFDLElBQUksQ0FBQyxDQUFDO2dDQUNsQixNQUFNLE9BQU8sQ0FBQyxVQUFVLENBQUMsUUFBUSxDQUFDLEtBQUssQ0FBQyxDQUFDOzRCQUMzQyxDQUFDOzRCQUNELElBQUksQ0FBQyxLQUFLO2dDQUFFLE9BQU87NEJBQ25CLG1CQUFtQixFQUFFLENBQUM7NEJBQ3RCLElBQUksbUJBQW1CLEdBQUcsS0FBSyxLQUFLLENBQUM7Z0NBQ25DLE9BQU8sQ0FBQyxHQUFHLENBQUMsR0FBRyxtQkFBbUIsaUJBQWlCLENBQUMsQ0FBQzs0QkFDdkQsTUFBTSxJQUFJLENBQUMsZUFBZSxDQUFDLEtBQUssQ0FBQyxDQUFDO3dCQUNwQyxDQUFDLEVBQ0QsSUFBSSxDQUNMLENBQUM7d0JBQ0YsTUFBTSxtQkFBbUIsQ0FBQyxPQUFPLENBQUMsS0FBSyxDQUFDLENBQUM7b0JBQzNDLENBQUM7b0JBQ0QsYUFBYSxFQUFFLEtBQUssRUFBRSxjQUFjLEVBQUUsRUFBRTt3QkFDdEMsT0FBTyxDQUFDLEdBQUcsQ0FBQyx1QkFBdUIsbUJBQW1CLFdBQVcsQ0FBQyxDQUFDO3dCQUNuRSxJQUFJLFFBQVEsRUFBRSxDQUFDOzRCQUNiLElBQUksQ0FBQyxLQUFLLENBQUMsUUFBUSxDQUFDLENBQUM7d0JBQ3ZCLENBQUM7d0JBQUEsQ0FBQzt3QkFDRixJQUFJLENBQUMsT0FBTyxFQUFFLENBQUM7b0JBQ2pCLENBQUM7aUJBQ0YsQ0FBQyxDQUNILENBQUM7WUFDSixDQUFDO1NBQ0YsQ0FBQyxDQUNILENBQUM7UUFDRixNQUFNLElBQUksQ0FBQyxPQUFPLENBQUM7SUFDckIsQ0FBQztDQUNGIn0=