@fin.cx/opendata
Version:
A comprehensive TypeScript library for accessing business data and real-time financial information. Features include German company data management with MongoDB integration, JSONL bulk processing, automated Handelsregister interactions, and real-time stoc
68 lines • 6.38 kB
JavaScript
import * as plugins from './plugins.js';
export class JsonlDataProcessor {
germanBusinessDataDir;
forEachFunction;
constructor(germanBusinessDataDirArg, forEachFunctionArg) {
this.germanBusinessDataDir = germanBusinessDataDirArg;
this.forEachFunction = forEachFunctionArg;
}
// TODO: define a mapper as argument instead of hard-coding it
async processDataFromUrl(dataUrlArg = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2') {
const done = plugins.smartpromise.defer();
const dataExists = await plugins.smartfs.directory(this.germanBusinessDataDir).exists();
if (!dataExists) {
await plugins.smartfs.directory(this.germanBusinessDataDir).create();
}
else {
}
const jsonlDataStream = await plugins.smartarchive.SmartArchive.create()
.url(dataUrlArg)
.toStreamFiles();
let totalRecordsCounter = 0;
let nextRest = '';
jsonlDataStream.pipe(new plugins.smartstream.SmartDuplex({
objectMode: true,
writeFunction: async (chunkArg, streamToolsArg) => {
const readStream = await chunkArg.createReadStream();
readStream.pipe(new plugins.smartstream.SmartDuplex({
objectMode: true,
writeFunction: async (chunkArg, streamToolsArg) => {
const currentString = nextRest + chunkArg.toString();
const lines = currentString.split('\n');
nextRest = lines.pop() ?? '';
console.log(`Got another ${lines.length} records.`);
const concurrentProcessor = new plugins.smartarray.ConcurrentProcessor(async (line) => {
let entry;
if (!line)
return;
try {
entry = JSON.parse(line);
}
catch (err) {
console.log(line);
await plugins.smartdelay.delayFor(10000);
}
if (!entry)
return;
totalRecordsCounter++;
if (totalRecordsCounter % 10000 === 0)
console.log(`${totalRecordsCounter} total records.`);
await this.forEachFunction(entry);
}, 1000);
await concurrentProcessor.process(lines);
},
finalFunction: async (streamToolsArg) => {
console.log(`finished processing ${totalRecordsCounter} records.`);
if (nextRest) {
JSON.parse(nextRest);
}
;
done.resolve();
},
}));
},
}));
await done.promise;
}
}
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY2xhc3Nlcy5qc29ubGRhdGEuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi90cy9jbGFzc2VzLmpzb25sZGF0YS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEtBQUssT0FBTyxNQUFNLGNBQWMsQ0FBQztBQXlDeEMsTUFBTSxPQUFPLGtCQUFrQjtJQUNyQixxQkFBcUIsQ0FBUztJQUMvQixlQUFlLENBQWlDO0lBRXZELFlBQVksd0JBQWdDLEVBQUUsa0JBQStDO1FBQzNGLElBQUksQ0FBQyxxQkFBcUIsR0FBRyx3QkFBd0IsQ0FBQztRQUN0RCxJQUFJLENBQUMsZUFBZSxHQUFHLGtCQUFrQixDQUFDO0lBQzVDLENBQUM7SUFFRCw4REFBOEQ7SUFDdkQsS0FBSyxDQUFDLGtCQUFrQixDQUM3QixVQUFVLEdBQUcsK0RBQStEO1FBRTVFLE1BQU0sSUFBSSxHQUFHLE9BQU8sQ0FBQyxZQUFZLENBQUMsS0FBSyxFQUFFLENBQUM7UUFDMUMsTUFBTSxVQUFVLEdBQUcsTUFBTSxPQUFPLENBQUMsT0FBTyxDQUFDLFNBQVMsQ0FBQyxJQUFJLENBQUMscUJBQXFCLENBQUMsQ0FBQyxNQUFNLEVBQUUsQ0FBQztRQUN4RixJQUFJLENBQUMsVUFBVSxFQUFFLENBQUM7WUFDaEIsTUFBTSxPQUFPLENBQUMsT0FBTyxDQUFDLFNBQVMsQ0FBQyxJQUFJLENBQUMscUJBQXFCLENBQUMsQ0FBQyxNQUFNLEVBQUUsQ0FBQztRQUN2RSxDQUFDO2FBQU0sQ0FBQztRQUNSLENBQUM7UUFFRCxNQUFNLGVBQWUsR0FBRyxNQUFNLE9BQU8sQ0FBQyxZQUFZLENBQUMsWUFBWSxDQUFDLE1BQU0sRUFBRTthQUNyRSxHQUFHLENBQUMsVUFBVSxDQUFDO2FBQ2YsYUFBYSxFQUFFLENBQUM7UUFDbkIsSUFBSSxtQkFBbUIsR0FBRyxDQUFDLENBQUM7UUFDNUIsSUFBSSxRQUFRLEdBQVcsRUFBRSxDQUFDO1FBQzFCLGVBQWUsQ0FBQyxJQUFJLENBQ2xCLElBQUksT0FBTyxDQUFDLFdBQVcsQ0FBQyxXQUFXLENBQUM7WUFDbEMsVUFBVSxFQUFFLElBQUk7WUFDaEIsYUFBYSxFQUFFLEtBQUssRUFBRSxRQUFzQyxFQUFFLGNBQWMsRUFBRSxFQUFFO2dCQUM5RSxNQUFNLFVBQVUsR0FBRyxNQUFNLFFBQVEsQ0FBQyxnQkFBZ0IsRUFBRSxDQUFDO2dCQUNyRCxVQUFVLENBQUMsSUFBSSxDQUNiLElBQUksT0FBTyxDQUFDLFdBQVcsQ0FBQyxXQUFXLENBQUM7b0JBQ2xDLFVBQVUsRUFBRSxJQUFJO29CQUNoQixhQUFhLEVBQUUsS0FBSyxFQUFFLFFBQWdCLEVBQUUsY0FBYyxFQUFFLEVBQUU7d0JBQ3hELE1BQU0sYUFBYSxHQUFHLFFBQVEsR0FBRyxRQUFRLENBQUMsUUFBUSxFQUFFLENBQUM7d0JBQ3JELE1BQU0sS0FBSyxHQUFHLGFBQWEsQ0FBQyxLQUFLLENBQUMsSUFBSSxDQUFDLENBQUM7d0JBQ3hDLFFBQVEsR0FBRyxLQUFLLENBQUMsR0FBRyxFQUFFLElBQUksRUFBRSxDQUFDO3dCQUM3QixPQUFPLENBQUMsR0FBRyxDQUFDLGVBQWUsS0FBSyxDQUFDLE1BQU0sV0FBVyxDQUFDLENBQUM7d0JBQ3BELE1BQU0sbUJBQW1CLEdBQUcsSUFBSSxPQUFPLENBQUMsVUFBVSxDQUFDLG1CQUFtQixDQUNwRSxLQUFLLEVBQUUsSUFBSSxFQUFFLEVBQUU7NEJBQ2IsSUFBSSxLQUFvQixDQUFDOzRCQUN6QixJQUFJLENBQUMsSUFBSTtnQ0FBRSxPQUFPOzRCQUNsQixJQUFJLENBQUM7Z0NBQ0gsS0FBSyxHQUFHLElBQUksQ0FBQyxLQUFLLENBQUMsSUFBSSxDQUFDLENBQUM7NEJBQzNCLENBQUM7NEJBQUMsT0FBTyxHQUFHLEVBQUUsQ0FBQztnQ0FDYixPQUFPLENBQUMsR0FBRyxDQUFDLElBQUksQ0FBQyxDQUFDO2dDQUNsQixNQUFNLE9BQU8sQ0FBQyxVQUFVLENBQUMsUUFBUSxDQUFDLEtBQUssQ0FBQyxDQUFDOzRCQUMzQyxDQUFDOzRCQUNELElBQUksQ0FBQyxLQUFLO2dDQUFFLE9BQU87NEJBQ25CLG1CQUFtQixFQUFFLENBQUM7NEJBQ3RCLElBQUksbUJBQW1CLEdBQUcsS0FBSyxLQUFLLENBQUM7Z0NBQ25DLE9BQU8sQ0FBQyxHQUFHLENBQUMsR0FBRyxtQkFBbUIsaUJBQWlCLENBQUMsQ0FBQzs0QkFDdkQsTUFBTSxJQUFJLENBQUMsZUFBZSxDQUFDLEtBQUssQ0FBQyxDQUFDO3dCQUNwQyxDQUFDLEVBQ0QsSUFBSSxDQUNMLENBQUM7d0JBQ0YsTUFBTSxtQkFBbUIsQ0FBQyxPQUFPLENBQUMsS0FBSyxDQUFDLENBQUM7b0JBQzNDLENBQUM7b0JBQ0QsYUFBYSxFQUFFLEtBQUssRUFBRSxjQUFjLEVBQUUsRUFBRTt3QkFDdEMsT0FBTyxDQUFDLEdBQUcsQ0FBQyx1QkFBdUIsbUJBQW1CLFdBQVcsQ0FBQyxDQUFDO3dCQUNuRSxJQUFJLFFBQVEsRUFBRSxDQUFDOzRCQUNiLElBQUksQ0FBQyxLQUFLLENBQUMsUUFBUSxDQUFDLENBQUM7d0JBQ3ZCLENBQUM7d0JBQUEsQ0FBQzt3QkFDRixJQUFJLENBQUMsT0FBTyxFQUFFLENBQUM7b0JBQ2pCLENBQUM7aUJBQ0YsQ0FBQyxDQUNILENBQUM7WUFDSixDQUFDO1NBQ0YsQ0FBQyxDQUNILENBQUM7UUFDRixNQUFNLElBQUksQ0FBQyxPQUFPLENBQUM7SUFDckIsQ0FBQztDQUNGIn0=