@fin.cx/opendata
Version:
A comprehensive TypeScript library for accessing business data and real-time financial information. Features include German company data management with MongoDB integration, JSONL bulk processing, automated Handelsregister interactions, and real-time stoc
78 lines • 6.81 kB
JavaScript
import * as plugins from './plugins.js';
import * as paths from './paths.js';
export class GermanBusinessData {
constructor(openDataRefArg) {
this.openDataRef = openDataRefArg;
}
async start() {
await this.update();
}
async stop() { }
async update() {
const done = plugins.smartpromise.defer();
const promiseArray = [];
const dataUrl = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2';
const dataExists = await plugins.smartfile.fs.isDirectory(paths.germanBusinessDataDir);
if (!dataExists) {
await plugins.smartfile.fs.ensureDir(paths.germanBusinessDataDir);
}
else {
}
const smartarchive = await plugins.smartarchive.SmartArchive.fromArchiveUrl(dataUrl);
promiseArray
.push();
const jsonlDataStream = await smartarchive.exportToStreamOfStreamFiles();
let totalRecordsCounter = 0;
let nextRest = '';
jsonlDataStream.pipe(new plugins.smartstream.SmartDuplex({
objectMode: true,
writeFunction: async (chunkArg, streamToolsArg) => {
const readStream = await chunkArg.createReadStream();
readStream.pipe(new plugins.smartstream.SmartDuplex({
objectMode: true,
writeFunction: async (chunkArg, streamToolsArg) => {
const currentString = nextRest + chunkArg.toString();
const lines = currentString.split('\n');
nextRest = lines.pop();
console.log(`Got another ${lines.length} records.`);
for (const line of lines) {
let entry;
if (!line)
continue;
try {
entry = JSON.parse(line);
}
catch (err) {
console.log(line);
await plugins.smartdelay.delayFor(10000);
}
if (!entry)
continue;
totalRecordsCounter++;
if (totalRecordsCounter % 10000 === 0)
console.log(`${totalRecordsCounter} total records.`);
const businessRecord = new this.openDataRef.CBusinessRecord();
businessRecord.data.name = entry.name;
await businessRecord.save();
// console.log(`stored ${businessRecord.data.name}`);
}
},
finalFunction: async (streamToolsArg) => {
if (!nextRest)
return;
JSON.parse(nextRest);
}
}));
},
}));
}
async getBusinessRecordByName(nameArg) {
const businessRecord = await this.openDataRef.CBusinessRecord.getInstance({
data: {
name: { $regex: `${nameArg}`, $options: "i" },
}
});
return businessRecord;
}
}
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY2xhc3Nlcy5nZXJtYW5idXNpbmVzc2RhdGEuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi90cy9jbGFzc2VzLmdlcm1hbmJ1c2luZXNzZGF0YS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEtBQUssT0FBTyxNQUFNLGNBQWMsQ0FBQztBQUN4QyxPQUFPLEtBQUssS0FBSyxNQUFNLFlBQVksQ0FBQztBQUdwQyxNQUFNLE9BQU8sa0JBQWtCO0lBRTdCLFlBQVksY0FBd0I7UUFDbEMsSUFBSSxDQUFDLFdBQVcsR0FBRyxjQUFjLENBQUM7SUFDcEMsQ0FBQztJQUVNLEtBQUssQ0FBQyxLQUFLO1FBQ2hCLE1BQU0sSUFBSSxDQUFDLE1BQU0sRUFBRSxDQUFDO0lBQ3RCLENBQUM7SUFDTSxLQUFLLENBQUMsSUFBSSxLQUFJLENBQUM7SUFFZixLQUFLLENBQUMsTUFBTTtRQUNqQixNQUFNLElBQUksR0FBRyxPQUFPLENBQUMsWUFBWSxDQUFDLEtBQUssRUFBRSxDQUFDO1FBQzFDLE1BQU0sWUFBWSxHQUFtQixFQUFFLENBQUM7UUFDeEMsTUFBTSxPQUFPLEdBQUcsK0RBQStELENBQUM7UUFDaEYsTUFBTSxVQUFVLEdBQUcsTUFBTSxPQUFPLENBQUMsU0FBUyxDQUFDLEVBQUUsQ0FBQyxXQUFXLENBQUMsS0FBSyxDQUFDLHFCQUFxQixDQUFDLENBQUM7UUFDdkYsSUFBSSxDQUFDLFVBQVUsRUFBRTtZQUNmLE1BQU0sT0FBTyxDQUFDLFNBQVMsQ0FBQyxFQUFFLENBQUMsU0FBUyxDQUFDLEtBQUssQ0FBQyxxQkFBcUIsQ0FBQyxDQUFDO1NBQ25FO2FBQU07U0FDTjtRQUVELE1BQU0sWUFBWSxHQUFHLE1BQU0sT0FBTyxDQUFDLFlBQVksQ0FBQyxZQUFZLENBQUMsY0FBYyxDQUFDLE9BQU8sQ0FBQyxDQUFDO1FBQ3JGLFlBQVk7YUFDVCxJQUFJLEVBRUgsQ0FBQztRQUNMLE1BQU0sZUFBZSxHQUFHLE1BQU0sWUFBWSxDQUFDLDJCQUEyQixFQUFFLENBQUM7UUFDekUsSUFBSSxtQkFBbUIsR0FBRyxDQUFDLENBQUM7UUFDNUIsSUFBSSxRQUFRLEdBQVcsRUFBRSxDQUFDO1FBQzFCLGVBQWUsQ0FBQyxJQUFJLENBQ2xCLElBQUksT0FBTyxDQUFDLFdBQVcsQ0FBQyxXQUFXLENBQUM7WUFDbEMsVUFBVSxFQUFFLElBQUk7WUFDaEIsYUFBYSxFQUFFLEtBQUssRUFBRSxRQUFzQyxFQUFFLGNBQWMsRUFBRSxFQUFFO2dCQUM5RSxNQUFNLFVBQVUsR0FBRyxNQUFNLFFBQVEsQ0FBQyxnQkFBZ0IsRUFBRSxDQUFDO2dCQUNyRCxVQUFVLENBQUMsSUFBSSxDQUNiLElBQUksT0FBTyxDQUFDLFdBQVcsQ0FBQyxXQUFXLENBQUM7b0JBQ2xDLFVBQVUsRUFBRSxJQUFJO29CQUNoQixhQUFhLEVBQUUsS0FBSyxFQUFFLFFBQWdCLEVBQUUsY0FBYyxFQUFFLEVBQUU7d0JBQ3hELE1BQU0sYUFBYSxHQUFHLFFBQVEsR0FBRyxRQUFRLENBQUMsUUFBUSxFQUFFLENBQUM7d0JBQ3JELE1BQU0sS0FBSyxHQUFHLGFBQWEsQ0FBQyxLQUFLLENBQUMsSUFBSSxDQUFDLENBQUM7d0JBQ3hDLFFBQVEsR0FBRyxLQUFLLENBQUMsR0FBRyxFQUFFLENBQUM7d0JBQ3ZCLE9BQU8sQ0FBQyxHQUFHLENBQUMsZUFBZSxLQUFLLENBQUMsTUFBTSxXQUFXLENBQUMsQ0FBQzt3QkFDcEQsS0FBSyxNQUFNLElBQUksSUFBSSxLQUFLLEVBQUU7NEJBQ3hCLElBQUksS0FBVSxDQUFDOzRCQUNmLElBQUksQ0FBQyxJQUFJO2dDQUFFLFNBQVM7NEJBQ3BCLElBQUk7Z0NBQ0YsS0FBSyxHQUFHLElBQUksQ0FBQyxLQUFLLENBQUMsSUFBSSxDQUFDLENBQUM7NkJBQzFCOzRCQUFDLE9BQU8sR0FBRyxFQUFFO2dDQUNaLE9BQU8sQ0FBQyxHQUFHLENBQUMsSUFBSSxDQUFDLENBQUM7Z0NBQ2xCLE1BQU0sT0FBTyxDQUFDLFVBQVUsQ0FBQyxRQUFRLENBQUMsS0FBSyxDQUFDLENBQUM7NkJBQzFDOzRCQUNELElBQUksQ0FBQyxLQUFLO2dDQUFFLFNBQVM7NEJBQ3JCLG1CQUFtQixFQUFFLENBQUM7NEJBQ3RCLElBQUksbUJBQW1CLEdBQUcsS0FBSyxLQUFLLENBQUM7Z0NBQUUsT0FBTyxDQUFDLEdBQUcsQ0FBQyxHQUFHLG1CQUFtQixpQkFBaUIsQ0FBQyxDQUFDOzRCQUM1RixNQUFNLGNBQWMsR0FBRyxJQUFJLElBQUksQ0FBQyxXQUFXLENBQUMsZUFBZSxFQUFFLENBQUM7NEJBQzlELGNBQWMsQ0FBQyxJQUFJLENBQUMsSUFBSSxHQUFHLEtBQUssQ0FBQyxJQUFJLENBQUM7NEJBQ3RDLE1BQU0sY0FBYyxDQUFDLElBQUksRUFBRSxDQUFDOzRCQUM1QixxREFBcUQ7eUJBQ3REO29CQUNILENBQUM7b0JBQ0QsYUFBYSxFQUFFLEtBQUssRUFBRSxjQUFjLEVBQUUsRUFBRTt3QkFDdEMsSUFBSSxDQUFDLFFBQVE7NEJBQUUsT0FBTzt3QkFDdEIsSUFBSSxDQUFDLEtBQUssQ0FBQyxRQUFRLENBQUMsQ0FBQztvQkFDdkIsQ0FBQztpQkFDRixDQUFDLENBQ0gsQ0FBQztZQUNKLENBQUM7U0FDRixDQUFDLENBQ0gsQ0FBQztJQUNKLENBQUM7SUFFTSxLQUFLLENBQUMsdUJBQXVCLENBQUMsT0FBZTtRQUNsRCxNQUFNLGNBQWMsR0FBRyxNQUFNLElBQUksQ0FBQyxXQUFXLENBQUMsZUFBZSxDQUFDLFdBQVcsQ0FBQztZQUN4RSxJQUFJLEVBQUU7Z0JBQ0osSUFBSSxFQUFFLEVBQUUsTUFBTSxFQUFFLEdBQUcsT0FBTyxFQUFFLEVBQUUsUUFBUSxFQUFFLEdBQUcsRUFBUzthQUNyRDtTQUNGLENBQUMsQ0FBQztRQUNILE9BQU8sY0FBYyxDQUFDO0lBQ3hCLENBQUM7Q0FDRiJ9