@naturalcycles/db-lib
Version:
Lowest Common Denominator API to supported Databases
41 lines (40 loc) • 1.54 kB
JavaScript
import { Readable } from 'node:stream';
import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
import { fs2 } from '@naturalcycles/nodejs-lib/fs2';
import { _pipeline, createReadStreamAsNDJSON, createWriteStreamAsNDJSON, } from '@naturalcycles/nodejs-lib/stream';
/**
* Persists in local filesystem as ndjson.
*/
export class LocalFilePersistencePlugin {
constructor(cfg = {}) {
this.cfg = {
storagePath: './tmp/localdb',
gzip: true,
...cfg,
};
}
cfg;
async ping() { }
async getTables() {
return (await fs2.readdirAsync(this.cfg.storagePath))
.filter(f => f.includes('.ndjson'))
.map(f => f.split('.ndjson')[0]);
}
async loadFile(table) {
await fs2.ensureDirAsync(this.cfg.storagePath);
const ext = `ndjson${this.cfg.gzip ? '.gz' : ''}`;
const filePath = `${this.cfg.storagePath}/${table}.${ext}`;
if (!(await fs2.pathExistsAsync(filePath)))
return [];
return await createReadStreamAsNDJSON(filePath).toArray();
}
async saveFiles(ops) {
await pMap(ops, async (op) => await this.saveFile(op.table, op.rows), { concurrency: 32 });
}
async saveFile(table, rows) {
await fs2.ensureDirAsync(this.cfg.storagePath);
const ext = `ndjson${this.cfg.gzip ? '.gz' : ''}`;
const filePath = `${this.cfg.storagePath}/${table}.${ext}`;
await _pipeline([Readable.from(rows), ...createWriteStreamAsNDJSON(filePath)]);
}
}