@naturalcycles/db-lib
Version:
Lowest Common Denominator API to supported Databases
253 lines (241 loc) • 8.43 kB
JavaScript
import { _by, _sortBy } from '@naturalcycles/js-lib/array';
import { _since, localTime } from '@naturalcycles/js-lib/datetime';
import { _assert } from '@naturalcycles/js-lib/error/assert.js';
import { generateJsonSchemaFromData } from '@naturalcycles/js-lib/json-schema';
import { _deepEquals, _filterUndefinedValues, _sortObjectDeep } from '@naturalcycles/js-lib/object';
import { _stringMapValues, } from '@naturalcycles/js-lib/types';
import { dimGrey } from '@naturalcycles/nodejs-lib/colors';
import { readableCreate } from '@naturalcycles/nodejs-lib/stream';
import { BaseCommonDB } from '../../commondb/base.common.db.js';
import { commonDBFullSupport } from '../../commondb/common.db.js';
import { queryInMemory } from '../../inmemory/queryInMemory.js';
/**
* Provides barebone implementation for "whole file" based CommonDB.
* "whole file" means that the persistence layer doesn't allow any querying,
* but allows to read the whole file or save the whole file.
* For example, Google Cloud Storage / S3 that store ndjson files will be such persistence.
*
* In contrast with InMemoryDB, FileDB stores *nothing* in memory.
* Each load/query operation loads *whole* file from the persitence layer.
* Each save operation saves *whole* file to the persistence layer.
*/
export class FileDB extends BaseCommonDB {
support = {
...commonDBFullSupport,
bufferValues: false, // todo: implement
insertSaveMethod: false,
updateSaveMethod: false,
patchByQuery: false,
createTable: false,
transactions: false, // todo
increment: false,
};
constructor(cfg) {
super();
this.cfg = {
sortObjects: true,
logFinished: true,
logger: console,
...cfg,
};
}
cfg;
async ping() {
await this.cfg.plugin.ping();
}
async getTables() {
const started = this.logStarted('getTables()');
const tables = await this.cfg.plugin.getTables();
this.logFinished(started, `getTables() ${tables.length} tables`);
return tables;
}
async getByIds(table, ids, _opt) {
const byId = _by(await this.loadFile(table), r => r.id);
return ids.map(id => byId[id]).filter(Boolean);
}
async saveBatch(table, rows, _opt) {
if (!rows.length)
return; // save some api calls
// 1. Load the whole file
const byId = _by(await this.loadFile(table), r => r.id);
// 2. Merge with new data (using ids)
let saved = 0;
rows.forEach(r => {
_assert(r.id, 'FileDB: row.id is required');
if (!_deepEquals(byId[r.id], r)) {
byId[r.id] = r;
saved++;
}
});
// Only save if there are changed rows
if (saved > 0) {
// 3. Save the whole file
await this.saveFile(table, _stringMapValues(byId));
}
}
async runQuery(q, _opt) {
return {
rows: queryInMemory(q, await this.loadFile(q.table)),
};
}
async runQueryCount(q, _opt) {
return (await this.loadFile(q.table)).length;
}
streamQuery(q, opt) {
const readable = readableCreate();
void this.runQuery(q, opt).then(({ rows }) => {
rows.forEach(r => readable.push(r));
readable.push(null); // done
});
return readable;
}
async deleteByQuery(q, _opt) {
const byId = _by(await this.loadFile(q.table), r => r.id);
let deleted = 0;
queryInMemory(q, _stringMapValues(byId)).forEach(r => {
delete byId[r.id];
deleted++;
});
if (deleted > 0) {
await this.saveFile(q.table, _stringMapValues(byId));
}
return deleted;
}
async deleteByIds(table, ids, _opt) {
const byId = _by(await this.loadFile(table), r => r.id);
let deleted = 0;
ids.forEach(id => {
if (!byId[id])
return;
delete byId[id];
deleted++;
});
if (deleted > 0) {
await this.saveFile(table, _stringMapValues(byId));
}
return deleted;
}
async getTableSchema(table) {
const rows = await this.loadFile(table);
return {
...generateJsonSchemaFromData(rows),
$id: `${table}.schema.json`,
};
}
// wrapper, to handle logging
async loadFile(table) {
const started = this.logStarted(`loadFile(${table})`);
const rows = await this.cfg.plugin.loadFile(table);
this.logFinished(started, `loadFile(${table}) ${rows.length} row(s)`);
return rows;
}
// wrapper, to handle logging, sorting rows before saving
async saveFile(table, _rows) {
// if (!_rows.length) return // NO, it should be able to save file with 0 rows!
// Sort the rows, if needed
const rows = this.sortRows(_rows);
const op = `saveFile(${table}) ${rows.length} row(s)`;
const started = this.logStarted(op);
await this.cfg.plugin.saveFiles([{ type: 'saveBatch', table, rows }]);
this.logFinished(started, op);
}
async saveFiles(ops) {
if (!ops.length)
return;
const op = `saveFiles ${ops.length} op(s):\n` + ops.map(o => `${o.table} (${o.rows.length})`).join('\n');
const started = this.logStarted(op);
await this.cfg.plugin.saveFiles(ops);
this.logFinished(started, op);
}
// override async createTransaction(): Promise<FileDBTransaction> {
// return new FileDBTransaction(this)
// }
sortRows(rows) {
rows = rows.map(r => _filterUndefinedValues(r));
if (this.cfg.sortOnSave) {
_sortBy(rows, r => r[this.cfg.sortOnSave.name], { mutate: true });
if (this.cfg.sortOnSave.descending)
rows.reverse(); // mutates
}
if (this.cfg.sortObjects) {
return _sortObjectDeep(rows);
}
return rows;
}
logStarted(op) {
if (this.cfg.logStarted) {
this.cfg.logger?.log(`>> ${op}`);
}
return localTime.nowUnixMillis();
}
logFinished(started, op) {
if (!this.cfg.logFinished)
return;
this.cfg.logger?.log(`<< ${op} ${dimGrey(`in ${_since(started)}`)}`);
}
}
// todo: get back and fix it
// Implementation is optimized for loading/saving _whole files_.
/*
export class FileDBTransaction implements DBTransaction {
constructor(private db: FileDB) {}
ops: DBOperation[] = []
async commit(): Promise<void> {
// data[table][id] => row
const data: StringMap<StringMap<ObjectWithId>> = {}
// 1. Load all tables data (concurrently)
const tables = _uniq(this.ops.map(o => o.table))
await pMap(
tables,
async table => {
const rows = await this.db.loadFile(table)
data[table] = _by(rows, r => r.id)
},
{ concurrency: 16 },
)
const backup = _deepCopy(data)
// 2. Apply ops one by one (in order)
this.ops.forEach(op => {
if (op.type === 'deleteByIds') {
op.ids.forEach(id => delete data[op.table]![id])
} else if (op.type === 'saveBatch') {
op.rows.forEach(r => {
if (!r.id) {
throw new Error('FileDB: row has an empty id')
}
data[op.table]![r.id] = r
})
} else {
throw new Error(`DBOperation not supported: ${(op as any).type}`)
}
})
// 3. Sort, turn it into ops
// Not filtering empty arrays, cause it's already filtered in this.saveFiles()
const ops: DBSaveBatchOperation[] = _stringMapEntries(data).map(([table, map]) => {
return {
type: 'saveBatch',
table,
rows: this.db.sortRows(_stringMapValues(map)),
}
})
// 4. Save all files
try {
await this.db.saveFiles(ops)
} catch (err) {
const ops: DBSaveBatchOperation[] = _stringMapEntries(backup).map(([table, map]) => {
return {
type: 'saveBatch',
table,
rows: this.db.sortRows(_stringMapValues(map)),
}
})
// Rollback, ignore rollback error (if any)
await this.db.saveFiles(ops).catch(_ => {})
throw err
}
}
async rollback(): Promise<void> {
this.ops = []
}
}
*/