@naturalcycles/db-lib
Version:
Lowest Common Denominator API to supported Databases
268 lines (267 loc) • 10.3 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.InMemoryDBTransaction = exports.InMemoryDB = void 0;
const node_stream_1 = require("node:stream");
const js_lib_1 = require("@naturalcycles/js-lib");
const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
const __1 = require("../..");
class InMemoryDB {
constructor(cfg) {
this.dbType = __1.CommonDBType.document;
this.support = {
...__1.commonDBFullSupport,
timeMachine: false,
};
// data[table][id] > {id: 'a', created: ... }
this.data = {};
this.cfg = {
// defaults
tablesPrefix: '',
forbidTransactionReadAfterWrite: true,
persistenceEnabled: false,
persistZip: true,
persistentStoragePath: './tmp/inmemorydb',
logger: console,
...cfg,
};
}
/**
* Returns internal "Data snapshot".
* Deterministic - jsonSorted.
*/
getDataSnapshot() {
return (0, js_lib_1._sortObjectDeep)(this.data);
}
async ping() { }
/**
* Resets InMemory DB data
*/
async resetCache(_table) {
if (_table) {
const table = this.cfg.tablesPrefix + _table;
this.cfg.logger.log(`reset ${table}`);
this.data[table] = {};
}
else {
;
(await this.getTables()).forEach(table => {
this.data[table] = {};
});
this.cfg.logger.log('reset');
}
}
async getTables() {
return Object.keys(this.data).filter(t => t.startsWith(this.cfg.tablesPrefix));
}
async getTableSchema(_table) {
const table = this.cfg.tablesPrefix + _table;
return {
...(0, js_lib_1.generateJsonSchemaFromData)((0, js_lib_1._stringMapValues)(this.data[table] || {})),
$id: `${table}.schema.json`,
};
}
async createTable(_table, _schema, opt = {}) {
const table = this.cfg.tablesPrefix + _table;
if (opt.dropIfExists) {
this.data[table] = {};
}
else {
this.data[table] ||= {};
}
}
async getByIds(_table, ids, _opt) {
const table = this.cfg.tablesPrefix + _table;
this.data[table] ||= {};
return ids.map(id => this.data[table][id]).filter(Boolean);
}
async saveBatch(_table, rows, opt = {}) {
const table = this.cfg.tablesPrefix + _table;
this.data[table] ||= {};
rows.forEach(r => {
if (!r.id) {
this.cfg.logger.warn({ rows });
throw new Error(`InMemoryDB doesn't support id auto-generation in saveBatch, row without id was given`);
}
if (opt.saveMethod === 'insert' && this.data[table][r.id]) {
throw new Error(`InMemoryDB: INSERT failed, entity exists: ${table}.${r.id}`);
}
if (opt.saveMethod === 'update' && !this.data[table][r.id]) {
throw new Error(`InMemoryDB: UPDATE failed, entity doesn't exist: ${table}.${r.id}`);
}
// JSON parse/stringify (deep clone) is to:
// 1. Not store values "by reference" (avoid mutation bugs)
// 2. Simulate real DB that would do something like that in a transport layer anyway
this.data[table][r.id] = JSON.parse(JSON.stringify(r), nodejs_lib_1.bufferReviver);
});
}
async deleteByQuery(q, _opt) {
const table = this.cfg.tablesPrefix + q.table;
if (!this.data[table])
return 0;
const ids = (0, __1.queryInMemory)(q, Object.values(this.data[table])).map(r => r.id);
return await this.deleteByIds(q.table, ids);
}
async deleteByIds(_table, ids, _opt) {
const table = this.cfg.tablesPrefix + _table;
if (!this.data[table])
return 0;
let count = 0;
ids.forEach(id => {
if (!this.data[table][id])
return;
delete this.data[table][id];
count++;
});
return count;
}
async patchByQuery(q, patch) {
if ((0, js_lib_1._isEmptyObject)(patch))
return 0;
const table = this.cfg.tablesPrefix + q.table;
const rows = (0, __1.queryInMemory)(q, Object.values(this.data[table] || {}));
rows.forEach(row => Object.assign(row, patch));
return rows.length;
}
async runQuery(q, _opt) {
const table = this.cfg.tablesPrefix + q.table;
return { rows: (0, __1.queryInMemory)(q, Object.values(this.data[table] || {})) };
}
async runQueryCount(q, _opt) {
const table = this.cfg.tablesPrefix + q.table;
return (0, __1.queryInMemory)(q, Object.values(this.data[table] || {})).length;
}
streamQuery(q, _opt) {
const table = this.cfg.tablesPrefix + q.table;
return node_stream_1.Readable.from((0, __1.queryInMemory)(q, Object.values(this.data[table] || {})));
}
async runInTransaction(fn, opt = {}) {
const tx = new InMemoryDBTransaction(this, {
readOnly: false,
...opt,
});
try {
await fn(tx);
await tx.commit();
}
catch (err) {
await tx.rollback();
throw err;
}
}
async incrementBatch(table, prop, incrementMap, _opt) {
const tbl = this.cfg.tablesPrefix + table;
this.data[tbl] ||= {};
const result = {};
for (const [id, by] of (0, js_lib_1._stringMapEntries)(incrementMap)) {
this.data[tbl][id] ||= { id };
const newValue = (this.data[tbl][id][prop] || 0) + by;
this.data[tbl][id][prop] = newValue;
result[id] = newValue;
}
return result;
}
/**
* Flushes all tables (all namespaces) at once.
*/
async flushToDisk() {
(0, js_lib_1._assert)(this.cfg.persistenceEnabled, 'flushToDisk() called but persistenceEnabled=false');
const { persistentStoragePath, persistZip } = this.cfg;
const started = js_lib_1.localTime.nowUnixMillis();
await nodejs_lib_1.fs2.emptyDirAsync(persistentStoragePath);
let tables = 0;
// infinite concurrency for now
await (0, js_lib_1.pMap)(Object.keys(this.data), async (table) => {
const rows = Object.values(this.data[table]);
if (rows.length === 0)
return; // 0 rows
tables++;
const fname = `${persistentStoragePath}/${table}.ndjson${persistZip ? '.gz' : ''}`;
await (0, nodejs_lib_1._pipeline)([node_stream_1.Readable.from(rows), ...nodejs_lib_1.fs2.createWriteStreamAsNDJSON(fname)]);
});
this.cfg.logger.log(`flushToDisk took ${(0, nodejs_lib_1.dimGrey)((0, js_lib_1._since)(started))} to save ${(0, nodejs_lib_1.yellow)(tables)} tables`);
}
/**
* Restores all tables (all namespaces) at once.
*/
async restoreFromDisk() {
(0, js_lib_1._assert)(this.cfg.persistenceEnabled, 'restoreFromDisk() called but persistenceEnabled=false');
const { persistentStoragePath } = this.cfg;
const started = js_lib_1.localTime.nowUnixMillis();
await nodejs_lib_1.fs2.ensureDirAsync(persistentStoragePath);
this.data = {}; // empty it in the beginning!
const files = (await nodejs_lib_1.fs2.readdirAsync(persistentStoragePath)).filter(f => f.includes('.ndjson'));
// infinite concurrency for now
await (0, js_lib_1.pMap)(files, async (file) => {
const fname = `${persistentStoragePath}/${file}`;
const table = file.split('.ndjson')[0];
const rows = await nodejs_lib_1.fs2.createReadStreamAsNDJSON(fname).toArray();
this.data[table] = (0, js_lib_1._by)(rows, r => r.id);
});
this.cfg.logger.log(`restoreFromDisk took ${(0, nodejs_lib_1.dimGrey)((0, js_lib_1._since)(started))} to read ${(0, nodejs_lib_1.yellow)(files.length)} tables`);
}
}
exports.InMemoryDB = InMemoryDB;
class InMemoryDBTransaction {
constructor(db, opt) {
this.db = db;
this.opt = opt;
this.ops = [];
// used to enforce forbidReadAfterWrite setting
this.writeOperationHappened = false;
}
async getByIds(table, ids, opt) {
if (this.db.cfg.forbidTransactionReadAfterWrite) {
(0, js_lib_1._assert)(!this.writeOperationHappened, `InMemoryDBTransaction: read operation attempted after write operation`);
}
return await this.db.getByIds(table, ids, opt);
}
async saveBatch(table, rows, opt) {
(0, js_lib_1._assert)(!this.opt.readOnly, `InMemoryDBTransaction: saveBatch(${table}) called in readOnly mode`);
this.writeOperationHappened = true;
this.ops.push({
type: 'saveBatch',
table,
rows,
opt,
});
}
async deleteByIds(table, ids, opt) {
(0, js_lib_1._assert)(!this.opt.readOnly, `InMemoryDBTransaction: deleteByIds(${table}) called in readOnly mode`);
this.writeOperationHappened = true;
this.ops.push({
type: 'deleteByIds',
table,
ids,
opt,
});
return ids.length;
}
async commit() {
const backup = (0, js_lib_1._deepCopy)(this.db.data);
try {
for (const op of this.ops) {
if (op.type === 'saveBatch') {
await this.db.saveBatch(op.table, op.rows, op.opt);
}
else if (op.type === 'deleteByIds') {
await this.db.deleteByIds(op.table, op.ids, op.opt);
}
else {
throw new Error(`DBOperation not supported: ${op.type}`);
}
}
this.ops = [];
}
catch (err) {
// rollback
this.ops = [];
this.db.data = backup;
this.db.cfg.logger.log('InMemoryDB transaction rolled back');
throw err;
}
}
async rollback() {
this.ops = [];
}
}
exports.InMemoryDBTransaction = InMemoryDBTransaction;