@naturalcycles/db-lib
Version:
Lowest Common Denominator API to supported Databases
195 lines (194 loc) • 7.93 kB
JavaScript
import { _isTruthy } from '@naturalcycles/js-lib';
import { Pipeline } from '@naturalcycles/nodejs-lib/stream';
import { BaseCommonDB } from '../../commondb/base.common.db.js';
import { commonDBFullSupport } from '../../commondb/common.db.js';
/**
* CommonDB implementation that proxies requests to downstream CommonDB
* and does in-memory caching.
*
* Queries always hit downstream (unless `onlyCache` is passed)
*/
export class CacheDB extends BaseCommonDB {
support = {
...commonDBFullSupport,
transactions: false,
increment: false,
};
constructor(cfg) {
super();
this.cfg = {
logger: console,
...cfg,
};
}
cfg;
async ping() {
await Promise.all([this.cfg.cacheDB.ping(), this.cfg.downstreamDB.ping()]);
}
/**
* Resets InMemory DB data
*/
// This method is no longer in the public API. Call it just on the InMemoryDB if needed.
// async resetCache(table?: string): Promise<void> {
// this.log(`resetCache ${table || 'all'}`)
// await this.cfg.cacheDB.resetCache(table)
// }
async getTables() {
return await this.cfg.downstreamDB.getTables();
}
async getTableSchema(table) {
return await this.cfg.downstreamDB.getTableSchema(table);
}
async createTable(table, schema, opt = {}) {
if (!opt.onlyCache && !this.cfg.onlyCache) {
await this.cfg.downstreamDB.createTable(table, schema, opt);
}
if (!opt.skipCache && !this.cfg.skipCache) {
await this.cfg.cacheDB.createTable(table, schema, opt);
}
}
async getByIds(table, ids, opt = {}) {
const resultMap = {};
const missingIds = [];
if (!opt.skipCache && !this.cfg.skipCache) {
const results = await this.cfg.cacheDB.getByIds(table, ids, opt);
results.forEach(r => (resultMap[r.id] = r));
missingIds.push(...ids.filter(id => !resultMap[id]));
if (this.cfg.logCached) {
this.cfg.logger?.log(`${table}.getByIds ${results.length} rows from cache: [${results
.map(r => r.id)
.join(', ')}]`);
}
}
if (missingIds.length && !opt.onlyCache && !this.cfg.onlyCache) {
const results = await this.cfg.downstreamDB.getByIds(table, missingIds, opt);
results.forEach(r => (resultMap[r.id] = r));
if (this.cfg.logDownstream) {
this.cfg.logger?.log(`${table}.getByIds ${results.length} rows from downstream: [${results
.map(r => r.id)
.join(', ')}]`);
}
if (!opt.skipCache) {
const cacheResult = this.cfg.cacheDB.saveBatch(table, results, opt);
if (this.cfg.awaitCache)
await cacheResult;
}
}
// return in right order
return ids.map(id => resultMap[id]).filter(_isTruthy);
}
async saveBatch(table, rows, opt = {}) {
if (!opt.onlyCache && !this.cfg.onlyCache) {
await this.cfg.downstreamDB.saveBatch(table, rows, opt);
if (this.cfg.logDownstream) {
this.cfg.logger?.log(`${table}.saveBatch ${rows.length} rows to downstream: [${rows
.map(r => r.id)
.join(', ')}]`);
}
}
if (!opt.skipCache && !this.cfg.skipCache) {
const cacheResult = this.cfg.cacheDB.saveBatch(table, rows, opt).then(() => {
if (this.cfg.logCached) {
this.cfg.logger?.log(`${table}.saveBatch ${rows.length} rows to cache: [${rows.map(r => r.id).join(', ')}]`);
}
});
if (this.cfg.awaitCache)
await cacheResult;
}
}
async runQuery(q, opt = {}) {
if (!opt.onlyCache && !this.cfg.onlyCache) {
const { rows, ...queryResult } = await this.cfg.downstreamDB.runQuery(q, opt);
if (this.cfg.logDownstream) {
this.cfg.logger?.log(`${q.table}.runQuery ${rows.length} rows from downstream`);
}
// Don't save to cache if it was a projection query
if (!opt.skipCache && !this.cfg.skipCache && !q._selectedFieldNames) {
const cacheResult = this.cfg.cacheDB.saveBatch(q.table, rows, opt);
if (this.cfg.awaitCache)
await cacheResult;
}
return { rows, ...queryResult };
}
if (opt.skipCache || this.cfg.skipCache)
return { rows: [] };
const { rows, ...queryResult } = await this.cfg.cacheDB.runQuery(q, opt);
if (this.cfg.logCached) {
this.cfg.logger?.log(`${q.table}.runQuery ${rows.length} rows from cache`);
}
return { rows, ...queryResult };
}
async runQueryCount(q, opt = {}) {
if (!opt.onlyCache && !this.cfg.onlyCache) {
return await this.cfg.downstreamDB.runQueryCount(q, opt);
}
const count = await this.cfg.cacheDB.runQueryCount(q, opt);
if (this.cfg.logCached) {
this.cfg.logger?.log(`${q.table}.runQueryCount ${count} rows from cache`);
}
return count;
}
streamQuery(q, opt = {}) {
if (!opt.onlyCache && !this.cfg.onlyCache) {
const pipeline = this.cfg.downstreamDB.streamQuery(q, opt);
// Don't save to cache if it was a projection query
if (!opt.skipCache && !this.cfg.skipCache && !q._selectedFieldNames) {
// todo: rethink if we really should download WHOLE stream into memory in order to save it to cache
// void obs
// .pipe(toArray())
// .toPromise()
// .then(async dbms => {
// await this.cfg.cacheDB.saveBatch(q.table, dbms as any)
// })
}
return pipeline;
}
if (opt.skipCache || this.cfg.skipCache)
return Pipeline.fromArray([]);
const pipeline = this.cfg.cacheDB.streamQuery(q, opt);
// if (this.cfg.logCached) {
// let count = 0
//
// void pMapStream(stream, async () => {
// count++
// }, {concurrency: 10})
// .then(length => {
// this.log(`${q.table}.streamQuery ${length} rows from cache`)
// })
// }
return pipeline;
}
async deleteByQuery(q, opt = {}) {
if (!opt.onlyCache && !this.cfg.onlyCache) {
const deletedIds = await this.cfg.downstreamDB.deleteByQuery(q, opt);
if (this.cfg.logDownstream) {
this.cfg.logger?.log(`${q.table}.deleteByQuery ${deletedIds} rows from downstream and cache`);
}
if (!opt.skipCache && !this.cfg.skipCache) {
const cacheResult = this.cfg.cacheDB.deleteByQuery(q, opt);
if (this.cfg.awaitCache)
await cacheResult;
}
return deletedIds;
}
if (opt.skipCache || this.cfg.skipCache)
return 0;
const deletedIds = await this.cfg.cacheDB.deleteByQuery(q, opt);
if (this.cfg.logCached) {
this.cfg.logger?.log(`${q.table}.deleteByQuery ${deletedIds} rows from cache`);
}
return deletedIds;
}
async patchByQuery(q, patch, opt = {}) {
let updated;
if (!opt.onlyCache && !this.cfg.onlyCache) {
updated = await this.cfg.downstreamDB.patchByQuery(q, patch, opt);
}
if (!opt.skipCache && !this.cfg.skipCache) {
const cacheResult = this.cfg.cacheDB.patchByQuery(q, patch, opt);
if (this.cfg.awaitCache)
updated ??= await cacheResult;
}
return updated || 0;
}
}