@naturalcycles/db-lib
Version:
Lowest Common Denominator API to supported Databases
1,158 lines (1,157 loc) • 46.8 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.CommonDaoTransaction = exports.CommonDao = void 0;
const js_lib_1 = require("@naturalcycles/js-lib");
const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
const cnst_1 = require("../cnst");
const dbQuery_1 = require("../query/dbQuery");
const common_dao_model_1 = require("./common.dao.model");
const isGAE = !!process.env['GAE_INSTANCE'];
const isCI = !!process.env['CI'];
/**
* Lowest common denominator API between supported Databases.
*
* DBM = Database model (how it's stored in DB)
* BM = Backend model (optimized for API access)
* TM = Transport model (optimized to be sent over the wire)
*/
class CommonDao {
constructor(cfg) {
this.cfg = cfg;
this.cfg = {
// Default is to NOT log in AppEngine and in CI,
// otherwise to log Operations
// e.g in Dev (local machine), Test - it will log operations (useful for debugging)
logLevel: isGAE || isCI ? common_dao_model_1.CommonDaoLogLevel.NONE : common_dao_model_1.CommonDaoLogLevel.OPERATIONS,
generateId: true,
assignGeneratedIds: false,
useCreatedProperty: true,
useUpdatedProperty: true,
validateOnLoad: true,
validateOnSave: true,
logger: console,
...cfg,
hooks: {
parseNaturalId: () => ({}),
beforeCreate: bm => bm,
anonymize: dbm => dbm,
onValidationError: err => err,
...cfg.hooks,
},
};
if (this.cfg.generateId) {
this.cfg.hooks.createRandomId ||= () => (0, nodejs_lib_1.stringId)();
}
else {
delete this.cfg.hooks.createRandomId;
}
}
// CREATE
create(part = {}, opt = {}) {
const bm = this.cfg.hooks.beforeCreate(part);
// First assignIdCreatedUpdated, then validate!
this.assignIdCreatedUpdated(bm, opt);
return this.validateAndConvert(bm, this.cfg.bmSchema, undefined, opt);
}
// GET
// overrides are disabled now, as they obfuscate errors when ID branded type is used
// async getById(id: undefined | null, opt?: CommonDaoOptions): Promise<null>
// async getById(id?: ID | null, opt?: CommonDaoOptions): Promise<BM | null>
async getById(id, opt = {}) {
if (!id)
return null;
const op = `getById(${id})`;
const table = opt.table || this.cfg.table;
const started = this.logStarted(op, table);
let dbm = (await (opt.tx || this.cfg.db).getByIds(table, [id], opt))[0];
if (dbm && this.cfg.hooks.afterLoad) {
dbm = (await this.cfg.hooks.afterLoad(dbm)) || undefined;
}
const bm = await this.dbmToBM(dbm, opt);
this.logResult(started, op, bm, table);
return bm || null;
}
async getByIdOrEmpty(id, part = {}, opt) {
const bm = await this.getById(id, opt);
if (bm)
return bm;
return this.create({ ...part, id }, opt);
}
// async getByIdAsDBM(id: undefined | null, opt?: CommonDaoOptions): Promise<null>
// async getByIdAsDBM(id?: ID | null, opt?: CommonDaoOptions): Promise<DBM | null>
async getByIdAsDBM(id, opt = {}) {
if (!id)
return null;
const op = `getByIdAsDBM(${id})`;
const table = opt.table || this.cfg.table;
const started = this.logStarted(op, table);
let [dbm] = await (opt.tx || this.cfg.db).getByIds(table, [id], opt);
if (dbm && this.cfg.hooks.afterLoad) {
dbm = (await this.cfg.hooks.afterLoad(dbm)) || undefined;
}
dbm = this.anyToDBM(dbm, opt);
this.logResult(started, op, dbm, table);
return dbm || null;
}
async getByIds(ids, opt = {}) {
if (!ids.length)
return [];
const op = `getByIds ${ids.length} id(s) (${(0, js_lib_1._truncate)(ids.slice(0, 10).join(', '), 50)})`;
const table = opt.table || this.cfg.table;
const started = this.logStarted(op, table);
let dbms = await (opt.tx || this.cfg.db).getByIds(table, ids, opt);
if (this.cfg.hooks.afterLoad && dbms.length) {
dbms = (await (0, js_lib_1.pMap)(dbms, async (dbm) => await this.cfg.hooks.afterLoad(dbm))).filter(js_lib_1._isTruthy);
}
const bms = await this.dbmsToBM(dbms, opt);
this.logResult(started, op, bms, table);
return bms;
}
async getByIdsAsDBM(ids, opt = {}) {
if (!ids.length)
return [];
const op = `getByIdsAsDBM ${ids.length} id(s) (${(0, js_lib_1._truncate)(ids.slice(0, 10).join(', '), 50)})`;
const table = opt.table || this.cfg.table;
const started = this.logStarted(op, table);
let dbms = await (opt.tx || this.cfg.db).getByIds(table, ids, opt);
if (this.cfg.hooks.afterLoad && dbms.length) {
dbms = (await (0, js_lib_1.pMap)(dbms, async (dbm) => await this.cfg.hooks.afterLoad(dbm))).filter(js_lib_1._isTruthy);
}
this.logResult(started, op, dbms, table);
return dbms;
}
async requireById(id, opt = {}) {
const r = await this.getById(id, opt);
if (!r) {
this.throwRequiredError(id, opt);
}
return r;
}
async requireByIdAsDBM(id, opt = {}) {
const r = await this.getByIdAsDBM(id, opt);
if (!r) {
this.throwRequiredError(id, opt);
}
return r;
}
throwRequiredError(id, opt) {
const table = opt.table || this.cfg.table;
throw new js_lib_1.AppError(`DB row required, but not found in ${table}`, {
table,
id,
});
}
/**
* Throws if readOnly is true
*/
requireWriteAccess() {
if (this.cfg.readOnly) {
throw new js_lib_1.AppError(cnst_1.DBLibError.DAO_IS_READ_ONLY, {
table: this.cfg.table,
});
}
}
/**
* Throws if readOnly is true
*/
requireObjectMutability(opt) {
if (this.cfg.immutable && !opt.allowMutability) {
throw new js_lib_1.AppError(cnst_1.DBLibError.OBJECT_IS_IMMUTABLE, {
table: this.cfg.table,
});
}
}
async ensureUniqueId(table, dbm) {
// todo: retry N times
const existing = await this.cfg.db.getByIds(table, [dbm.id]);
if (existing.length) {
throw new js_lib_1.AppError(cnst_1.DBLibError.NON_UNIQUE_ID, {
table,
ids: existing.map(i => i.id),
});
}
}
async getBy(by, value, limit = 0, opt) {
return await this.query().filterEq(by, value).limit(limit).runQuery(opt);
}
async getOneBy(by, value, opt) {
const [bm] = await this.query().filterEq(by, value).limit(1).runQuery(opt);
return bm || null;
}
async getAll(opt) {
return await this.query().runQuery(opt);
}
// QUERY
/**
* Pass `table` to override table
*/
query(table) {
return new dbQuery_1.RunnableDBQuery(this, table);
}
async runQuery(q, opt) {
const { rows } = await this.runQueryExtended(q, opt);
return rows;
}
async runQuerySingleColumn(q, opt) {
(0, js_lib_1._assert)(q._selectedFieldNames?.length === 1, `runQuerySingleColumn requires exactly 1 column to be selected: ${q.pretty()}`);
const col = q._selectedFieldNames[0];
const { rows } = await this.runQueryExtended(q, opt);
return rows.map((r) => r[col]);
}
/**
* Convenience method that runs multiple queries in parallel and then merges their results together.
* Does deduplication by id.
* Order is not guaranteed, as queries run in parallel.
*/
async runUnionQueries(queries, opt) {
const results = (await (0, js_lib_1.pMap)(queries, async (q) => (await this.runQueryExtended(q, opt)).rows)).flat();
return (0, js_lib_1._uniqBy)(results, r => r.id);
}
async runQueryExtended(q, opt = {}) {
this.validateQueryIndexes(q); // throws if query uses `excludeFromIndexes` property
q.table = opt.table || q.table;
const op = `runQuery(${q.pretty()})`;
const started = this.logStarted(op, q.table);
let { rows, ...queryResult } = await this.cfg.db.runQuery(q, opt);
const partialQuery = !!q._selectedFieldNames;
if (this.cfg.hooks.afterLoad && rows.length) {
rows = (await (0, js_lib_1.pMap)(rows, async (dbm) => await this.cfg.hooks.afterLoad(dbm))).filter(js_lib_1._isTruthy);
}
const bms = partialQuery ? rows : await this.dbmsToBM(rows, opt);
this.logResult(started, op, bms, q.table);
return {
rows: bms,
...queryResult,
};
}
async runQueryAsDBM(q, opt) {
const { rows } = await this.runQueryExtendedAsDBM(q, opt);
return rows;
}
async runQueryExtendedAsDBM(q, opt = {}) {
this.validateQueryIndexes(q); // throws if query uses `excludeFromIndexes` property
q.table = opt.table || q.table;
const op = `runQueryAsDBM(${q.pretty()})`;
const started = this.logStarted(op, q.table);
let { rows, ...queryResult } = await this.cfg.db.runQuery(q, opt);
if (this.cfg.hooks.afterLoad && rows.length) {
rows = (await (0, js_lib_1.pMap)(rows, async (dbm) => await this.cfg.hooks.afterLoad(dbm))).filter(js_lib_1._isTruthy);
}
const partialQuery = !!q._selectedFieldNames;
const dbms = partialQuery ? rows : this.anyToDBMs(rows, opt);
this.logResult(started, op, dbms, q.table);
return { rows: dbms, ...queryResult };
}
async runQueryCount(q, opt = {}) {
this.validateQueryIndexes(q); // throws if query uses `excludeFromIndexes` property
q.table = opt.table || q.table;
const op = `runQueryCount(${q.pretty()})`;
const started = this.logStarted(op, q.table);
const count = await this.cfg.db.runQueryCount(q, opt);
if (this.cfg.logLevel >= common_dao_model_1.CommonDaoLogLevel.OPERATIONS) {
this.cfg.logger?.log(`<< ${q.table}.${op}: ${count} row(s) in ${(0, js_lib_1._since)(started)}`);
}
return count;
}
async streamQueryForEach(q, mapper, opt = {}) {
this.validateQueryIndexes(q); // throws if query uses `excludeFromIndexes` property
q.table = opt.table || q.table;
opt.skipValidation = opt.skipValidation !== false; // default true
opt.errorMode ||= js_lib_1.ErrorMode.SUPPRESS;
const partialQuery = !!q._selectedFieldNames;
const op = `streamQueryForEach(${q.pretty()})`;
const started = this.logStarted(op, q.table, true);
let count = 0;
await (0, nodejs_lib_1._pipeline)([
this.cfg.db.streamQuery(q, opt),
(0, nodejs_lib_1.transformMap)(async (dbm) => {
count++;
if (partialQuery)
return dbm;
if (this.cfg.hooks.afterLoad) {
dbm = (await this.cfg.hooks.afterLoad(dbm));
if (dbm === null)
return js_lib_1.SKIP;
}
return await this.dbmToBM(dbm, opt);
}, {
errorMode: opt.errorMode,
}),
(0, nodejs_lib_1.transformMap)(mapper, {
...opt,
predicate: js_lib_1._passthroughPredicate, // to be able to logProgress
}),
// LogProgress should be AFTER the mapper, to be able to report correct stats
(0, nodejs_lib_1.transformLogProgress)({
metric: q.table,
...opt,
}),
(0, nodejs_lib_1.writableVoid)(),
]);
if (this.cfg.logLevel >= common_dao_model_1.CommonDaoLogLevel.OPERATIONS) {
this.cfg.logger?.log(`<< ${q.table}.${op}: ${count} row(s) in ${(0, js_lib_1._since)(started)}`);
}
}
async streamQueryAsDBMForEach(q, mapper, opt = {}) {
this.validateQueryIndexes(q); // throws if query uses `excludeFromIndexes` property
q.table = opt.table || q.table;
opt.skipValidation = opt.skipValidation !== false; // default true
opt.errorMode ||= js_lib_1.ErrorMode.SUPPRESS;
const partialQuery = !!q._selectedFieldNames;
const op = `streamQueryAsDBMForEach(${q.pretty()})`;
const started = this.logStarted(op, q.table, true);
let count = 0;
await (0, nodejs_lib_1._pipeline)([
this.cfg.db.streamQuery(q, opt),
(0, nodejs_lib_1.transformMap)(async (dbm) => {
count++;
if (partialQuery)
return dbm;
if (this.cfg.hooks.afterLoad) {
dbm = (await this.cfg.hooks.afterLoad(dbm));
if (dbm === null)
return js_lib_1.SKIP;
}
return this.anyToDBM(dbm, opt);
}, {
errorMode: opt.errorMode,
}),
(0, nodejs_lib_1.transformMap)(mapper, {
...opt,
predicate: js_lib_1._passthroughPredicate, // to be able to logProgress
}),
// LogProgress should be AFTER the mapper, to be able to report correct stats
(0, nodejs_lib_1.transformLogProgress)({
metric: q.table,
...opt,
}),
(0, nodejs_lib_1.writableVoid)(),
]);
if (this.cfg.logLevel >= common_dao_model_1.CommonDaoLogLevel.OPERATIONS) {
this.cfg.logger?.log(`<< ${q.table}.${op}: ${count} row(s) in ${(0, js_lib_1._since)(started)}`);
}
}
/**
* Stream as Readable, to be able to .pipe() it further with support of backpressure.
*/
streamQueryAsDBM(q, opt = {}) {
this.validateQueryIndexes(q); // throws if query uses `excludeFromIndexes` property
q.table = opt.table || q.table;
opt.skipValidation = opt.skipValidation !== false; // default true
opt.errorMode ||= js_lib_1.ErrorMode.SUPPRESS;
const partialQuery = !!q._selectedFieldNames;
const stream = this.cfg.db.streamQuery(q, opt);
if (partialQuery)
return stream;
return stream
.on('error', err => stream.emit('error', err))
.pipe((0, nodejs_lib_1.transformMap)(async (dbm) => {
if (this.cfg.hooks.afterLoad) {
dbm = (await this.cfg.hooks.afterLoad(dbm));
if (dbm === null)
return js_lib_1.SKIP;
}
return this.anyToDBM(dbm, opt);
}, {
errorMode: opt.errorMode,
}));
}
/**
* Stream as Readable, to be able to .pipe() it further with support of backpressure.
*
* Please note that this stream is currently not async-iteration friendly, because of
* `through2` usage.
* Will be migrated/fixed at some point in the future.
*
* You can do `.pipe(transformNoOp)` to make it "valid again".
*/
streamQuery(q, opt = {}) {
this.validateQueryIndexes(q); // throws if query uses `excludeFromIndexes` property
q.table = opt.table || q.table;
opt.skipValidation = opt.skipValidation !== false; // default true
opt.errorMode ||= js_lib_1.ErrorMode.SUPPRESS;
const stream = this.cfg.db.streamQuery(q, opt);
const partialQuery = !!q._selectedFieldNames;
if (partialQuery)
return stream;
// This almost works, but hard to implement `errorMode: THROW_AGGREGATED` in this case
// return stream.flatMap(async (dbm: DBM) => {
// if (this.cfg.hooks!.afterLoad) {
// dbm = (await this.cfg.hooks!.afterLoad(dbm))!
// if (dbm === null) return [] // SKIP
// }
//
// return [await this.dbmToBM(dbm, opt)] satisfies BM[]
// }, {
// concurrency: 16,
// })
return (stream
// optimization: 1 validation is enough
// .pipe(transformMap<any, DBM>(dbm => this.anyToDBM(dbm, opt), safeOpt))
// .pipe(transformMap<DBM, BM>(dbm => this.dbmToBM(dbm, opt), safeOpt))
.on('error', err => stream.emit('error', err))
.pipe((0, nodejs_lib_1.transformMap)(async (dbm) => {
if (this.cfg.hooks.afterLoad) {
dbm = (await this.cfg.hooks.afterLoad(dbm));
if (dbm === null)
return js_lib_1.SKIP;
}
return await this.dbmToBM(dbm, opt);
}, {
errorMode: opt.errorMode,
}))
// this can make the stream async-iteration-friendly
// but not applying it now for perf reasons
// UPD: applying, to be compliant with `.toArray()`, etc.
.on('error', err => stream.emit('error', err))
.pipe((0, nodejs_lib_1.transformNoOp)()));
}
async queryIds(q, opt = {}) {
this.validateQueryIndexes(q); // throws if query uses `excludeFromIndexes` property
q.table = opt.table || q.table;
const { rows } = await this.cfg.db.runQuery(q.select(['id']), opt);
return rows.map(r => r.id);
}
streamQueryIds(q, opt = {}) {
this.validateQueryIndexes(q); // throws if query uses `excludeFromIndexes` property
q.table = opt.table || q.table;
opt.errorMode ||= js_lib_1.ErrorMode.SUPPRESS;
// Experimental: using `.map()`
const stream = this.cfg.db
.streamQuery(q.select(['id']), opt)
// .on('error', err => stream.emit('error', err))
.map((r) => r.id);
// const stream: ReadableTyped<string> = this.cfg.db
// .streamQuery<DBM>(q.select(['id']), opt)
// .on('error', err => stream.emit('error', err))
// .pipe(
// transformMapSimple<DBM, string>(r => r.id, {
// errorMode: ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
// }),
// )
return stream;
}
async streamQueryIdsForEach(q, mapper, opt = {}) {
this.validateQueryIndexes(q); // throws if query uses `excludeFromIndexes` property
q.table = opt.table || q.table;
opt.errorMode ||= js_lib_1.ErrorMode.SUPPRESS;
const op = `streamQueryIdsForEach(${q.pretty()})`;
const started = this.logStarted(op, q.table, true);
let count = 0;
await (0, nodejs_lib_1._pipeline)([
this.cfg.db.streamQuery(q.select(['id']), opt).map(r => {
count++;
return r.id;
}),
(0, nodejs_lib_1.transformMap)(mapper, {
...opt,
predicate: js_lib_1._passthroughPredicate,
}),
// LogProgress should be AFTER the mapper, to be able to report correct stats
(0, nodejs_lib_1.transformLogProgress)({
metric: q.table,
...opt,
}),
(0, nodejs_lib_1.writableVoid)(),
]);
if (this.cfg.logLevel >= common_dao_model_1.CommonDaoLogLevel.OPERATIONS) {
this.cfg.logger?.log(`<< ${q.table}.${op}: ${count} id(s) in ${(0, js_lib_1._since)(started)}`);
}
}
/**
* Mutates!
* "Returns", just to have a type of "Saved"
*/
assignIdCreatedUpdated(obj, opt = {}) {
const now = js_lib_1.localTime.nowUnix();
if (this.cfg.useCreatedProperty) {
obj.created ||= obj.updated || now;
}
if (this.cfg.useUpdatedProperty) {
obj.updated = opt.preserveUpdatedCreated && obj.updated ? obj.updated : now;
}
if (this.cfg.generateId) {
obj.id ||= (this.cfg.hooks.createNaturalId?.(obj) ||
this.cfg.hooks.createRandomId());
}
return obj;
}
// SAVE
/**
* Convenience method to replace 3 operations (loading+patching+saving) with one:
*
* 1. Loads the row by id.
* 1.1 Creates the row (via this.create()) if it doesn't exist
* (this will cause a validation error if Patch has not enough data for the row to be valid).
* 2. Applies the patch on top of loaded data.
* 3. Saves (as fast as possible since the read) with the Patch applied, but only if the data has changed.
*/
async patchById(id, patch, opt = {}) {
if (this.cfg.patchInTransaction && !opt.tx) {
// patchInTransaction means that we should run this op in Transaction
// But if opt.tx is passed - means that we are already in a Transaction,
// and should just continue as-is
return await this.patchByIdInTransaction(id, patch, opt);
}
let patched;
const loaded = await this.getById(id, {
// Skipping validation here for performance reasons.
// Validation is going to happen on save anyway, just down below.
skipValidation: true,
...opt,
});
if (loaded) {
patched = { ...loaded, ...patch };
if ((0, js_lib_1._deepJsonEquals)(loaded, patched)) {
// Skipping the save operation, as data is the same
return patched;
}
}
else {
const table = opt.table || this.cfg.table;
(0, js_lib_1._assert)(opt.createIfMissing, `DB row required, but not found in ${table}`, {
id,
table,
});
patched = this.create({ ...patch, id }, opt);
}
return await this.save(patched, opt);
}
/**
* Like patchById, but runs all operations within a Transaction.
*/
async patchByIdInTransaction(id, patch, opt) {
return await this.runInTransaction(async (daoTx) => {
return await this.patchById(id, patch, { ...opt, tx: daoTx.tx });
});
}
/**
* Same as patchById, but takes the whole object as input.
* This "whole object" is mutated with the patch and returned.
* Otherwise, similar behavior as patchById.
* It still loads the row from the DB.
*/
async patch(bm, patch, opt = {}) {
if (this.cfg.patchInTransaction && !opt.tx) {
// patchInTransaction means that we should run this op in Transaction
// But if opt.tx is passed - means that we are already in a Transaction,
// and should just continue as-is
return await this.patchInTransaction(bm, patch, opt);
}
if (opt.skipDBRead) {
const patched = {
...bm,
...patch,
};
if ((0, js_lib_1._deepJsonEquals)(bm, patched)) {
// Skipping the save operation, as data is the same
return bm;
}
Object.assign(bm, patch);
}
else {
const loaded = await this.requireById(bm.id, {
// Skipping validation here for performance reasons.
// Validation is going to happen on save anyway, just down below.
skipValidation: true,
...opt,
});
const loadedWithPatch = {
...loaded,
...patch,
};
// Make `bm` exactly the same as `loadedWithPatch`
(0, js_lib_1._objectAssignExact)(bm, loadedWithPatch);
if ((0, js_lib_1._deepJsonEquals)(loaded, loadedWithPatch)) {
// Skipping the save operation, as data is the same
return bm;
}
}
return await this.save(bm, opt);
}
/**
* Like patch, but runs all operations within a Transaction.
*/
async patchInTransaction(bm, patch, opt) {
return await this.runInTransaction(async (daoTx) => {
return await this.patch(bm, patch, { ...opt, tx: daoTx.tx });
});
}
/**
* Mutates with id, created, updated
*/
async save(bm, opt = {}) {
this.requireWriteAccess();
if (opt.skipIfEquals) {
// We compare with convertedBM, to account for cases when some extra property is assigned to bm,
// which should be removed post-validation, but it breaks the "equality check"
// Post-validation the equality check should work as intended
const convertedBM = this.validateAndConvert(bm, this.cfg.bmSchema, 'save', opt);
if ((0, js_lib_1._deepJsonEquals)(convertedBM, opt.skipIfEquals)) {
// Skipping the save operation
return bm;
}
}
const idWasGenerated = !bm.id && this.cfg.generateId;
this.assignIdCreatedUpdated(bm, opt); // mutates
(0, js_lib_1._typeCast)(bm);
let dbm = await this.bmToDBM(bm, opt); // validates BM
if (this.cfg.hooks.beforeSave) {
dbm = (await this.cfg.hooks.beforeSave(dbm));
if (dbm === null)
return bm;
}
const table = opt.table || this.cfg.table;
if (opt.ensureUniqueId && idWasGenerated)
await this.ensureUniqueId(table, dbm);
if (this.cfg.immutable && !opt.allowMutability && !opt.saveMethod) {
opt = { ...opt, saveMethod: 'insert' };
}
const op = `save(${dbm.id})`;
const started = this.logSaveStarted(op, bm, table);
const { excludeFromIndexes } = this.cfg;
const assignGeneratedIds = opt.assignGeneratedIds || this.cfg.assignGeneratedIds;
await (opt.tx || this.cfg.db).saveBatch(table, [dbm], {
excludeFromIndexes,
assignGeneratedIds,
...opt,
});
if (assignGeneratedIds) {
bm.id = dbm.id;
}
this.logSaveResult(started, op, table);
return bm;
}
async saveAsDBM(dbm, opt = {}) {
this.requireWriteAccess();
const table = opt.table || this.cfg.table;
// assigning id in case it misses the id
// will override/set `updated` field, unless opts.preserveUpdated is set
const idWasGenerated = !dbm.id && this.cfg.generateId;
this.assignIdCreatedUpdated(dbm, opt); // mutates
let row = this.anyToDBM(dbm, opt);
if (opt.ensureUniqueId && idWasGenerated)
await this.ensureUniqueId(table, row);
if (this.cfg.immutable && !opt.allowMutability && !opt.saveMethod) {
opt = { ...opt, saveMethod: 'insert' };
}
const op = `saveAsDBM(${row.id})`;
const started = this.logSaveStarted(op, row, table);
const { excludeFromIndexes } = this.cfg;
const assignGeneratedIds = opt.assignGeneratedIds || this.cfg.assignGeneratedIds;
if (this.cfg.hooks.beforeSave) {
row = (await this.cfg.hooks.beforeSave(row));
if (row === null)
return dbm;
}
await (opt.tx || this.cfg.db).saveBatch(table, [row], {
excludeFromIndexes,
assignGeneratedIds,
...opt,
});
if (assignGeneratedIds) {
dbm.id = row.id;
}
this.logSaveResult(started, op, table);
return row;
}
async saveBatch(bms, opt = {}) {
if (!bms.length)
return [];
this.requireWriteAccess();
const table = opt.table || this.cfg.table;
bms.forEach(bm => this.assignIdCreatedUpdated(bm, opt));
let dbms = await this.bmsToDBM(bms, opt);
if (this.cfg.hooks.beforeSave && dbms.length) {
dbms = (await (0, js_lib_1.pMap)(dbms, async (dbm) => await this.cfg.hooks.beforeSave(dbm))).filter(js_lib_1._isTruthy);
}
if (opt.ensureUniqueId)
throw new js_lib_1.AppError('ensureUniqueId is not supported in saveBatch');
if (this.cfg.immutable && !opt.allowMutability && !opt.saveMethod) {
opt = { ...opt, saveMethod: 'insert' };
}
const op = `saveBatch ${dbms.length} row(s) (${(0, js_lib_1._truncate)(dbms
.slice(0, 10)
.map(bm => bm.id)
.join(', '), 50)})`;
const started = this.logSaveStarted(op, bms, table);
const { excludeFromIndexes } = this.cfg;
const assignGeneratedIds = opt.assignGeneratedIds || this.cfg.assignGeneratedIds;
await (opt.tx || this.cfg.db).saveBatch(table, dbms, {
excludeFromIndexes,
assignGeneratedIds,
...opt,
});
if (assignGeneratedIds) {
dbms.forEach((dbm, i) => (bms[i].id = dbm.id));
}
this.logSaveResult(started, op, table);
return bms;
}
async saveBatchAsDBM(dbms, opt = {}) {
if (!dbms.length)
return [];
this.requireWriteAccess();
const table = opt.table || this.cfg.table;
dbms.forEach(dbm => this.assignIdCreatedUpdated(dbm, opt)); // mutates
let rows = this.anyToDBMs(dbms, opt);
if (opt.ensureUniqueId)
throw new js_lib_1.AppError('ensureUniqueId is not supported in saveBatch');
if (this.cfg.immutable && !opt.allowMutability && !opt.saveMethod) {
opt = { ...opt, saveMethod: 'insert' };
}
const op = `saveBatchAsDBM ${rows.length} row(s) (${(0, js_lib_1._truncate)(rows
.slice(0, 10)
.map(bm => bm.id)
.join(', '), 50)})`;
const started = this.logSaveStarted(op, rows, table);
const { excludeFromIndexes } = this.cfg;
const assignGeneratedIds = opt.assignGeneratedIds || this.cfg.assignGeneratedIds;
if (this.cfg.hooks.beforeSave && rows.length) {
rows = (await (0, js_lib_1.pMap)(rows, async (row) => await this.cfg.hooks.beforeSave(row))).filter(js_lib_1._isTruthy);
}
await (opt.tx || this.cfg.db).saveBatch(table, rows, {
excludeFromIndexes,
assignGeneratedIds,
...opt,
});
if (assignGeneratedIds) {
rows.forEach((row, i) => (dbms[i].id = row.id));
}
this.logSaveResult(started, op, table);
return rows;
}
/**
* "Streaming" is implemented by buffering incoming rows into **batches**
* (of size opt.chunkSize, which defaults to 500),
* and then executing db.saveBatch(chunk) with the concurrency
* of opt.chunkConcurrency (which defaults to 32).
*/
streamSaveTransform(opt = {}) {
this.requireWriteAccess();
const table = opt.table || this.cfg.table;
opt.skipValidation ??= true;
opt.errorMode ||= js_lib_1.ErrorMode.SUPPRESS;
if (this.cfg.immutable && !opt.allowMutability && !opt.saveMethod) {
opt = { ...opt, saveMethod: 'insert' };
}
const excludeFromIndexes = opt.excludeFromIndexes || this.cfg.excludeFromIndexes;
const { beforeSave } = this.cfg.hooks;
const { chunkSize = 500, chunkConcurrency = 32, errorMode } = opt;
return [
(0, nodejs_lib_1.transformMap)(async (bm) => {
this.assignIdCreatedUpdated(bm, opt); // mutates
let dbm = await this.bmToDBM(bm, opt);
if (beforeSave) {
dbm = (await beforeSave(dbm));
if (dbm === null)
return js_lib_1.SKIP;
}
return dbm;
}, {
errorMode,
}),
(0, nodejs_lib_1.transformChunk)({ chunkSize }),
(0, nodejs_lib_1.transformMap)(async (batch) => {
await this.cfg.db.saveBatch(table, batch, {
...opt,
excludeFromIndexes,
});
return batch;
}, {
concurrency: chunkConcurrency,
errorMode,
flattenArrayOutput: true,
}),
(0, nodejs_lib_1.transformLogProgress)({
metric: 'saved',
...opt,
}),
// just to satisfy and simplify typings
// It's easier to return Transform[], rather than (Transform | Writable)[]
(0, nodejs_lib_1.writableVoid)(),
];
}
// DELETE
/**
* @returns number of deleted items
*/
async deleteById(id, opt = {}) {
if (!id)
return 0;
return await this.deleteByIds([id], opt);
}
async deleteByIds(ids, opt = {}) {
if (!ids.length)
return 0;
this.requireWriteAccess();
this.requireObjectMutability(opt);
const op = `deleteByIds(${ids.join(', ')})`;
const table = opt.table || this.cfg.table;
const started = this.logStarted(op, table);
const count = await (opt.tx || this.cfg.db).deleteByIds(table, ids, opt);
this.logSaveResult(started, op, table);
return count;
}
/**
* Pass `chunkSize: number` (e.g 500) option to use Streaming: it will Stream the query, chunk by 500, and execute
* `deleteByIds` for each chunk concurrently (infinite concurrency).
* This is expected to be more memory-efficient way of deleting large number of rows.
*/
async deleteByQuery(q, opt = {}) {
this.validateQueryIndexes(q); // throws if query uses `excludeFromIndexes` property
this.requireWriteAccess();
this.requireObjectMutability(opt);
q.table = opt.table || q.table;
const op = `deleteByQuery(${q.pretty()})`;
const started = this.logStarted(op, q.table);
let deleted = 0;
if (opt.chunkSize) {
const { chunkSize, chunkConcurrency = 32 } = opt;
await (0, nodejs_lib_1._pipeline)([
this.cfg.db.streamQuery(q.select(['id']), opt).map(r => r.id),
(0, nodejs_lib_1.transformChunk)({ chunkSize }),
(0, nodejs_lib_1.transformMap)(async (ids) => {
await this.cfg.db.deleteByIds(q.table, ids, opt);
deleted += ids.length;
}, {
predicate: js_lib_1._passthroughPredicate,
concurrency: chunkConcurrency,
}),
// LogProgress should be AFTER the mapper, to be able to report correct stats
(0, nodejs_lib_1.transformLogProgress)({
metric: q.table,
logEvery: 2, // 500 * 2 === 1000
chunkSize,
...opt,
}),
(0, nodejs_lib_1.writableVoid)(),
]);
}
else {
deleted = await this.cfg.db.deleteByQuery(q, opt);
}
this.logSaveResult(started, op, q.table);
return deleted;
}
async patchByIds(ids, patch, opt = {}) {
if (!ids.length)
return 0;
return await this.patchByQuery(this.query().filterIn('id', ids), patch, opt);
}
async patchByQuery(q, patch, opt = {}) {
this.validateQueryIndexes(q); // throws if query uses `excludeFromIndexes` property
this.requireWriteAccess();
this.requireObjectMutability(opt);
q.table = opt.table || q.table;
const op = `patchByQuery(${q.pretty()})`;
const started = this.logStarted(op, q.table);
const updated = await this.cfg.db.patchByQuery(q, patch, opt);
this.logSaveResult(started, op, q.table);
return updated;
}
/**
* Caveat: it doesn't update created/updated props.
*
* @experimental
*/
async increment(prop, id, by = 1, opt = {}) {
this.requireWriteAccess();
this.requireObjectMutability(opt);
const { table } = this.cfg;
const op = `increment`;
const started = this.logStarted(op, table);
const result = await this.cfg.db.incrementBatch(table, prop, {
[id]: by,
});
this.logSaveResult(started, op, table);
return result[id];
}
/**
* Caveat: it doesn't update created/updated props.
*
* @experimental
*/
async incrementBatch(prop, incrementMap, opt = {}) {
this.requireWriteAccess();
this.requireObjectMutability(opt);
const { table } = this.cfg;
const op = `incrementBatch`;
const started = this.logStarted(op, table);
const result = await this.cfg.db.incrementBatch(table, prop, incrementMap);
this.logSaveResult(started, op, table);
return result;
}
async dbmToBM(_dbm, opt = {}) {
if (!_dbm)
return;
// optimization: no need to run full joi DBM validation, cause BM validation will be run
// const dbm = this.anyToDBM(_dbm, opt)
let dbm = { ..._dbm, ...this.cfg.hooks.parseNaturalId(_dbm.id) };
if (opt.anonymize) {
dbm = this.cfg.hooks.anonymize(dbm);
}
// DBM > BM
const bm = ((await this.cfg.hooks.beforeDBMToBM?.(dbm)) || dbm);
// Validate/convert BM
return this.validateAndConvert(bm, this.cfg.bmSchema, 'load', opt);
}
async dbmsToBM(dbms, opt = {}) {
return await (0, js_lib_1.pMap)(dbms, async (dbm) => await this.dbmToBM(dbm, opt));
}
async bmToDBM(bm, opt) {
if (bm === undefined)
return;
// bm gets assigned to the new reference
bm = this.validateAndConvert(bm, this.cfg.bmSchema, 'save', opt);
// BM > DBM
return ((await this.cfg.hooks.beforeBMToDBM?.(bm)) || bm);
}
async bmsToDBM(bms, opt = {}) {
// try/catch?
return await (0, js_lib_1.pMap)(bms, async (bm) => await this.bmToDBM(bm, opt));
}
anyToDBM(dbm, opt = {}) {
if (!dbm)
return;
// this shouldn't be happening on load! but should on save!
// this.assignIdCreatedUpdated(dbm, opt)
dbm = { ...dbm, ...this.cfg.hooks.parseNaturalId(dbm.id) };
// todo: is this the right place?
// todo: is anyToDBM even needed?
if (opt.anonymize) {
dbm = this.cfg.hooks.anonymize(dbm);
}
// Validate/convert DBM
// return this.validateAndConvert(dbm, this.cfg.dbmSchema, DBModelType.DBM, opt)
return dbm;
}
anyToDBMs(entities, opt = {}) {
return entities.map(entity => this.anyToDBM(entity, opt));
}
/**
* Returns *converted value* (NOT the same reference).
* Does NOT mutate the object.
* Validates (unless `skipValidation=true` passed).
*/
validateAndConvert(obj, schema, op, // this is to skip validation if validateOnLoad/Save is false
opt = {}) {
// Kirill 2021-10-18: I realized that there's little reason to keep removing null values
// So, from now on we'll preserve them
// "undefined" values, I believe, are/were not saved to/from DB anyway (due to e.g JSON.stringify removing them)
// But let's keep watching it!
//
// Filter null and undefined values
// obj = _filterNullishValues(obj as any)
// We still filter `undefined` values here, because `beforeDBMToBM` can return undefined values
// and they can be annoying with snapshot tests
obj = (0, js_lib_1._filterUndefinedValues)(obj);
// Return as is if no schema is passed or if `skipConversion` is set
if (!schema ||
opt.skipValidation ||
(op === 'load' && !this.cfg.validateOnLoad) ||
(op === 'save' && !this.cfg.validateOnSave)) {
return obj;
}
// This will Convert and Validate
const table = opt.table || this.cfg.table;
const objectName = table;
let error;
let convertedValue;
if (schema instanceof js_lib_1.ZodSchema) {
// Zod schema
const vr = (0, js_lib_1.zSafeValidate)(obj, schema);
error = vr.error;
convertedValue = vr.data;
}
else if (schema instanceof nodejs_lib_1.AjvSchema) {
// Ajv schema
convertedValue = obj; // because Ajv mutates original object
error = schema.getValidationError(obj, {
objectName,
});
}
else {
// Joi
const start = js_lib_1.localTime.nowUnixMillis();
const vr = (0, nodejs_lib_1.getValidationResult)(obj, schema, objectName);
const tookMillis = js_lib_1.localTime.nowUnixMillis() - start;
this.cfg.onValidationTime?.({
tookMillis,
table,
obj,
});
error = vr.error;
convertedValue = vr.value;
}
// If we care about validation and there's an error
if (error) {
const processedError = this.cfg.hooks.onValidationError(error);
if (processedError)
throw processedError;
}
return convertedValue;
}
async getTableSchema() {
return await this.cfg.db.getTableSchema(this.cfg.table);
}
async createTable(schema, opt) {
this.requireWriteAccess();
await this.cfg.db.createTable(this.cfg.table, schema, opt);
}
/**
* Proxy to this.cfg.db.ping
*/
async ping() {
await this.cfg.db.ping();
}
async runInTransaction(fn, opt) {
let r;
await this.cfg.db.runInTransaction(async (tx) => {
const daoTx = new CommonDaoTransaction(tx, this.cfg.logger);
try {
r = await fn(daoTx);
}
catch (err) {
await daoTx.rollback(); // graceful rollback that "never throws"
throw err;
}
}, opt);
return r;
}
/**
* Throws if query uses a property that is in `excludeFromIndexes` list.
*/
validateQueryIndexes(q) {
const { excludeFromIndexes } = this.cfg;
if (!excludeFromIndexes)
return;
for (const f of q._filters) {
(0, js_lib_1._assert)(!excludeFromIndexes.includes(f.name), `cannot query on non-indexed property: ${this.cfg.table}.${f.name}`, {
query: q.pretty(),
});
}
}
logResult(started, op, res, table) {
if (!this.cfg.logLevel)
return;
let logRes;
const args = [];
if (Array.isArray(res)) {
logRes = `${res.length} row(s)`;
if (res.length && this.cfg.logLevel >= common_dao_model_1.CommonDaoLogLevel.DATA_FULL) {
args.push('\n', ...res.slice(0, 10)); // max 10 items
}
}
else if (res) {
logRes = `1 row`;
if (this.cfg.logLevel >= common_dao_model_1.CommonDaoLogLevel.DATA_SINGLE) {
args.push('\n', res);
}
}
else {
logRes = `undefined`;
}
this.cfg.logger?.log(`<< ${table}.${op}: ${logRes} in ${(0, js_lib_1._since)(started)}`, ...args);
}
logSaveResult(started, op, table) {
if (!this.cfg.logLevel)
return;
this.cfg.logger?.log(`<< ${table}.${op} in ${(0, js_lib_1._since)(started)}`);
}
logStarted(op, table, force = false) {
if (this.cfg.logStarted || force) {
this.cfg.logger?.log(`>> ${table}.${op}`);
}
return js_lib_1.localTime.nowUnixMillis();
}
logSaveStarted(op, items, table) {
if (this.cfg.logStarted) {
const args = [`>> ${table}.${op}`];
if (Array.isArray(items)) {
if (items.length && this.cfg.logLevel >= common_dao_model_1.CommonDaoLogLevel.DATA_FULL) {
args.push('\n', ...items.slice(0, 10));
}
else {
args.push(`${items.length} row(s)`);
}
}
else {
if (this.cfg.logLevel >= common_dao_model_1.CommonDaoLogLevel.DATA_SINGLE) {
args.push(items);
}
}
this.cfg.logger?.log(...args);
}
return js_lib_1.localTime.nowUnixMillis();
}
}
exports.CommonDao = CommonDao;
/**
* Transaction context.
* Has similar API than CommonDao, but all operations are performed in the context of the transaction.
*/
class CommonDaoTransaction {
constructor(tx, logger) {
this.tx = tx;
this.logger = logger;
}
/**
* Perform a graceful rollback without throwing/re-throwing any error.
*/
async rollback() {
try {
await this.tx.rollback();
}
catch (err) {
// graceful rollback without re-throw
this.logger.error(err);
}
}
async getById(dao, id, opt) {
return await dao.getById(id, { ...opt, tx: this.tx });
}
async getByIds(dao, ids, opt) {
return await dao.getByIds(ids, { ...opt, tx: this.tx });
}
// todo: Queries inside Transaction are not supported yet
// async runQuery<BM extends PartialObjectWithId, DBM extends ObjectWithId>(
// dao: CommonDao<BM, DBM, any>,
// q: DBQuery<DBM>,
// opt?: CommonDaoOptions,
// ): Promise<BM[]> {
// try {
// return await dao.runQuery(q, { ...opt, tx: this.tx })
// } catch (err) {
// await this.rollback()
// throw err
// }
// }
async save(dao, bm, opt) {
return await dao.save(bm, { ...opt, tx: this.tx });
}
async saveBatch(dao, bms, opt) {
return await dao.saveBatch(bms, { ...opt, tx: this.tx });
}
/**
* DaoTransaction.patch does not load from DB.
* It assumes the bm was previously loaded in the same Transaction, hence could not be
* concurrently modified. Hence it's safe to not sync with DB.
*
* So, this method is a rather simple convenience "Object.assign and then save".
*/
async patch(dao, bm, patch, opt) {
const skipIfEquals = (0, js_lib_1._deepCopy)(bm);
Object.assign(bm, patch);
return await dao.save(bm, { ...opt, skipIfEquals, tx: this.tx });
}
async deleteById(dao, id, opt) {
if (!id)
return 0;
return await this.deleteByIds(dao, [id], opt);
}
async deleteByIds(dao, ids, opt) {
return await dao.deleteByIds(ids, { ...opt, tx: this.tx });
}
}
exports.CommonDaoTransaction = CommonDaoTransaction;