@getanthill/datastore
Version:
Event-Sourced Datastore
525 lines • 21 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.init = exports.Models = void 0;
const get_1 = __importDefault(require("lodash/get"));
const omit_1 = __importDefault(require("lodash/omit"));
const pick_1 = __importDefault(require("lodash/pick"));
const isEqual_1 = __importDefault(require("lodash/isEqual"));
const Factory_1 = require("./Factory");
const constants_1 = require("../constants");
const internalModels = __importStar(require("./internal"));
const graph_1 = __importStar(require("./graph"));
class Models {
constructor(config, services) {
this.MODELS = new Map();
this.GRAPH = null;
this.config = config;
this.services = services;
this.reset();
this.loadModels(config.models);
}
async initInternalModels() {
this.services.telemetry.logger.debug('[Models] Creating internal models indexes...');
await this.createModelIndexes(internalModels._internal_models);
await this.createModelIndexes(internalModels._cache);
await this.createModelIndexes(internalModels._logs);
if (this.services.config.authz.isEnabled === true) {
await this.createModelIndexes(internalModels.attributes);
await this.createModelIndexes(internalModels.policies);
}
this.services.telemetry.logger.debug('[Models] Internal models indexes created');
}
addModel(modelConfig, canReplace = false) {
if (canReplace === false && this.MODELS.has(modelConfig.name)) {
const err = new Error('Model already exists');
// @ts-ignore
err.details = [
{
name: modelConfig.name,
},
];
throw err;
}
const factory = (0, Factory_1.Factory)(modelConfig, this.services);
this.MODELS.set(modelConfig.name, factory);
this.services.telemetry.logger.debug('[Models] Model added', {
name: modelConfig.name,
});
return factory;
}
loadModels(models, canReplace) {
for (const modelConfig of models) {
if (modelConfig.is_enabled === true) {
this.addModel(modelConfig, canReplace);
}
else {
this.removeModel(modelConfig.name);
}
}
}
reset() {
this.MODELS = new Map();
this.GRAPH = null;
this.addModel(internalModels._internal_models);
this.addModel(internalModels._cache);
this.addModel(internalModels._logs);
if (this.services.config.authz.isEnabled === true) {
this.addModel(internalModels.attributes);
this.addModel(internalModels.policies);
}
this.services.telemetry.logger.debug('[Models] Internal models initialized');
return this;
}
hasModel(modelName) {
return this.MODELS.has(modelName);
}
getModel(modelName) {
if (this.MODELS.has(modelName)) {
return this.MODELS.get(modelName);
}
throw new Error('Invalid Model');
}
isInternalModel(modelName) {
return modelName.startsWith('internal_') || modelName.startsWith('_');
}
getModelByCorrelationField(correlationField) {
for (const [modelName, model] of this.MODELS.entries()) {
if (this.isInternalModel(modelName) === false) {
if (model.getCorrelationField() === correlationField) {
return model;
}
}
}
return null;
}
load() {
return this.reload(false);
}
async reload(canReplace = true) {
const query = {
is_enabled: true,
name: {
$nin: ['internal_models', '_logs'],
},
};
if (this.services.config.features.loadOnlyModels !== null) {
this.services.telemetry.logger.debug('[Models] Loading those models only', {
models: this.services.config.features.loadOnlyModels,
});
query.name.$in = this.services.config.features.loadOnlyModels;
}
const modelsInDb = await this.getModel('internal_models')
.find(this.services.mongodb, query)
.toArray();
this.loadModels(modelsInDb, canReplace);
}
async createModel(modelConfig) {
try {
if (this.hasModel(modelConfig.name)) {
throw new Error('Model already exists');
}
const internalModel = this.factory(Models.INTERNAL_MODELS_COLLECTION);
const model = await internalModel.create({
type: 'EVENT_TYPE_CREATED',
is_enabled: false,
db: constants_1.DEFAULT_DATABASE_NAME,
...modelConfig,
});
if (model.state.is_enabled === true && 'indexes' in modelConfig) {
await this.createModelIndexes(model.state);
}
return model;
}
catch (err) {
if (err.message === 'Model already exists' ||
err.message.includes('E11000')) {
const err = new Error('Model already exists');
// @ts-ignore
err.details = [
{
name: modelConfig.name,
},
];
throw err;
}
throw err;
}
}
getModelIndexes(model) {
return Promise.all([
model
.getStatesCollection(model.db(this.services.mongodb))
.listIndexes()
.toArray(),
model
.getEventsCollection(model.db(this.services.mongodb))
.listIndexes()
.toArray(),
model
.getSnapshotsCollection(model.db(this.services.mongodb))
.listIndexes()
.toArray(),
]);
}
createModelCollections(modelConfig, model) {
return Promise.all([
model
.db(this.services.mongodb)
.createCollection(modelConfig.name)
.catch(
/* istanbul ignore next */ () => {
/* istanbul ignore next */
this.services.telemetry.logger.debug('[Models] Collection already exists', {
model: modelConfig.name,
});
}),
model
.db(this.services.mongodb)
.createCollection(`${modelConfig.name}_events`)
.catch(
/* istanbul ignore next */ () => {
/* istanbul ignore next */
this.services.telemetry.logger.debug('[Models] Collection already exists', {
model: `${modelConfig.name}_events`,
});
}),
model
.db(this.services.mongodb)
.createCollection(`${modelConfig.name}_snapshots`)
.catch(
/* istanbul ignore next */ () => {
/* istanbul ignore next */
this.services.telemetry.logger.debug('[Models] Collection already exists', {
model: `${modelConfig.name}_snapshots`,
});
}),
]);
}
async createModelIndexes(modelConfig) {
const model = (0, Factory_1.Factory)(modelConfig, this.services);
this.services.telemetry.logger.info('[Models] Model indexes creation...', {
model: modelConfig.name,
});
await this.createModelCollections(modelConfig, model);
const indexesBeforeUpdate = await this.getModelIndexes(model);
const internalIndexes = model.getRequiredIndexes();
// Updated indexes:
const indexes = (modelConfig.indexes ?? []).map((index) => {
/**
* @fixme this is a hack to allow the creation of a nested index
* in MongoDB in same time to store the modelConfig in the
* database.
*
* We should escape keys including a dot from any kind of model,
* not only internal ones to surpass this limitation.
*/
const _index = {
...index,
fields: {},
};
for (const field in index.fields) {
/* @ts-ignore */
_index.fields[field.replace(/:+/g, '.')] = index.fields[field];
}
return _index;
});
// Create any kind of index as defined in the configuration:
this.services.telemetry.logger.debug('[Models] Specific Model indexes creation...', {
model: modelConfig.name,
internal_indexes: internalIndexes,
specific_indexes: indexes,
});
indexes.push(...internalIndexes[modelConfig.name].map((i) => ({
collection: modelConfig.name,
fields: i[0],
opts: i[1],
})));
indexes.push(...internalIndexes[`${modelConfig.name}_events`].map((i) => ({
collection: `${modelConfig.name}_events`,
fields: i[0],
opts: i[1],
})));
indexes.push(...internalIndexes[`${modelConfig.name}_snapshots`].map((i) => ({
collection: `${modelConfig.name}_snapshots`,
fields: i[0],
opts: i[1],
})));
const _indexesBeforeUpdate = [
...indexesBeforeUpdate[0].map((i) => ({
collection: modelConfig.name,
fields: i.key,
opts: (0, omit_1.default)(i, 'key', 'v'),
})),
...indexesBeforeUpdate[1].map((i) => ({
collection: `${modelConfig.name}_events`,
fields: i.key,
opts: (0, omit_1.default)(i, 'key', 'v'),
})),
...indexesBeforeUpdate[2].map((i) => ({
collection: `${modelConfig.name}_snapshots`,
fields: i.key,
opts: (0, omit_1.default)(i, 'key', 'v'),
})),
];
/**
* Ensure that this index is created before any other user-defined
* unique index.
* @see Generic.ts L320 for more details
*/
indexes.unshift(indexes.find((def) => def.opts.name === 'correlation_id_unicity'));
const alreadyInitialized = new Set();
const initialized = new Map();
for (const index of indexes) {
try {
const { collection, fields, opts } = index;
const indexId = JSON.stringify({
collection,
fields: Object.keys(fields).sort().join(',').toLowerCase(),
});
if (alreadyInitialized.has(indexId)) {
this.services.telemetry.logger.debug('[Models] Index already seen and initialized (duplicate). No action performed.', {
index,
index_id: indexId,
});
continue;
}
alreadyInitialized.add(indexId);
if (_indexesBeforeUpdate.findIndex((_index) => (0, isEqual_1.default)(_index, index)) !==
-1) {
this.services.telemetry.logger.debug('[Models] Index already initialized. No action performed.', {
index,
index_id: indexId,
});
continue;
}
if (opts.unique !== true) {
// @ts-expect-error index is valid here
fields['_id'] = 1;
}
await model
.db(this.services.mongodb)
.collection(collection)
// @ts-expect-error fields is a valid IndexSpecification
.createIndex(fields, opts);
initialized.set(index.opts.name, JSON.stringify(index));
/* istanbul ignore next */
}
catch (err) {
/* istanbul ignore next */
this.services.telemetry.logger.warn('[Models] Index creation error', {
err,
});
}
}
(initialized.size > 0 &&
this.services.telemetry.logger.info('[Models] Model indexes created...', {
model: modelConfig.name,
created: Array.from(initialized.keys()),
})) ||
this.services.telemetry.logger.debug('[Models] Model indexes created...', {
model: modelConfig.name,
created: Array.from(initialized.values()),
});
return this.getModelIndexes(model);
}
async updateModel(modelName, modelConfig) {
const InternalModels = this.getModel(Models.INTERNAL_MODELS_COLLECTION);
this.services.telemetry.logger.debug('[Models] Updating Model...', {
model: modelName,
});
const [targetModelState] = await InternalModels.find(this.services.mongodb, {
name: modelName,
}).toArray();
if (!targetModelState) {
throw new Error('Invalid Model');
}
const targetModel = this.factory(Models.INTERNAL_MODELS_COLLECTION, targetModelState.model_id);
const model = await targetModel?.update({
type: constants_1.EVENT_TYPE_UPDATED,
...modelConfig,
});
this.services.telemetry.logger.info('[Models] Model updated', {
model: modelName,
});
if (model.state.is_enabled === true && 'indexes' in modelConfig) {
await this.createModelIndexes(model.state);
}
return targetModel;
}
removeModel(modelName) {
this.MODELS.delete(modelName);
return this;
}
factory(modelName, correlationId) {
const Model = this.getModel(modelName);
return new Model(this.services, correlationId);
}
getGraph(options) {
if (this.GRAPH !== null) {
return this.GRAPH;
}
this.GRAPH = (0, graph_1.default)(this, options);
return this.GRAPH;
}
setGraph(graph) {
this.GRAPH = graph;
}
getEntitiesFromGraph(modelName, query, options) {
return (0, graph_1.getEntitiesFromGraph)(this.services, modelName, query, options);
}
async getFromCache(cacheId) {
if (this.services.config.features.cache.isEnabled !== true) {
return null;
}
const scope = this.services.config.features.cache.scope;
const Cache = this.getModel('_cache');
const [cacheEntry] = await Cache.find(this.services.mongodb, {
cache_id: `${scope}/${cacheId}`,
}).toArray();
return cacheEntry?.value ?? null;
}
setToCache(cacheId, value, expiresBy = new Date().toISOString()) {
if (this.services.config.features.cache.isEnabled !== true) {
return null;
}
const scope = this.services.config.features.cache.scope;
const Cache = this.getModel('_cache');
const oasCache = new Cache(this.services, `${scope}/${cacheId}`);
return (oasCache
// @ts-ignore we cannot assert at this point if the event type is of type CREATED or UPDATED
.upsert({
value,
expires_by: expiresBy,
})
.catch((err) => {
this.services.telemetry.logger.error('[Models] Failed caching...', {
err,
});
}));
}
async rotateEncryptionKeyOnCollection(Model, collection, query, encryptedFields) {
const cursor = collection.find(query);
let count = 0;
while (await cursor.hasNext()) {
const entity = await cursor.next();
if (entity === null) {
return count;
}
const decryptedValue = Model.decrypt(entity);
const decryptedFields = encryptedFields.filter((encryptedField) => !(0, get_1.default)(decryptedValue, encryptedField)?.encrypted);
const encryptedValue = await Model.encrypt((0, pick_1.default)(decryptedValue, decryptedFields));
await collection.updateOne({
_id: entity._id,
}, {
$set: encryptedValue,
});
count += 1;
}
return count;
}
async rotateEncryptionKey(onlyModels = null) {
for (const Model of this.MODELS.values()) {
const modelConfig = Model.getModelConfig();
if (Array.isArray(onlyModels) && !onlyModels.includes(modelConfig.name)) {
continue;
}
const encryptedFields = modelConfig.encrypted_fields ?? [];
if (encryptedFields.length === 0) {
continue;
}
const eligibleEncryptionKeys = Model.getEligibleKeys(Model.getEncryptionKeys());
const hashedEligibleEncryptionKeys = Model.getEligibleKeys(Model.getHashesEncryptionKeys());
this.services.telemetry.logger.info('[Models] Encryption key rotation started', {
model: modelConfig.name,
});
const query = {
$or: [],
};
encryptedFields.forEach((encryptedField) => query.$or.push({
$and: [
{
[encryptedField]: {
$exists: true,
},
},
...hashedEligibleEncryptionKeys.map((key) => ({
[encryptedField + '.encrypted']: {
$not: new RegExp('^' + key.slice(0, 6) + ':', 'i'),
},
})),
...eligibleEncryptionKeys.map((key) => ({
[encryptedField + '.encrypted']: {
$not: new RegExp('^' + key.slice(0, 6) + ':', 'i'),
},
})),
],
}));
const counts = await Promise.all([
this.rotateEncryptionKeyOnCollection(Model, Model.getStatesCollection(Model.db(this.services.mongodb)), query, encryptedFields),
this.rotateEncryptionKeyOnCollection(Model, Model.getEventsCollection(Model.db(this.services.mongodb)), query, encryptedFields),
this.rotateEncryptionKeyOnCollection(Model, Model.getSnapshotsCollection(Model.db(this.services.mongodb)), query, encryptedFields),
]);
this.services.telemetry.logger.info('[Models] Encryption key rotation ended', {
model: Model.getModelConfig().name,
entities_count: counts[0],
events_count: counts[1],
snapshots_count: counts[2],
});
}
}
async log(level, modelName, correlationId, message, context) {
try {
const log = this.factory('_logs');
await log.create({
type: constants_1.EVENT_TYPE_CREATED,
level,
model: modelName,
correlation_id: correlationId,
message,
context,
});
return log;
}
catch (err) {
this.services.telemetry.logger.error('[Models] Failed logging...', {
level,
model: modelName,
message,
err,
});
}
}
}
exports.Models = Models;
Models.INTERNAL_MODELS_COLLECTION = internalModels._internal_models.name;
function init(config, services) {
return new Models(config, services);
}
exports.init = init;
//# sourceMappingURL=index.js.map