@getanthill/datastore/src/models/index.ts

Event-Sourced Datastore

import type {
  ModelConfig,
  ModelIndex,
  GenericType,
  Services,
  AnyObject,
  ModelInstance,
} from '../typings';
import type { GraphI } from './graph';

import get from 'lodash/get';
import omit from 'lodash/omit';
import pick from 'lodash/pick';
import isEqual from 'lodash/isEqual';

import { Factory } from './Factory';
import {
  DEFAULT_DATABASE_NAME,
  EVENT_TYPE_CREATED,
  EVENT_TYPE_UPDATED,
} from '../constants';

import * as internalModels from './internal';
import graph, { getEntitiesFromGraph } from './graph';
import { Collection } from 'mongodb';

export interface ModelsConfig {
  models: ModelConfig[];
}

export class Models {
  static INTERNAL_MODELS_COLLECTION: string =
    internalModels._internal_models.name;
  MODELS: Map<string, GenericType> = new Map();
  GRAPH: GraphI | null = null;
  config: ModelsConfig;
  services: Services;

  constructor(config: ModelsConfig, services: Services) {
    this.config = config;
    this.services = services;

    this.reset();

    this.loadModels(config.models);
  }

  async initInternalModels(): Promise<void> {
    this.services.telemetry.logger.debug(
      '[Models] Creating internal models indexes...',
    );

    await this.createModelIndexes(internalModels._internal_models);
    await this.createModelIndexes(internalModels._cache);
    await this.createModelIndexes(internalModels._logs);

    if (this.services.config.authz.isEnabled === true) {
      await this.createModelIndexes(internalModels.attributes);
      await this.createModelIndexes(internalModels.policies);
    }

    this.services.telemetry.logger.debug(
      '[Models] Internal models indexes created',
    );
  }

  addModel(modelConfig: ModelConfig, canReplace = false): any {
    if (canReplace === false && this.MODELS.has(modelConfig.name)) {
      const err = new Error('Model already exists');

      // @ts-ignore
      err.details = [
        {
          name: modelConfig.name,
        },
      ];

      throw err;
    }

    const factory = Factory(modelConfig, this.services);

    this.MODELS.set(modelConfig.name, factory);
    this.services.telemetry.logger.debug('[Models] Model added', {
      name: modelConfig.name,
    });

    return factory;
  }

  loadModels(models: ModelConfig[], canReplace?: boolean) {
    for (const modelConfig of models) {
      if (modelConfig.is_enabled === true) {
        this.addModel(modelConfig, canReplace);
      } else {
        this.removeModel(modelConfig.name);
      }
    }
  }

  reset(): Models {
    this.MODELS = new Map();
    this.GRAPH = null;

    this.addModel(internalModels._internal_models);
    this.addModel(internalModels._cache);
    this.addModel(internalModels._logs);

    if (this.services.config.authz.isEnabled === true) {
      this.addModel(internalModels.attributes);
      this.addModel(internalModels.policies);
    }

    this.services.telemetry.logger.debug(
      '[Models] Internal models initialized',
    );

    return this;
  }

  hasModel(modelName: string) {
    return this.MODELS.has(modelName);
  }

  getModel(modelName: string): GenericType {
    if (this.MODELS.has(modelName)) {
      return this.MODELS.get(modelName)!;
    }

    throw new Error('Invalid Model');
  }

  isInternalModel(modelName: string): boolean {
    return modelName.startsWith('internal_') || modelName.startsWith('_');
  }

  getModelByCorrelationField(correlationField: string) {
    for (const [modelName, model] of this.MODELS.entries()) {
      if (this.isInternalModel(modelName) === false) {
        if (model.getCorrelationField() === correlationField) {
          return model;
        }
      }
    }

    return null;
  }

  load() {
    return this.reload(false);
  }

  async reload(canReplace = true) {
    const query: any = {
      is_enabled: true,
      name: {
        $nin: ['internal_models', '_logs'],
      },
    };

    if (this.services.config.features.loadOnlyModels !== null) {
      this.services.telemetry.logger.debug(
        '[Models] Loading those models only',
        {
          models: this.services.config.features.loadOnlyModels,
        },
      );

      query.name.$in = this.services.config.features.loadOnlyModels;
    }

    const modelsInDb = await this.getModel('internal_models')
      .find(this.services.mongodb, query)
      .toArray();

    this.loadModels(modelsInDb, canReplace);
  }

  async createModel(modelConfig: ModelConfig) {
    try {
      if (this.hasModel(modelConfig.name)) {
        throw new Error('Model already exists');
      }

      const internalModel = this.factory(Models.INTERNAL_MODELS_COLLECTION);

      const model = await internalModel.create({
        type: 'EVENT_TYPE_CREATED',
        is_enabled: false,
        db: DEFAULT_DATABASE_NAME,
        ...modelConfig,
      });

      if (model.state.is_enabled === true && 'indexes' in modelConfig) {
        await this.createModelIndexes(model.state);
      }

      return model;
    } catch (err: any) {
      if (
        err.message === 'Model already exists' ||
        err.message.includes('E11000')
      ) {
        const err = new Error('Model already exists');

        // @ts-ignore
        err.details = [
          {
            name: modelConfig.name,
          },
        ];

        throw err;
      }

      throw err;
    }
  }

  getModelIndexes(model: GenericType): Promise<any[]> {
    return Promise.all([
      model
        .getStatesCollection(model.db(this.services.mongodb))
        .listIndexes()
        .toArray(),
      model
        .getEventsCollection(model.db(this.services.mongodb))
        .listIndexes()
        .toArray(),
      model
        .getSnapshotsCollection(model.db(this.services.mongodb))
        .listIndexes()
        .toArray(),
    ]);
  }

  createModelCollections(
    modelConfig: ModelConfig,
    model: GenericType,
  ): Promise<any[]> {
    return Promise.all([
      model
        .db(this.services.mongodb)
        .createCollection(modelConfig.name)
        .catch(
          /* istanbul ignore next */ () => {
            /* istanbul ignore next */
            this.services.telemetry.logger.debug(
              '[Models] Collection already exists',
              {
                model: modelConfig.name,
              },
            );
          },
        ),
      model
        .db(this.services.mongodb)
        .createCollection(`${modelConfig.name}_events`)
        .catch(
          /* istanbul ignore next */ () => {
            /* istanbul ignore next */
            this.services.telemetry.logger.debug(
              '[Models] Collection already exists',
              {
                model: `${modelConfig.name}_events`,
              },
            );
          },
        ),
      model
        .db(this.services.mongodb)
        .createCollection(`${modelConfig.name}_snapshots`)
        .catch(
          /* istanbul ignore next */ () => {
            /* istanbul ignore next */
            this.services.telemetry.logger.debug(
              '[Models] Collection already exists',
              {
                model: `${modelConfig.name}_snapshots`,
              },
            );
          },
        ),
    ]);
  }

  async createModelIndexes(modelConfig: ModelConfig) {
    const model = Factory(modelConfig, this.services);
    this.services.telemetry.logger.info('[Models] Model indexes creation...', {
      model: modelConfig.name,
    });

    await this.createModelCollections(modelConfig, model);

    const indexesBeforeUpdate = await this.getModelIndexes(model);
    const internalIndexes = model.getRequiredIndexes();

    // Updated indexes:
    const indexes = (modelConfig.indexes ?? []).map((index) => {
      /**
       * @fixme this is a hack to allow the creation of a nested index
       * in MongoDB in same time to store the modelConfig in the
       * database.
       *
       * We should escape keys including a dot from any kind of model,
       * not only internal ones to surpass this limitation.
       */
      const _index: ModelIndex = {
        ...index,
        fields: {},
      };

      for (const field in index.fields) {
        /* @ts-ignore */
        _index.fields[field.replace(/:+/g, '.')] = index.fields[field];
      }

      return _index;
    });

    // Create any kind of index as defined in the configuration:
    this.services.telemetry.logger.debug(
      '[Models] Specific Model indexes creation...',
      {
        model: modelConfig.name,
        internal_indexes: internalIndexes,
        specific_indexes: indexes,
      },
    );

    indexes.push(
      ...internalIndexes[modelConfig.name].map((i) => ({
        collection: modelConfig.name,
        fields: i[0],
        opts: i[1],
      })),
    );

    indexes.push(
      ...internalIndexes[`${modelConfig.name}_events`].map((i) => ({
        collection: `${modelConfig.name}_events`,
        fields: i[0],
        opts: i[1],
      })),
    );

    indexes.push(
      ...internalIndexes[`${modelConfig.name}_snapshots`].map((i) => ({
        collection: `${modelConfig.name}_snapshots`,
        fields: i[0],
        opts: i[1],
      })),
    );

    const _indexesBeforeUpdate = [
      ...indexesBeforeUpdate[0].map((i: any) => ({
        collection: modelConfig.name,
        fields: i.key,
        opts: omit(i, 'key', 'v'),
      })),
      ...indexesBeforeUpdate[1].map((i: any) => ({
        collection: `${modelConfig.name}_events`,
        fields: i.key,
        opts: omit(i, 'key', 'v'),
      })),
      ...indexesBeforeUpdate[2].map((i: any) => ({
        collection: `${modelConfig.name}_snapshots`,
        fields: i.key,
        opts: omit(i, 'key', 'v'),
      })),
    ];

    /**
     * Ensure that this index is created before any other user-defined
     * unique index.
     * @see Generic.ts L320 for more details
     */
    indexes.unshift(
      indexes.find((def) => def.opts.name === 'correlation_id_unicity')!,
    );

    const alreadyInitialized = new Set();
    const initialized: Map<string, string> = new Map();
    for (const index of indexes) {
      try {
        const { collection, fields, opts } = index;
        const indexId = JSON.stringify({
          collection,
          fields: Object.keys(fields).sort().join(',').toLowerCase(),
        });

        if (alreadyInitialized.has(indexId)) {
          this.services.telemetry.logger.debug(
            '[Models] Index already seen and initialized (duplicate). No action performed.',
            {
              index,
              index_id: indexId,
            },
          );

          continue;
        }

        alreadyInitialized.add(indexId);

        if (
          _indexesBeforeUpdate.findIndex((_index) => isEqual(_index, index)) !==
          -1
        ) {
          this.services.telemetry.logger.debug(
            '[Models] Index already initialized. No action performed.',
            {
              index,
              index_id: indexId,
            },
          );

          continue;
        }

        if (opts.unique !== true) {
          // @ts-expect-error index is valid here
          fields['_id'] = 1;
        }

        await model
          .db(this.services.mongodb)
          .collection(collection)
          // @ts-expect-error fields is a valid IndexSpecification
          .createIndex(fields, opts);

        initialized.set(index.opts.name!, JSON.stringify(index));
        /* istanbul ignore next */
      } catch (err: any) {
        /* istanbul ignore next */
        this.services.telemetry.logger.warn('[Models] Index creation error', {
          err,
        });
      }
    }

    (initialized.size > 0 &&
      this.services.telemetry.logger.info('[Models] Model indexes created...', {
        model: modelConfig.name,
        created: Array.from(initialized.keys()),
      })) ||
      this.services.telemetry.logger.debug(
        '[Models] Model indexes created...',
        {
          model: modelConfig.name,
          created: Array.from(initialized.values()),
        },
      );

    return this.getModelIndexes(model);
  }

  async updateModel(modelName: string, modelConfig: Partial<ModelConfig>) {
    const InternalModels = this.getModel(Models.INTERNAL_MODELS_COLLECTION);

    this.services.telemetry.logger.debug('[Models] Updating Model...', {
      model: modelName,
    });

    const [targetModelState] = await InternalModels.find(
      this.services.mongodb,
      {
        name: modelName,
      },
    ).toArray();

    if (!targetModelState) {
      throw new Error('Invalid Model');
    }

    const targetModel = this.factory(
      Models.INTERNAL_MODELS_COLLECTION,
      targetModelState.model_id,
    );

    const model = await targetModel?.update({
      type: EVENT_TYPE_UPDATED,
      ...modelConfig,
    });
    this.services.telemetry.logger.info('[Models] Model updated', {
      model: modelName,
    });

    if (model.state.is_enabled === true && 'indexes' in modelConfig) {
      await this.createModelIndexes(model.state);
    }

    return targetModel;
  }

  removeModel(modelName: string): Models {
    this.MODELS.delete(modelName);

    return this;
  }

  factory(modelName: string, correlationId?: string) {
    const Model = this.getModel(modelName);

    return new Model(this.services, correlationId);
  }

  getGraph(options?: any): GraphI {
    if (this.GRAPH !== null) {
      return this.GRAPH;
    }

    this.GRAPH = graph(this, options);

    return this.GRAPH;
  }

  setGraph(graph: GraphI | null) {
    this.GRAPH = graph;
  }

  getEntitiesFromGraph(
    modelName: string,
    query: any,
    options?: {
      graph?: GraphI;
      models?: string[];
      withCorrelationFieldOnly?: boolean;
      handler?: (services: Services, Model: any, entity: any) => any;
    },
  ): Promise<Map<string, any>> {
    return getEntitiesFromGraph(this.services, modelName, query, options);
  }

  async getFromCache(cacheId: string) {
    if (this.services.config.features.cache.isEnabled !== true) {
      return null;
    }

    const scope: string = this.services.config.features.cache.scope;
    const Cache = this.getModel('_cache');

    const [cacheEntry] = await Cache.find(this.services.mongodb, {
      cache_id: `${scope}/${cacheId}`,
    }).toArray();

    return cacheEntry?.value ?? null;
  }

  setToCache(
    cacheId: string,
    value: AnyObject,
    expiresBy: string = new Date().toISOString(),
  ) {
    if (this.services.config.features.cache.isEnabled !== true) {
      return null;
    }

    const scope: string = this.services.config.features.cache.scope;
    const Cache = this.getModel('_cache');

    const oasCache = new Cache(this.services, `${scope}/${cacheId}`);

    return (
      oasCache
        // @ts-ignore we cannot assert at this point if the event type is of type CREATED or UPDATED
        .upsert({
          value,
          expires_by: expiresBy,
        })
        .catch((err) => {
          this.services.telemetry.logger.error('[Models] Failed caching...', {
            err,
          });
        })
    );
  }

  async rotateEncryptionKeyOnCollection(
    Model: GenericType,
    collection: Collection,
    query: object,
    encryptedFields: string[],
  ) {
    const cursor = collection.find(query);
    let count = 0;

    while (await cursor.hasNext()) {
      const entity = await cursor.next();

      if (entity === null) {
        return count;
      }

      const decryptedValue = Model.decrypt(entity);
      const decryptedFields = encryptedFields.filter(
        (encryptedField) => !get(decryptedValue, encryptedField)?.encrypted,
      );

      const encryptedValue = await Model.encrypt(
        pick(decryptedValue, decryptedFields),
      );

      await collection.updateOne(
        {
          _id: entity._id,
        },
        {
          $set: encryptedValue,
        },
      );

      count += 1;
    }

    return count;
  }

  async rotateEncryptionKey(onlyModels: string[] | null = null) {
    for (const Model of this.MODELS.values()) {
      const modelConfig = Model.getModelConfig();

      if (Array.isArray(onlyModels) && !onlyModels.includes(modelConfig.name)) {
        continue;
      }

      const encryptedFields = modelConfig.encrypted_fields ?? [];

      if (encryptedFields.length === 0) {
        continue;
      }

      const eligibleEncryptionKeys = Model.getEligibleKeys(
        Model.getEncryptionKeys(),
      );
      const hashedEligibleEncryptionKeys = Model.getEligibleKeys(
        Model.getHashesEncryptionKeys(),
      );

      this.services.telemetry.logger.info(
        '[Models] Encryption key rotation started',
        {
          model: modelConfig.name,
        },
      );
      const query: { $or: { [key: string]: any }[] } = {
        $or: [],
      };
      encryptedFields.forEach((encryptedField) =>
        query.$or.push({
          $and: [
            {
              [encryptedField]: {
                $exists: true,
              },
            },
            ...hashedEligibleEncryptionKeys.map((key) => ({
              [encryptedField + '.encrypted']: {
                $not: new RegExp('^' + key.slice(0, 6) + ':', 'i'),
              },
            })),
            ...eligibleEncryptionKeys.map((key) => ({
              [encryptedField + '.encrypted']: {
                $not: new RegExp('^' + key.slice(0, 6) + ':', 'i'),
              },
            })),
          ],
        }),
      );

      const counts: number[] = await Promise.all([
        this.rotateEncryptionKeyOnCollection(
          Model,
          Model.getStatesCollection(Model.db(this.services.mongodb)),
          query,
          encryptedFields,
        ),
        this.rotateEncryptionKeyOnCollection(
          Model,
          Model.getEventsCollection(Model.db(this.services.mongodb)),
          query,
          encryptedFields,
        ),
        this.rotateEncryptionKeyOnCollection(
          Model,
          Model.getSnapshotsCollection(Model.db(this.services.mongodb)),
          query,
          encryptedFields,
        ),
      ]);
      this.services.telemetry.logger.info(
        '[Models] Encryption key rotation ended',
        {
          model: Model.getModelConfig().name,
          entities_count: counts[0],
          events_count: counts[1],
          snapshots_count: counts[2],
        },
      );
    }
  }

  async log(
    level: number,
    modelName: string,
    correlationId: string,
    message: string,
    context: any,
  ): Promise<ModelInstance | void> {
    try {
      const log = this.factory('_logs');

      await log.create({
        type: EVENT_TYPE_CREATED,
        level,
        model: modelName,
        correlation_id: correlationId,
        message,
        context,
      });

      return log;
    } catch (err) {
      this.services.telemetry.logger.error('[Models] Failed logging...', {
        level,
        model: modelName,
        message,
        err,
      });
    }
  }
}

export function init(config: ModelsConfig, services: Services): Models {
  return new Models(config, services);
}