@getanthill/datastore/src/sdk/Datastore.ts

Event-Sourced Datastore

import type {
  RawAxiosRequestHeaders,
  AxiosResponse,
  RawAxiosResponseHeaders,
  AxiosResponseHeaders,
} from 'axios';
import type {
  DatastoreImportFixture,
  DatastoreImportLink,
  ModelConfig,
  Telemetry,
} from '../typings';

import { EventEmitter } from 'events';

import {
  cloneDeep,
  get as _get,
  omit,
  mapValues,
  isObject,
  merge,
  mergeWith,
} from 'lodash';

import * as jsonpatch from 'fast-json-patch';

import Core from './Core';
import Streams, { StreamClose, StreamHandler } from './Streams';

import { Iteration, MultiQuery, walkMulti } from './utils';

export const ERROR_MISSING_MODEL_NAME = new Error('Missing Model name');
export const ERROR_MISSING_CORRELATION_ID = new Error('Missing Correlation ID');
export const ERROR_MISSING_JSON_PATCH = new Error('Missing JSON Patch');
export const ERROR_STREAM_MAX_RECONNECTION_ATTEMPTS_REACHED = new Error(
  'Max reconnection attempts reached for streaming',
);

function mergeWithArrays(objValue: any, srcValue: []) {
  if (Array.isArray(objValue)) {
    return objValue.concat(srcValue);
  }
}

function mapValuesDeep(obj: any, cb: any, key?: any): any {
  if (Array.isArray(obj)) {
    return obj.map((val: any, key: any) => mapValuesDeep(val, cb, key));
  } else if (isObject(obj)) {
    return mapValues(obj, (val: any) => mapValuesDeep(val, cb, key));
  } else {
    return cb(obj, key);
  }
}

export interface DatastoreConfig {
  /**
   * Datastore API base URL
   * default: http://localhost:3001
   */
  baseUrl?: string;
  /**
   * API Access token
   */
  token?: string;
  /**
   * Requests timeout in milliseconds
   */
  timeout?: number;
  /**
   * Log HTTP errors
   */
  debug?: boolean;
  /**
   * Telemetry instance of logging and metrics
   */
  telemetry?: Telemetry; // @getanthill/telemetry
  /**
   * Type of the stream connector:
   * - 'http': Server Sent Events HTTP Stream
   * - 'amqp': RabbitMQ stream
   */
  connector?: 'http' | 'amqp';
  /**
   * Force to fetch documents from primary to guarantee
   * accessing the most up-to-date data.
   */
  forcePrimary?: true;
  walk?: {
    maxPageSize?: number;
  };
}

export default class Datastore extends EventEmitter {
  name = '';
  config: DatastoreConfig = {
    baseUrl: 'http://localhost:3001',
    timeout: 10000,
    token: 'token',
    debug: false,
    connector: 'http',
    walk: {
      maxPageSize: Infinity,
    },
  };
  telemetry?: Telemetry;

  public core: Core;
  public streams: Streams;

  constructor(config: DatastoreConfig = {}) {
    super();

    this.config = merge({}, this.config, config);

    this.core = new Core(this.config);
    this.streams = new Streams(this.config, this.core);

    if (this.config.telemetry) {
      this.telemetry = this.config.telemetry;
    }
  }

  heartbeat(): Promise<AxiosResponse> {
    return this.core.request({ method: 'get', url: '/heartbeat' });
  }

  _checkCorrelationIdExistence(correlationId: string) {
    if (!correlationId) {
      throw ERROR_MISSING_CORRELATION_ID;
    }
  }

  _checkModelNameExistence(model: Partial<ModelConfig> | undefined) {
    if (!model || !model.name) {
      throw ERROR_MISSING_MODEL_NAME;
    }
  }

  getModels(): Promise<AxiosResponse> {
    return this.core.request({
      method: 'get',
      url: this.core.getPath('admin'),
    });
  }

  getGraph(): Promise<AxiosResponse> {
    return this.core.request({
      method: 'get',
      url: this.core.getPath('admin', 'graph'),
    });
  }

  getModel(model: string): Promise<AxiosResponse> {
    return this.core.request({
      method: 'get',
      url: this.core.getPath('admin'),
      params: {
        model,
      },
    });
  }

  rotateEncryptionKeys(models?: string[]): Promise<AxiosResponse> {
    return this.core.request({
      method: 'post',
      url: this.core.getPath('admin', 'rotate', 'keys'),
      params: { models },
    });
  }

  createModel(modelConfig: ModelConfig): Promise<AxiosResponse> {
    this._checkModelNameExistence(modelConfig);

    return this.core.request({
      method: 'post',
      url: this.core.getPath('admin'),
      data: modelConfig,
    });
  }

  updateModel(modelConfig: Partial<ModelConfig>): Promise<AxiosResponse> {
    this._checkModelNameExistence(modelConfig);

    return this.core.request({
      method: 'post',
      url: this.core.getPath('admin', modelConfig.name!),
      data: modelConfig,
    });
  }

  createModelIndexes(modelConfig?: ModelConfig): Promise<AxiosResponse> {
    this._checkModelNameExistence(modelConfig);

    return this.core.request({
      method: 'post',
      url: this.core.getPath('admin', modelConfig!.name, 'indexes'),
      data: modelConfig,
    });
  }

  getSchema(modelName: string): Promise<AxiosResponse> {
    this._checkModelNameExistence({ name: modelName });
    return this.core.request({
      method: 'get',
      url: this.core.getPath('admin', modelName, 'schema'),
    });
  }

  encrypt(
    modelName: string,
    data: object[],
    fields: string[] = [],
  ): Promise<AxiosResponse> {
    this._checkModelNameExistence({ name: modelName });
    return this.core.request({
      method: 'post',
      url: this.core.getPath(modelName, 'encrypt'),
      params: {
        fields,
      },
      data,
    });
  }

  decrypt(
    modelName: string,
    data: object[],
    fields: string[] = [],
  ): Promise<AxiosResponse> {
    this._checkModelNameExistence({ name: modelName });
    return this.core.request({
      method: 'post',
      url: this.core.getPath(modelName, 'decrypt'),
      params: {
        fields,
      },
      data,
    });
  }

  create(
    modelName: string,
    payload: object,
    headers?: RawAxiosRequestHeaders,
  ): Promise<AxiosResponse> {
    return this.core.request({
      method: 'post',
      url: this.core.getPath(modelName),
      data: payload,
      headers,
    });
  }

  apply(
    modelName: string,
    correlationId: string,
    eventType: string,
    eventVersion: string,
    payload: object,
    headers?: RawAxiosRequestHeaders,
  ): Promise<AxiosResponse> {
    this._checkCorrelationIdExistence(correlationId);

    return this.core.request({
      method: 'post',
      url: this.core.getPath(
        modelName,
        correlationId,
        eventType.toLowerCase(),
        eventVersion,
      ),
      data: payload,
      headers,
    });
  }

  update<T>(
    modelName: string,
    correlationId: string,
    payload: T,
    headers?: RawAxiosRequestHeaders,
  ): Promise<AxiosResponse> {
    this._checkCorrelationIdExistence(correlationId);

    return this.core.request({
      method: 'post',
      url: this.core.getPath(modelName, correlationId),
      data: payload,
      headers,
    });
  }

  patch(
    modelName: string,
    correlationId: string,
    jsonPatch: object[],
    headers?: RawAxiosRequestHeaders,
  ): Promise<AxiosResponse> {
    this._checkCorrelationIdExistence(correlationId);
    if (!jsonPatch) {
      throw ERROR_MISSING_JSON_PATCH;
    }

    return this.core.request({
      method: 'patch',
      url: this.core.getPath(modelName, correlationId),
      data: {
        json_patch: jsonPatch,
      },
      headers,
    });
  }

  get(
    modelName: string,
    correlationId: string,
    headers?: {
      'force-primary'?: true;
    },
  ): Promise<AxiosResponse> {
    this._checkCorrelationIdExistence(correlationId);

    return this.core.request({
      method: 'get',
      url: this.core.getPath(modelName, correlationId),
      headers,
    });
  }

  async count(model: string, query: object, source = 'entities') {
    const { headers } = await this[source !== 'events' ? 'find' : 'allEvents'](
      model,
      query,
      0,
      0,
    );

    return Number.parseInt(headers.count!, 10);
  }

  find(
    model: string,
    query: object,
    page?: number,
    pageSize?: number,
    headers: {
      page?: number;
      'page-size'?: number;
      'cursor-last-id'?: string;
      'force-primary'?: true;
    } = {},
  ): Promise<AxiosResponse> {
    if (page !== undefined) {
      headers.page = page;
    }

    if (pageSize !== undefined) {
      headers['page-size'] = pageSize;
    }

    return this.core.request({
      method: 'get',
      url: this.core.getPath(model),
      params: query,
      headers,
    });
  }

  events(
    model: string,
    correlationId: string,
    page?: number,
    pageSize?: number,
  ): Promise<AxiosResponse> {
    this._checkCorrelationIdExistence(correlationId);

    const headers: { page?: number; 'page-size'?: number } = {};
    if (page !== undefined) {
      headers.page = page;
    }

    if (pageSize !== undefined) {
      headers['page-size'] = pageSize;
    }

    return this.core.request({
      method: 'get',
      url: this.core.getPath(model, correlationId, 'events'),
      headers,
    });
  }

  allEvents(
    model: string,
    query: object = {},
    page?: number,
    pageSize?: number,
    headers: {
      page?: number;
      'page-size'?: number;
      'cursor-last-id'?: string;
    } = {},
  ): Promise<AxiosResponse> {
    if (page !== undefined) {
      headers.page = page;
    }

    if (pageSize !== undefined) {
      headers['page-size'] = pageSize;
    }

    return this.core.request({
      method: 'get',
      url: this.core.getPath(model, 'events'),
      headers,
      params: query,
    });
  }

  async firstEventVersion(
    model: string,
    query: object,
    sort: any,
    defaultValue: number,
    headers?: {
      page?: number;
      'page-size'?: number;
      'cursor-last-id'?: string;
    },
  ): Promise<number> {
    const _query = cloneDeep(query);

    if ('updated_at' in _query) {
      // @ts-ignore
      _query.created_at = _query.updated_at;
      delete _query.updated_at;
    }

    const {
      data: [event],
    } = await this.allEvents(
      model,
      {
        ..._query,
        // @ts-ignore
        _sort: sort,
        _fields: {
          version: 1,
        },
      },
      0,
      1,
      headers,
    );

    return event ? event.version : defaultValue;
  }

  async minEventsVersion(
    model: string,
    query: object,
    headers?: {
      page?: number;
      'page-size'?: number;
      'cursor-last-id'?: string;
    },
  ): Promise<number> {
    return this.firstEventVersion(
      model,
      query,
      {
        version: 1,
      },
      0,
      headers,
    );
  }

  async maxEventsVersion(
    model: string,
    query: object,
    headers?: {
      page?: number;
      'page-size'?: number;
      'cursor-last-id'?: string;
    },
  ): Promise<number> {
    return this.firstEventVersion(
      model,
      query,
      {
        version: -1,
      },
      -1,
      headers,
    );
  }

  async version(
    model: string,
    correlationId: string,
    version: number | Date | string,
  ): Promise<AxiosResponse> {
    this._checkCorrelationIdExistence(correlationId);

    try {
      return await this.core.request({
        method: 'get',
        url: this.core.getPath(model, correlationId, version),
      });
    } catch (err: any) {
      if (err?.response?.status === 404) {
        const res = err.response as AxiosResponse;

        res.data = null;

        return res;
      }

      throw err;
    }
  }

  at(
    model: string,
    correlationId: string,
    date: Date | string,
  ): Promise<AxiosResponse> {
    return this.version(model, correlationId, date);
  }

  restore(
    model: string,
    correlationId: string,
    version: number,
  ): Promise<AxiosResponse> {
    this._checkCorrelationIdExistence(correlationId);

    return this.core.request({
      method: 'post',
      url: this.core.getPath(model, correlationId, version, 'restore'),
    });
  }

  snapshot(
    model: string,
    correlationId: string,
    options?: {
      /**
       * Version of the state to snapshot.
       */
      version?: number | Date | string;
      /**
       * If true, delete past events from the database.
       */
      clean?: boolean;
    },
  ): Promise<AxiosResponse> {
    this._checkCorrelationIdExistence(correlationId);

    return this.core.request({
      method: 'post',
      url: this.core.getPath(model, correlationId, 'snapshot'),
      params: options,
    });
  }

  data(
    model: string,
    correlationId: string,
    models?: string[],
  ): Promise<AxiosResponse> {
    this._checkCorrelationIdExistence(correlationId);

    return this.core.request({
      method: 'get',
      url: this.core.getPath(model, correlationId, 'data'),
      params: {
        models,
      },
    });
  }

  archive(
    model: string,
    correlationId: string,
    deep = false,
    models?: string[],
  ): Promise<AxiosResponse> {
    this._checkCorrelationIdExistence(correlationId);

    return this.core.request({
      method: 'post',
      url: this.core.getPath(model, correlationId, 'archive'),
      params: {
        deep,
        models,
      },
    });
  }

  unarchive(
    model: string,
    correlationId: string,
    deep = false,
    models?: string[],
  ): Promise<AxiosResponse> {
    this._checkCorrelationIdExistence(correlationId);

    return this.core.request({
      method: 'post',
      url: this.core.getPath(model, correlationId, 'unarchive'),
      params: {
        deep,
        models,
      },
    });
  }

  delete(
    model: string,
    correlationId: string,
    deep = false,
    models?: string[],
  ): Promise<AxiosResponse> {
    this._checkCorrelationIdExistence(correlationId);

    return this.core.request({
      method: 'delete',
      url: this.core.getPath(model, correlationId),
      params: {
        deep,
        models,
      },
    });
  }

  aggregate(pipeline: any[], headers?: any): Promise<AxiosResponse> {
    return this.core.request({
      method: 'post',
      url: this.core.getPath('aggregate'),
      headers,
      data: pipeline,
    });
  }

  _interpolate(str: string, params: any) {
    if (typeof str !== 'string') {
      return str;
    }

    const matches = str.matchAll(/\$\{([^}]+)\}/g);
    let res = str;

    for (const match of matches) {
      res = res.replace(match[0], _get(params, match[1]));
    }

    return res;
  }

  async import(
    data: DatastoreImportFixture[],
    modelConfigs: { [key: string]: ModelConfig },
    options: {
      dryRun: boolean;
    } = { dryRun: false },
    entities = new Map<string, object>(),
  ): Promise<Map<string, object>> {
    for (const {
      model,
      id,
      idempotency,
      omit_on_update = [],
      links = [],
      entity,
    } of data) {
      let _entity;

      let modifiedIdempotency = {
        ...idempotency,
      };

      let modifiedEntity = {
        ...entity,
      };

      for (const l of links) {
        let _link: DatastoreImportLink;
        if (l.id) {
          _link = entities.get(l.id) as DatastoreImportLink;
        } else {
          // Always the first one taken:
          const res = await this.find(l.model, {
            ...l.idempotency,
            _must_hash: true,
          });

          if (res.data.length === 0) {
            const err = new Error('[Link] Idempotency condition violation');
            // @ts-ignore
            err.details = {
              ...data,
              link: l,
              idempotency: l.idempotency,
              candidates: res.data,
            };
            throw err;
          }

          const { data: decrypted } = await this.decrypt(model, res.data);
          _link = decrypted[0];
        }

        const linkResult = mapValuesDeep(
          l.map,
          (v: keyof DatastoreImportLink, k: any) =>
            _link[v] ?? this._interpolate(v, _link),
        );

        modifiedEntity = mergeWith(
          {},
          modifiedEntity,
          linkResult,
          mergeWithArrays,
        );

        if (l.is_idempotency_condition === true) {
          modifiedIdempotency = mergeWith(
            {},
            modifiedIdempotency,
            linkResult,
            mergeWithArrays,
          );
        }
      }

      const { data: candidates } = await this.find(model, {
        ...modifiedIdempotency,
        _must_hash: true,
      });

      if (candidates.length > 1) {
        const err = new Error('Idempotency condition violation');
        // @ts-ignore
        err.details = {
          ...data,
          idempotency: modifiedIdempotency,
          candidates,
        };
        throw err;
      }

      _entity = modifiedEntity;
      if (candidates.length === 0) {
        if (options.dryRun === false) {
          const correlationId =
            modifiedIdempotency?.[modelConfigs?.[model]?.correlation_field];
          if (correlationId) {
            const res = await this.update(
              model,
              correlationId,
              modifiedEntity,
              {
                upsert: true,
              },
            );
            _entity = res.data;
          } else {
            const res = await this.create(model, modifiedEntity);
            _entity = res.data;
          }
        }
      } else {
        const candidate = omit(
          candidates[0],
          'created_at',
          'updated_at',
          'version',
          modelConfigs[model].correlation_field,
        );

        const {
          data: [decryptedCandidate],
        } = await this.decrypt(model, [candidate]);

        const payload = omit(
          {
            ..._entity,
            ...modifiedEntity,
          },
          'created_at',
          'updated_at',
          'version',
          ...omit_on_update,
        );

        const diff = jsonpatch.compare(
          omit(decryptedCandidate, ...omit_on_update),
          payload,
        );

        if (diff.length > 0 && options.dryRun === false) {
          const res = await this.update(
            model,
            candidates[0][modelConfigs[model].correlation_field],
            payload,
          );

          _entity = res.data;
        } else {
          _entity = candidates[0];
        }

        const rollback = jsonpatch.compare(
          omit(
            _entity,
            'created_at',
            'updated_at',
            'version',
            ...omit_on_update,
          ),
          omit(
            candidates[0],
            'created_at',
            'updated_at',
            'version',
            ...omit_on_update,
          ),
        );

        // Store the diff
        _entity.__update__ = diff;
        _entity.__rollback__ = rollback;
      }

      const { data: decryted } = await this.decrypt(model, [_entity]);
      _entity = decryted[0];

      entities.set(id, _entity);
    }

    return entities;
  }

  async walkNext(
    model: string,
    query: object,
    source: string,
    page: number,
    pageSize: number,
    opts: {
      current_version: number;
      version_ordered?: boolean;
      cursor_last_id?: string;
      cursor_last_correlation_id: string;
      headers?: any;
    },
  ) {
    const _query: { version?: number } = cloneDeep(query);
    const isVersionOrdered: boolean = opts?.version_ordered === true;

    if (isVersionOrdered === true) {
      _query.version = opts?.current_version;
    }

    const headers = {
      ...opts?.headers,
      'cursor-last-id': opts?.cursor_last_id,
      'cursor-last-correlation-id': opts?.cursor_last_correlation_id,
    };

    if (source === 'events') {
      return this.allEvents(model, _query, page, pageSize, headers);
    } else {
      return this.find(model, _query, page, pageSize, headers);
    }
  }

  static async walkMulti(
    datastores: Map<string, Datastore>,
    queries: MultiQuery[],
    handler: (res: any, query: MultiQuery, queryIteration: Iteration) => any,
    opts?: {
      page_size: number;
      sort_handler?: (a: any, b: any) => any;
      sleep?: number;
      version_ordered?: boolean;
      handle_in_order?: boolean;
      handle_in_parallel?: boolean;
      chunk_size?: number;
      is_mutating?: boolean;
    },
  ) {
    return walkMulti(
      datastores,
      queries,
      opts?.page_size,
      handler,
      opts,
      opts?.sort_handler,
    );
  }

  async walk(
    model: string,
    query: object,
    handler: (...args: any[]) => Promise<void> | void,
    pageSize = 10,
    source: MultiQuery['source'] = 'entities',
    headers?: any,
    opts?: {
      sleep?: number;
      version_ordered?: boolean;
      handle_in_order?: boolean;
      handle_in_parallel?: boolean;
      chunk_size?: number;
      is_mutating?: boolean;
    },
  ) {
    const effectivePageSize = Math.min(
      pageSize,
      this.config?.walk?.maxPageSize ?? Infinity,
    );
    return walkMulti(
      new Map([['datastore', this]]),
      [
        {
          datastore: 'datastore',
          model,
          query,
          source,
          headers,
        },
      ],
      effectivePageSize,
      handler,
      opts,
    );
  }

  async updateOverwhelmingly<T>(
    model: string,
    query: object,
    handler: (entity: T) => Promise<T>,
    progress: (
      stats: {
        total: number;
        done: number;
        error: number;
        progress: number;
        restored: number;
      },
      entity: T,
      headers: RawAxiosResponseHeaders | AxiosResponseHeaders,
    ) => void,
    pageSize?: number,
  ) {
    const {
      data: { [model]: modelConfig },
    } = await this.getModel(model);

    const correlationField = modelConfig.correlation_field;

    const total = await this.count(model, query);

    const stats = {
      total,
      done: 0,
      error: 0,
      progress: 0,
      restored: 0,
    };

    await this.walk(
      model,
      query,
      async (obj: any) => {
        try {
          const payload = await handler(obj);

          const { data, headers } = await this.update<T>(
            model,
            obj[correlationField],
            payload,
          );

          stats.done += 1;
          stats.progress = stats.done / stats.total;
          progress(stats, data, headers);
        } catch (err) {
          stats.error += 1;
          await this.restore(model, obj[correlationField], obj.version);
          stats.restored += 1;
        }
      },
      pageSize,
    );

    return stats;
  }

  /**
   * @deprecated in favor to `datastore.streams.getStreamId`
   */
  /* istanbul ignore next */
  _streamId(model: string, source: string, query?: object): string {
    return this.streams.getStreamId(model, source, query);
  }

  /**
   * @deprecated in favor to `datastore.streams.listen`
   */
  async listen(
    model: string,
    source: 'events' | 'entities',
    query?: object,
    options?: any,
  ): Promise<StreamClose> {
    return this.streams.listen(model, source, query, {
      ...options,
      forward: this,
    });
  }

  /**
   * @deprecated in favor to `datastore.streams.close`
   */
  close(streamId: string) {
    return this.streams.close(streamId);
  }

  /**
   * @deprecated in favor to `datastore.streams.closeAll`
   */
  closeAll() {
    return this.streams.closeAll();
  }

  /**
   * @deprecated in favor to `datastore.streams.stream`
   */
  /* istanbul ignore next */
  stream(
    handler: StreamHandler,
    model = 'all',
    source: 'entities' | 'events' = 'entities',
    data: object[] = [],
  ): Promise<StreamClose> {
    return this.streams.streamHTTP(handler, model, source, data);
  }
}