@getanthill/datastore/src/api/models/controllers.ts

Event-Sourced Datastore

import type { NextFunction, Request, Response } from 'express';
import type { AccessLevel, AnyObject, Services } from '../../typings';

import omit from 'lodash/omit';

import {
  controllerBuilder,
  mapFindQuery,
  decrypt as decryptData,
  getQueryFromCursorLastId,
  buildCursorLastId,
  getVersionFromDate,
} from '../utils';

import * as handlers from '../events/handlers';
import {
  getTokensByRole,
  getAuthorizationToken,
  isAuthorized,
} from '../middleware';

export function create(services: Services) {
  return controllerBuilder(
    services,
    services.metrics.incrementApiCreate,
    async (req: Request, res: Response, next: NextFunction) => {
      const entity = await handlers.created(services, req, req.body);

      // @ts-ignore
      res.body = entity.state;

      services.telemetry.logger.debug('[api/models#create] Entity created', {
        model: req.params.model,
        entity: entity.state,
      });

      next();
    },
  );
}

export function update(services: Services) {
  return controllerBuilder(
    services,
    services.metrics.incrementApiUpdate,
    async (req: Request, res: Response, next: NextFunction) => {
      const entity = await handlers.updated(services, req, req.body);

      // @ts-ignore
      res.body = entity.state;

      services.telemetry.logger.debug('[api/models#update] Entity updated', {
        model: req.params.model,
        entity: entity.state,
      });

      next();
    },
  );
}

export function patch(services: Services) {
  return controllerBuilder(
    services,
    services.metrics.incrementApiPatch,
    async (req: Request, res: Response, next: NextFunction) => {
      const entity = await handlers.patched(services, req, req.body);

      // @ts-ignore
      res.body = entity.state;

      next();
    },
  );
}

export function apply(services: Services) {
  return controllerBuilder(
    services,
    services.metrics.incrementApiApply,
    async (req: Request, res: Response, next: NextFunction) => {
      const entity = await handlers.applied(services, req, req.body);

      // @ts-ignore
      res.body = entity.state;

      next();
    },
  );
}

export function get(services: Services) {
  const decryptTokens = getTokensByRole(
    services.config.security.tokens,
    'decrypt',
  );

  return controllerBuilder(
    services,
    services.metrics.incrementApiGet,
    async (req: Request, res: Response, next: NextFunction) => {
      const mustDecrypt: boolean =
        req.header('decrypt') === 'true' &&
        !!isAuthorized(decryptTokens, getAuthorizationToken(req));
      const forcePrimary: boolean = req.header('force-primary') === 'true';

      const model = services.models.factory(
        req.params.model,
        req.params.correlation_id,
        {
          forcePrimary,
        },
      );

      await model.getState();

      if (model.state === null) {
        throw new Error('Not Found');
      }

      if (mustDecrypt === true) {
        // @ts-ignore
        res.body = await decryptData(
          services,
          res.locals as { id: string; level?: AccessLevel },
          req.params.model,
          model.state,
        );
      } else {
        // @ts-ignore
        res.body = model.state;
      }

      next();
    },
    {
      message: {
        'Unauthorized field processing': 403,
        'Event schema validation error': 422,
      },
    },
  );
}

export function timetravel(services: Services) {
  return controllerBuilder(
    services,
    services.metrics.incrementApiTimetravel,
    async (req: Request, res: Response, next: NextFunction) => {
      const withResponseValidation: boolean =
        req.header('with-response-validation') !== 'false';

      const Model = services.models.getModel(req.params.model);

      const model = services.models.factory(
        req.params.model,
        req.params.correlation_id,
      );

      const version = await getVersionFromDate(
        services,
        Model,
        req.params.correlation_id,
        req.params.version,
      );

      await model.getState();

      if (model.state === null) {
        throw new Error('Not Found');
      }

      const state = await model.getStateAtVersion(version!, false);

      // @ts-ignore
      res.body = state;

      if (withResponseValidation === false) {
        services.metrics.incrementApiTimetravel({
          state: '200',
          model: req.params.model,
        });

        return res.json(state);
      }

      next();
    },
    {
      message: {
        'Event schema validation error': 422,
      },
    },
  );
}

export function restore(services: Services) {
  return controllerBuilder(
    services,
    services.metrics.incrementApiRestore,
    async (req: Request, res: Response, next: NextFunction) => {
      /* @ts-ignore */
      const entity = await handlers.restored(services, req, req.body);

      // @ts-ignore
      res.body = entity.state;

      next();
    },
    {
      message: {
        'Event schema validation error': 422,
        'State version does not exist': 404,
        /* @ts-ignore */
        'Can not rollback a restoration event': async (
          _err,
          req: Request & { locals: any },
          res: Response,
        ) => {
          const { currentVersion, model } = req.locals;

          /* istanbul ignore next */
          if (currentVersion > 0 && model) {
            // Update the model state due to the reduction fail on the
            // restoration event:
            await model.getState();
            // Special use case of a restoration handled by admin failing due
            // to index unicity violation:
            await model.restore(currentVersion);
          }

          return 409;
        },
      },
    },
  );
}

export function find(services: Services) {
  const decryptTokens = getTokensByRole(
    services.config.security.tokens,
    'decrypt',
  );

  return controllerBuilder(
    services,
    services.metrics.incrementApiFind,
    async (req: Request, res: Response, next: NextFunction) => {
      const Model = services.models.getModel(req.params.model);
      const correlationField = Model.getModelConfig().correlation_field;

      // Map the query parameters to request services.MongoDb accordingly
      const { query: mappedQuery, options } = mapFindQuery(Model, req.query);
      let _mappedQuery = mappedQuery;

      // Retrieve pagination parameters:
      const page: number = Number.parseInt(req.header('page') ?? '0', 10);
      const pageSize: number = Number.parseInt(
        req.header('page-size') ?? '1000',
        10,
      );
      const cursorLastId: string = req.header('cursor-last-id') as string;
      const cursorLastCorrelationId: string = req.header(
        'cursor-last-correlation-id',
      ) as string;
      const withResponseValidation: boolean =
        req.header('with-response-validation') !== 'false';
      const mustDecrypt: boolean =
        req.header('decrypt') === 'true' &&
        !!isAuthorized(decryptTokens, getAuthorizationToken(req));
      const forcePrimary: boolean = req.header('force-primary') === 'true';

      services.telemetry.logger.debug('[api/models#find] Query', {
        model: req.params.model,
        query: req.query,
        mapped_query: mappedQuery,
        options,
        page: page,
        page_size: pageSize,
        headers: req.headers,
      });

      if (mappedQuery === null) {
        res.set({
          'correlation-field': correlationField,
          'cursor-last-id': cursorLastId,
          'cursor-last-correlation-id': cursorLastCorrelationId,
          'page-count': 0,
          page,
          'page-size': pageSize,
          count: 0,
        });

        // @ts-ignore
        res.body = [];

        services.metrics.incrementApiFind({
          state: '200',
          model: req.params.model,
        });

        return next();
      }

      if (cursorLastId) {
        _mappedQuery = {
          $and: [
            _mappedQuery,
            getQueryFromCursorLastId(
              Model,
              options,
              cursorLastId,
              cursorLastCorrelationId,
            ),
          ],
        };
      }

      /**
       * @fixme warning on unindexed queries.
       */
      const [cursor, promiseCount] = [
        Model.find(services.mongodb, _mappedQuery, {
          ...options,
          forcePrimary,
        }),
        pageSize !== 0 ? -1 : Model.count(services.mongodb, mappedQuery),
      ];

      if (!cursorLastId) {
        cursor.skip(pageSize * page);
      }

      cursor.batchSize(pageSize * 2);

      let entities = [];

      let i = 0;
      let lastReturnedFound = false;
      // i < pageSize useful if pageSize === 0;
      while (i < pageSize && (await cursor.hasNext())) {
        const entity = await cursor.next();

        if (cursorLastCorrelationId && lastReturnedFound === false) {
          lastReturnedFound =
            entity[correlationField] === cursorLastCorrelationId;

          continue;
        }

        entities.push(omit(entity, '_id'));
        i++;

        if (i >= pageSize) {
          break;
        }
      }

      /**
       * @warn This cursor closing is critical to
       * avoid possible memory leaks on some collections.
       */
      await cursor.close();

      /**
       * Adding the correlation field to the response headers
       * @todo need to apply this to every request and in the API documentation
       */
      const lastEntity = entities[entities.length - 1];
      res.set({
        'correlation-field': correlationField,
        'cursor-last-id': buildCursorLastId(lastEntity, options),
        'cursor-last-correlation-id': lastEntity?.[correlationField] ?? '',
      });

      // Pagination headers if not requested with cursor last ID:
      const count = await promiseCount;
      if (count !== -1) {
        res.set({
          'page-count': Math.ceil(count / pageSize),
          page,
          'page-size': pageSize,
          count,
        });
      }

      if (mustDecrypt === true) {
        entities = await Promise.all(
          entities.map((data) =>
            decryptData(
              services,
              res.locals as { id: string; level?: AccessLevel },
              req.params.model,
              data,
            ),
          ),
        );
      }

      // @ts-ignore
      res.body = entities;

      services.telemetry.logger.debug('[api/models#find] Response', {
        model: req.params.model,
        count,
      });

      if (withResponseValidation === false) {
        services.metrics.incrementApiFind({
          state: '200',
          model: req.params.model,
        });

        return res.json(entities);
      }

      next();
    },
    {
      message: {
        'Unauthorized field processing': 403,
        'Event schema validation error': 422,
      },
    },
  );
}

export function getEvents(services: Services) {
  return controllerBuilder(
    services,
    services.metrics.incrementApiEvents,
    async (req: Request, res: Response, _next: NextFunction) => {
      const Model = services.models.getModel(req.params.model);
      const correlationField = Model.getModelConfig().correlation_field;

      // Retrieve pagination parameters:
      const page: number = Number.parseInt(req.header('page') ?? '0', 10);
      const pageSize: number = Number.parseInt(
        req.header('page-size') ?? '1000',
        10,
      );
      const cursorLastId: string = req.header('cursor-last-id') as string;
      const cursorLastCorrelationId: string = req.header(
        'cursor-last-correlation-id',
      ) as string;

      let cursor;
      let count;
      const correlationId = (req.params.correlation_id ||
        req.query.correlation_id) as string;
      const eventsCollection = Model.getEventsCollection(
        Model.db(services.mongodb),
      );

      let _mappedQuery;
      let _options;
      if (correlationId) {
        const model = services.models.factory(req.params.model, correlationId);

        await model.getState();

        if (model.state === null) {
          throw new Error('Not Found');
        }

        // @ts-ignore - Warning about the req.query.version!!! 0 == false...
        const version = Number.parseInt(req.query.version || '-1', 10);
        const query: any = {
          [Model.getCorrelationField()]: correlationId,
          version: {
            $gt: version,
          },
        };
        _options = { sort: { version: 1 } };

        _mappedQuery = query;

        if (cursorLastId) {
          _mappedQuery = {
            $and: [
              _mappedQuery,
              getQueryFromCursorLastId(
                Model,
                _options,
                cursorLastId,
                cursorLastCorrelationId,
              ),
            ],
          };
        }

        services.telemetry.logger.debug('[api/models#getEvents] Query', {
          model: req.params.model,
          query: req.query,
          mapped_query: query,
          page: page,
          page_size: pageSize,
          headers: req.headers,
        });

        cursor = eventsCollection.find(_mappedQuery).sort(_options.sort);
        count = pageSize !== 0 ? -1 : await eventsCollection.count(query);
      } else {
        // Map the query parameters to request services.MongoDb accordingly
        const { query: mappedQuery, options } = mapFindQuery(Model, req.query);
        _mappedQuery = mappedQuery;
        _options = options;

        if (mappedQuery === null) {
          res.set({
            'correlation-field': correlationField,
            'cursor-last-id': '',
            'cursor-last-correlation-id': '',
            'page-count': 0,
            page: 0,
            'page-size': pageSize,
            count: 0,
          });

          // @ts-ignore
          res.body = [];

          // @ts-ignore
          return res.json(res.body);
        }

        if (cursorLastId) {
          _mappedQuery = {
            $and: [
              _mappedQuery,
              getQueryFromCursorLastId(
                Model,
                _options,
                cursorLastId,
                cursorLastCorrelationId,
              ),
            ],
          };
        }

        services.telemetry.logger.debug('[api/models#getEvents] Query', {
          model: req.params.model,
          query: req.query,
          mapped_query: mappedQuery,
          page: page,
          page_size: pageSize,
          headers: req.headers,
        });

        cursor = eventsCollection.find(_mappedQuery, options);
        count = pageSize !== 0 ? -1 : await eventsCollection.count(mappedQuery);
      }

      if (!cursorLastId) {
        cursor.skip(pageSize * page);
      }

      cursor.batchSize(pageSize * 2);

      const events = [];

      let i = 0;
      let lastReturnedFound = false;
      while (i < pageSize && (await cursor.hasNext())) {
        const entity = await cursor.next();

        if (cursorLastCorrelationId && lastReturnedFound === false) {
          lastReturnedFound = true;
          lastReturnedFound =
            `${entity[correlationField]}:${entity.version}` ===
            cursorLastCorrelationId;

          continue;
        }

        events.push(omit(entity, '_id', 'updated_at'));
        i++;
        if (i >= pageSize) {
          break;
        }
      }

      /**
       * @warn This cursor closing is critical to
       * avoid possible memory leaks on some collections.
       */
      await cursor.close();

      /**
       * Adding the correlation field to the response headers
       * @todo need to apply this to every request and in the API documentation
       */
      const lastEvent = events[events.length - 1];
      res.set({
        'correlation-field': correlationField,
        'cursor-last-id': buildCursorLastId(lastEvent, _options),
        'cursor-last-correlation-id': lastEvent
          ? `${lastEvent?.[correlationField]}:${lastEvent?.version}`
          : '',
      });

      // Pagination headers if not requested with cursor last ID:
      if (count !== -1) {
        res.set({
          'page-count': Math.ceil(count / pageSize),
          page,
          'page-size': pageSize,
          count,
        });
      }

      // @ts-ignore
      res.body = events;

      /**
       * @fixme possible route clash on OpenAPI
       * response validation
       */
      return res.json(events);
    },
    {
      message: {
        'Event schema validation error': 422,
      },
    },
  );
}

export function createSnapshot(services: Services) {
  return controllerBuilder(
    services,
    services.metrics.incrementApiSnapshot,
    async (req: Request, res: Response, next: NextFunction) => {
      const Model = services.models.getModel(req.params.model);
      const version = await getVersionFromDate(
        services,
        Model,
        req.params.correlation_id,
        req.query.version as string | number | undefined,
      );
      const removePastEvents = Boolean(req.query.clean);

      const snapshot = await Model.snapshot(
        services.mongodb,
        req.params.correlation_id,
        {
          version,
          removePastEvents,
        },
      );

      // @ts-ignore
      res.body = snapshot;

      next();
    },
    {
      message: {
        'Snapshot state is invalid': 422,
        'Event schema validation error': 422,
      },
    },
  );
}

export function encrypt(services: Services) {
  return controllerBuilder(
    services,
    services.metrics.incrementApiEncrypt,
    async (req: Request, res: Response, next: NextFunction) => {
      const Model = services.models.getModel(req.params.model);

      const q =
        'q' in req.query ? JSON.parse(req.query.q as string) : req.query;

      // @ts-ignore
      res.body = req.body.map((data) => Model.encrypt(data, q.fields));

      next();
    },
  );
}

export function decrypt(services: Services) {
  return controllerBuilder(
    services,
    services.metrics.incrementApiDecrypt,
    async (req: Request, res: Response, next: NextFunction) => {
      const q =
        'q' in req.query ? JSON.parse(req.query.q as string) : req.query;

      // @ts-ignore
      res.body = await Promise.all(
        req.body.map((data: AnyObject) =>
          decryptData(
            services,
            res.locals as { id: string; level?: AccessLevel },
            req.params.model,
            data,
            q.fields as string[],
          ),
        ),
      );

      next();
    },
    {
      message: {
        'Unauthorized field processing': 403,
        'Unsupported state or unable to authenticate data': 500,
      },
    },
  );
}

export const graph =
  (meterName: keyof Services['metrics'], handler: (...args: any[]) => any) =>
  (services: Services) => {
    const meter = services.metrics[meterName]! as (labels: {
      state: string;
      model: string;
    }) => void;

    return controllerBuilder(
      services,
      meter,
      async (req: Request, res: Response, next: NextFunction) => {
        meter({
          state: 'request',
          model: req.params.model,
        });

        const Model = services.models.getModel(req.params.model);
        const q =
          'q' in req.query ? JSON.parse(req.query.q as string) : req.query;

        const entities = await services.models.getEntitiesFromGraph(
          req.params.model,
          {
            [Model.getCorrelationField()]: req.params.correlation_id,
          },
          {
            graph: services.models.getGraph(),
            models: `${q.deep}` === 'true' ? undefined : [req.params.model],
            handler: async (services: Services, Model: any, entity: any) => {
              if (
                !q.models ||
                (q.models as string[]).includes(Model.getModelConfig().name)
              ) {
                return handler(services, Model, entity);
              }

              return entity;
            },
          },
        );

        meter({
          state: '200',
          model: req.params.model,
        });

        return res.json(Array.from(entities.values()));
      },
      {
        message: {
          'Entity archived too recently': 422,
        },
      },
    );
  };

export const archive = graph(
  'incrementApiArchive',
  async (services: Services, Model: any, entity: any, req: any) => {
    const e = new Model(services, entity[Model.getCorrelationField()]);

    await e.archive();

    return e.state;
  },
);

export const unarchive = graph(
  'incrementApiUnarchive',
  async (services: Services, Model: any, entity: any) => {
    const e = new Model(services, entity[Model.getCorrelationField()]);

    await e.unarchive();

    return e.state;
  },
);

export const deleteEntity = graph(
  'incrementApiDelete',
  async (services: Services, Model: any, entity: any) => {
    const e = new Model(services, entity[Model.getCorrelationField()]);

    await e.delete();

    return e.state;
  },
);

export function getGraphData(services: Services) {
  return controllerBuilder(
    services,
    services.metrics.incrementApiAdminGetGraph,
    async (req: Request, res: Response, next: NextFunction) => {
      services.metrics.incrementApiAdminGetGraph({
        state: 'request',
        model: req.params.model,
      });

      const Model = services.models.getModel(req.params.model);
      const q =
        'q' in req.query ? JSON.parse(req.query.q as string) : req.query;
      const models: string[] =
        (q.models as string[]) || Array.from(services.models.MODELS.keys());

      const entities: any[] = [];
      await services.models.getEntitiesFromGraph(
        req.params.model,
        {
          [Model.getCorrelationField()]: req.params.correlation_id,
        },
        {
          graph: services.models.getGraph(),
          handler: (services: Services, Model: any, entity: any) => {
            entities.push({
              model: Model.getModelConfig().name,
              entity,
            });

            return entity;
          },
        },
      );

      services.metrics.incrementApiAdminGetGraph({
        state: '200',
        model: req.params.model,
      });

      return res.json(entities.filter((e) => models.includes(e.model)));
    },
  );
}