@getanthill/datastore/src/sdk/projections/index.integration.test.ts

Event-Sourced Datastore

import path from 'node:path';

import setup from '../../setup';

import { main } from '.';

import thingsModelConfig from '../../templates/examples/things.json';

import * as runner from '../runner';

import services from '../../services';

describe('sdk/projections', () => {
  let mongodbSource;
  let mongodbDestination;
  let serverSource;
  let source;
  let sourceInstance;

  let serverDestination;
  let destination;
  let destinationInstance;
  let destinationServices;
  let uuid;

  beforeAll(async () => {
    jest.setTimeout(30000);

    [, mongodbSource, , , serverSource, source, , sourceInstance] =
      await setup.startApi({
        mode: 'development',
        features: {
          api: {
            admin: true,
          },
        },
      });

    source.config.debug = false;

    [
      ,
      mongodbDestination,
      ,
      ,
      serverDestination,
      destination,
      destinationServices,
      destinationInstance,
    ] = await setup.startApi({
      mode: 'development',
      features: {
        api: {
          admin: true,
        },
      },
    });

    destination.config.debug = false;

    services.datastores = new Map([
      ['source', source],
      ['destination', destination],
    ]);

    await source.createModel({
      ...thingsModelConfig,
      is_enabled: true,
      db: 'datastore',
      name: 'projections',
      correlation_field: 'projection_id',
    });

    await source.createModel({
      is_enabled: true,
      db: 'datastore',
      name: 'accounts',
      correlation_field: 'account_id',
      schema: {
        model: {
          type: 'object',
          properties: {
            firstname: { type: 'string' },
          },
        },
      },
    });

    await source.createModel({
      is_enabled: true,
      db: 'datastore',
      name: 'profiles',
      correlation_field: 'profile_id',
      schema: {
        model: {
          type: 'object',
          properties: {
            account_id: { type: 'string' },
          },
        },
      },
    });
  });

  beforeEach(async () => {
    uuid = setup.uuid();

    jest
      .spyOn(process, 'exit')
      // @ts-ignore
      .mockImplementation(() => null);
  });

  afterAll(async () => {
    await setup.teardownDb(mongodbSource);
    await setup.teardownDb(mongodbDestination);

    await setup.stopApi(sourceInstance);
    await setup.stopApi(destinationInstance);
  });

  describe('#main', () => {
    beforeEach(async () => {
      await Promise.all([
        mongodbDestination
          .db('datastore_write')
          .collection('my_projections')
          .deleteMany({}),
        mongodbSource
          .db('datastore_write')
          .collection('accounts')
          .deleteMany({}),
        mongodbSource
          .db('datastore_write')
          .collection('profiles')
          .deleteMany({}),
        mongodbDestination
          .db('datastore_write')
          .collection('internal_models')
          .deleteMany({}),
      ]);

      destinationServices.models.reset();
      await destinationServices.models.reload();
    });

    afterEach(() => {
      jest.restoreAllMocks();
    });

    it('throws an error if the datastore instance does not respond to heartbeat', async () => {
      const ds = services.datastores.get('source')!;
      const heartbeatMock = jest
        .spyOn(ds, 'heartbeat')
        .mockImplementation(async () => {
          throw new Error('Stop here');
        });

      const projectionConfig = await main(
        new URL(
          `?configuration_path=` +
            path.resolve(__dirname, '__fixtures__/projection.json'),
          'ds://projections',
        ),
        { ...services, datastores: new Map([['source', ds]]) },
      ).catch(() => null);

      expect(heartbeatMock).toHaveBeenCalledTimes(1);
    });

    it('does not throw an error if the datastore instance does not respond to heartbeat but `heartbeat=false` config is set', async () => {
      const ds = services.datastores.get('source')!;
      const heartbeatMock = jest
        .spyOn(ds, 'heartbeat')
        .mockImplementation(async () => {
          throw new Error('Stop here');
        });

      const projectionConfig = await main(
        new URL(
          `?heartbeat=false&configuration_path=` +
            path.resolve(__dirname, '__fixtures__/projection.json'),
          'ds://projections',
        ),
        { ...services, datastores: new Map([['source', ds]]) },
      ).catch(() => null);

      expect(heartbeatMock).toHaveBeenCalledTimes(0);
    });

    it('skips the model update if no model is defined', async () => {
      const { data: account } = await source.create('accounts', {
        firstname: `alice:${uuid}`,
      });

      const { data: profile } = await source.create('profiles', {
        account_id: account.account_id,
      });

      await source.update('accounts', account.account_id, {
        firstname: `bernard:${uuid}`,
      });

      const { data: projection } = await source.create('projections', {
        name: `my_projections_${uuid}`,
        from: {
          datastore: 'source',
          model: 'accounts',
          source: 'entities',
        },
        trigger: {
          query: {
            firstname: `bernard:${uuid}`,
          },
        },
        pipeline: [
          {
            type: 'fetch',
            datastore: 'source',
            model: 'accounts',
            source: 'events',
            destination: 'account_events',
            map: [
              {
                from: 'account.account_id',
                to: 'account_id',
              },
            ],
          },
          {
            type: 'fetch',
            datastore: 'source',
            model: 'profiles',
            map: [
              {
                from: 'account.account_id',
                to: 'account_id',
              },
            ],
          },
          {
            type: 'persist',
            datastore: 'destination',
            model: 'my_projections',
            payload: {
              performed: true,
            },
          },
        ],
      });

      const projectionConfig = await main(
        new URL(
          `?projection_id=${projection.projection_id}`,
          'ds://projections',
        ),
        services,
      );

      // No model has been initialized:
      const { data: models } = await destination.getModels();
      expect(models).toEqual({});
    });

    it('does not throw an error in case of exception on the validation step', async () => {
      const { data: account } = await source.create('accounts', {
        firstname: `alice:${uuid}`,
      });

      const { data: profile } = await source.create('profiles', {
        account_id: account.account_id,
      });

      await source.update('accounts', account.account_id, {
        firstname: `bernard:${uuid}`,
      });

      const { data: projection } = await source.create('projections', {
        name: 'my_projections',
        model: {
          correlation_field: 'my_projection_id',
        },
        triggers: [
          {
            datastore: 'source',
            model: 'accounts',
            source: 'entities',
            query: {
              firstname: `bernard:${uuid}`,
            },
          },
          {
            datastore: 'unknown',
            model: 'accounts',
            source: 'entities',
            query: {
              firstname: `bernard:${uuid}`,
            },
          },
        ],
        pipeline: [
          {
            type: 'validate',
            path: 'entity',
            must_throw: true,
            schema: {
              type: 'string',
            },
          },
        ],
      });

      const projectionConfig = await main(
        new URL(
          `?heartbeat=false&progress=1&projection_id=${projection.projection_id}`,
          'ds://projections',
        ),
        services,
      );

      expect(projectionConfig).toMatchObject({
        triggers: [
          {
            datastore: 'source',
            model: 'accounts',
            source: 'entities',
            query: {
              firstname: `bernard:${uuid}`,
            },
          },
          {
            datastore: 'unknown',
            model: 'accounts',
            source: 'entities',
            query: {
              firstname: `bernard:${uuid}`,
            },
          },
        ],
      });

      await projectionConfig.start();

      const { stats } = await projectionConfig.handler(account, {});

      await projectionConfig.stop();

      expect(stats).toEqual({
        count: 1,
        total: 1,
        processed: 0,
        skipped: 1,
        failed: 0,
      });
    });

    it('does not throw an error in case of exception if an entity is not found on a fetch step', async () => {
      const { data: account } = await source.create('accounts', {
        firstname: `alice:${uuid}`,
      });

      const { data: profile } = await source.create('profiles', {
        account_id: account.account_id,
      });

      await source.update('accounts', account.account_id, {
        firstname: `bernard:${uuid}`,
      });

      const { data: projection } = await source.create('projections', {
        name: 'my_projections',
        model: {
          correlation_field: 'my_projection_id',
        },
        from: {
          datastore: 'source',
          model: 'accounts',
          source: 'entities',
        },
        trigger: {
          query: {
            firstname: `bernard:${uuid}`,
          },
        },
        pipeline: [
          {
            type: 'fetch',
            datastore: 'source',
            model: 'accounts',
            as_entity: true,
            query: {
              account_id: 'invalid',
            },
          },
        ],
      });

      const projectionConfig = await main(
        new URL(
          `?projection_id=${projection.projection_id}`,
          'ds://projections',
        ),
        services,
      );

      expect(projectionConfig).toMatchObject({
        triggers: [
          {
            datastore: 'source',
            model: 'accounts',
            query: {},
            source: 'entities',
          },
        ],
      });

      await projectionConfig.start();

      const { stats } = await projectionConfig.handler(account, {});

      await projectionConfig.stop();

      expect(stats).toEqual({
        count: 1,
        total: 1,
        processed: 0,
        skipped: 1,
        failed: 0,
      });
    });

    it('throws an exception in case of an internal exception', async () => {
      const { data: account } = await source.create('accounts', {
        firstname: `alice:${uuid}`,
      });

      const { data: profile } = await source.create('profiles', {
        account_id: account.account_id,
      });

      await source.update('accounts', account.account_id, {
        firstname: `bernard:${uuid}`,
      });

      const { data: projection } = await source.create('projections', {
        name: 'my_projections',
        model: {
          correlation_field: 'my_projection_id',
        },
        from: {
          datastore: 'source',
          model: 'accounts',
          source: 'entities',
        },
        trigger: {
          query: {
            firstname: `bernard:${uuid}`,
          },
        },
        pipeline: [
          {
            type: 'fetch',
            datastore: 'source',
            model: 'accounts',
            as_entity: true,
            query: {
              account_id: 'invalid',
            },
          },
        ],
      });

      const projectionConfig = await main(
        new URL(
          `?projection_id=${projection.projection_id}`,
          'ds://projections',
        ),
        services,
      );

      expect(projectionConfig).toMatchObject({
        triggers: [
          {
            datastore: 'source',
            model: 'accounts',
            query: {},
            source: 'entities',
          },
        ],
      });

      const { datastores: ds } = await projectionConfig.start();
      const _walk = ds.source.walk;
      jest.spyOn(ds.source, 'walk').mockImplementation(() => {
        throw new Error('Ooops');
      });

      let error;
      try {
        const { stats } = await projectionConfig.handler(account, {});
      } catch (err) {
        error = err;
      }

      await projectionConfig.stop();

      expect(error).toEqual(new Error('Ooops'));

      ds.source.walk = _walk;
    });

    it('performs the projection and persists the result', async () => {
      const { data: account } = await source.create('accounts', {
        firstname: `alice:${uuid}`,
      });

      const { data: profile } = await source.create('profiles', {
        account_id: account.account_id,
      });

      await source.update('accounts', account.account_id, {
        firstname: `bernard:${uuid}`,
      });

      const { data: projection } = await source.create('projections', {
        name: 'my_projections',
        model: {
          correlation_field: 'my_projection_id',
        },
        from: {
          datastore: 'source',
          model: 'accounts',
          source: 'entities',
        },
        trigger: {
          query: {
            firstname: `bernard:${uuid}`,
          },
        },
        pipeline: [
          {
            type: 'fetch',
            datastore: 'source',
            model: 'accounts',
            source: 'events',
            destination: 'account_events',
            map: [
              {
                from: 'account.account_id',
                to: 'account_id',
              },
            ],
          },
          {
            type: 'fetch',
            datastore: 'source',
            model: 'profiles',
            map: [
              {
                from: 'account.account_id',
                to: 'account_id',
              },
            ],
          },
          {
            type: 'persist',
            datastore: 'destination',
            model: 'my_projections',
            payload: {
              performed: true,
            },
          },
        ],
      });

      const projectionConfig = await main(
        new URL(
          `?init=true&projection_id=${projection.projection_id}`,
          'ds://projections',
        ),
        services,
      );

      expect(projectionConfig).toMatchObject({
        triggers: [
          {
            datastore: 'source',
            model: 'accounts',
            query: {},
            source: 'entities',
          },
        ],
      });

      await projectionConfig.start();

      await projectionConfig.handler(account, {});

      await projectionConfig.stop();

      const { data } = await destination.find('my_projections', {
        performed: true,
      });

      expect(data).toMatchObject([
        {
          version: 0,
          performed: true,
        },
      ]);
    });

    it('performs an incremental projection with runner invokation', async () => {
      const replay = runner.replay();

      const { data: account } = await source.create('accounts', {
        firstname: `alice:${uuid}`,
      });

      const { data: profile } = await source.create('profiles', {
        account_id: account.account_id,
      });

      await source.update('accounts', account.account_id, {
        firstname: `bernard:${uuid}`,
      });

      const { data: projection } = await source.create('projections', {
        name: 'my_projections',
        model: {
          correlation_field: 'my_projection_id',
          schema: {
            model: {
              type: 'object',
              additionalProperties: true,
              properties: {},
            },
          },
        },
        from: {
          datastore: 'source',
          model: 'accounts',
          source: 'entities',
        },
        pipeline: [
          {
            type: 'persist',
            datastore: 'destination',
            model: 'my_projections',
            correlation_field: 'my_projection_id',
            map: [
              {
                from: 'entity.account_id',
                to: 'my_projection_id',
              },
              {
                from: 'entity.firstname',
                to: 'firstname',
              },
            ],
            headers: {
              upsert: 'true',
            },
          },
        ],
      });

      const handlerUrl: string = `/projections?init=true&projection_id=${projection.projection_id}&is_incremental=true`;

      let projectionConfig = await main(
        new URL(handlerUrl, 'ds://projections'),
        services,
      );

      expect(projectionConfig).toMatchObject({
        triggers: [
          {
            datastore: 'source',
            model: 'accounts',
            query: {},
            source: 'entities',
          },
        ],
      });

      /**
       * First execution of the projection, the entity
       * must be created
       */
      await replay(
        [handlerUrl],
        {
          pageSize: 100,
          exitTimeout: 100,
          verbose: true,
          cwd: '',
        },
        {},
        {
          main: async () => projectionConfig,
        },
      );

      let res = await destination.find('my_projections', {});

      expect(res.data).toMatchObject([
        {
          version: 0,
          firstname: `bernard:${uuid}`,
        },
      ]);

      /**
       * Replaying the projection without any impact
       */
      projectionConfig = await main(
        new URL(handlerUrl, 'ds://projections'),
        services,
      );

      expect(projectionConfig).toMatchObject({
        triggers: [
          {
            datastore: 'source',
            model: 'accounts',
            query: {
              updated_at: {
                'date($gt)': `${res.data[0].updated_at}`,
              },
            },
            source: 'entities',
          },
        ],
      });

      await replay(
        [handlerUrl],
        {
          exitTimeout: 100,
          verbose: true,
          cwd: '',
        },
        {},
        {
          main: async () => projectionConfig,
        },
      );

      res = await destination.find('my_projections', {});

      expect(res.data).toMatchObject([
        {
          version: 0,
          firstname: `bernard:${uuid}`,
        },
      ]);

      /**
       * Replaying the projection with an entity updated
       * since last execution
       */
      await source.update('accounts', account.account_id, {
        firstname: 'Charles',
      });

      projectionConfig = await main(
        new URL(handlerUrl, 'ds://projections'),
        services,
      );

      expect(projectionConfig).toMatchObject({
        triggers: [
          {
            datastore: 'source',
            model: 'accounts',
            query: {
              updated_at: {
                'date($gt)': `${res.data[0].updated_at}`,
              },
            },
            source: 'entities',
          },
        ],
      });

      await replay(
        [handlerUrl],
        {
          exitTimeout: 100,
          verbose: true,
          cwd: '',
        },
        {},
        {
          main: async () => projectionConfig,
        },
      );

      res = await destination.find('my_projections', {});

      expect(res.data).toMatchObject([
        {
          version: 1,
          firstname: 'Charles',
        },
      ]);

      const countEvents = await destination.count(
        'my_projections',
        {},
        'events',
      );
      expect(countEvents).toEqual(3);
    });
  });
});