@getanthill/datastore/src/sdk/projections/index.test.ts

Event-Sourced Datastore

import path from 'node:path';

import Datastore from '../Datastore';
import Aggregator from '../aggregator/Aggregator';

import * as projections from '.';

import thingsModelConfig from '../../templates/examples/things.json';

jest.mock('../Datastore');

describe('sdk/projections', () => {
  const DateNow = Date.now;
  const DEFAULT_DATASTORE_CONFIGS = process.env.DATASTORE_CONFIGS;

  let source;
  let destination;
  let datastores;
  let aggregator;
  beforeEach(() => {
    Date.now = jest
      .fn()
      .mockImplementation(() => new Date('2021-01-01').getTime());

    source = new Datastore();
    source.axios = {
      request: jest.fn(),
    };
    destination = new Datastore();
    destination.axios = {
      request: jest.fn(),
    };

    // @ts-ignore
    Datastore.mockImplementation((config) => ({
      config,
      heartbeat: jest.fn(),
    }));

    datastores = new Map([
      ['source', source],
      ['destination', destination],
    ]);
    aggregator = new Aggregator(datastores);
  });

  afterEach(() => {
    process.env.DATASTORE_CONFIGS = DEFAULT_DATASTORE_CONFIGS;
    Date.now = DateNow;

    jest.resetAllMocks();
  });

  describe('#getProjectionConfiguration', () => {
    it('throws an error if no `projection_id` is defined to fetch the configuration', async () => {
      let error;
      try {
        const configuration = await projections.getProjectionConfiguration(
          new URL('', 'ds://projections'),
          datastores,
        );
      } catch (err) {
        error = err;
      }

      expect(error.message).toEqual('Projection configuration not found');
    });

    it('throws an error if the configuration is invalid', async () => {
      source.get = jest.fn().mockImplementation(() => ({
        data: {
          name: 'projection',
          triggers: 'invalid',
        },
      }));

      let error;
      try {
        const configuration = await projections.getProjectionConfiguration(
          new URL('/?projection_id=projection_id', 'ds://projections'),
          datastores,
        );
      } catch (err) {
        error = err;
      }

      expect(error.message).toEqual('Validation failed');
    });

    it('returns the required configuration', async () => {
      source.get = jest.fn().mockImplementation(() => ({
        data: {
          name: 'projection',
        },
      }));

      const configuration = await projections.getProjectionConfiguration(
        new URL('/?projection_id=projection_id', 'ds://projections'),
        datastores,
      );

      expect(source.get).toHaveBeenCalledWith('projections', 'projection_id');
      expect(configuration).toEqual({
        name: 'projection',
      });
    });

    it('returns the required configuration based on a specific key search', async () => {
      source.find = jest.fn().mockImplementation(() => ({
        data: [
          {
            name: 'projection',
          },
        ],
      }));

      const configuration = await projections.getProjectionConfiguration(
        new URL(
          '/?projection_field=name&projection_id=projection_id',
          'ds://projections',
        ),
        datastores,
      );

      expect(source.find).toHaveBeenCalledWith('projections', {
        name: 'projection_id',
      });
      expect(configuration).toEqual({
        name: 'projection',
      });
    });

    it('returns the required configuration from another projection source', async () => {
      destination.get = jest.fn().mockImplementation(() => ({
        data: {
          name: 'projection',
        },
      }));

      const configuration = await projections.getProjectionConfiguration(
        new URL(
          '/?source=destination&projection_id=projection_id',
          'ds://projections',
        ),
        datastores,
      );

      expect(source.get).not.toHaveBeenCalled();
      expect(destination.get).toHaveBeenCalledWith(
        'projections',
        'projection_id',
      );
      expect(configuration).toEqual({
        name: 'projection',
      });
    });

    it('returns the required configuration from a sub path if required', async () => {
      source.get = jest.fn().mockImplementation(() => ({
        data: {
          value: {
            name: 'projection',
          },
        },
      }));

      const configuration = await projections.getProjectionConfiguration(
        new URL('/?projection_id=projection_id&path=value', 'ds://projections'),
        datastores,
      );

      expect(source.get).toHaveBeenCalledWith('projections', 'projection_id');
      expect(configuration).toEqual({
        name: 'projection',
      });
    });

    it('returns the required configuration from a specific filesystem path', async () => {
      source.get = jest.fn().mockImplementation(() => ({
        data: {
          value: {
            name: 'projection',
          },
        },
      }));

      const configuration = await projections.getProjectionConfiguration(
        new URL(
          '/?configuration_path=' +
            path.resolve(__dirname, '__fixtures__/projection.json'),
          'ds://projections',
        ),
        datastores,
      );

      expect(configuration).toMatchObject({
        name: 'fixture_projection',
      });
    });
  });

  describe('#getDestinationDatastore', () => {
    it('returns the first datastore if none is defined', () => {
      const destination = projections.getDestinationDatastore(
        new URL('', 'ds://projections'),
        datastores,
      );

      expect(destination).toEqual(datastores.get('destination'));
    });

    it('returns the datastore defined in `destination`', () => {
      const destination = projections.getDestinationDatastore(
        new URL('destination=destination', 'ds://projections'),
        datastores,
      );

      expect(destination).toEqual(datastores.get('destination'));
    });
  });

  describe('#initDestinationModel', () => {
    it('throws an error if the configuration does not have a name', async () => {
      let error;

      try {
        await projections.initDestinationModel(
          new URL(
            '/?projection_id=projection_id&path=value',
            'ds://projections',
          ),
          datastores,
          {
            model: {
              correlation_field: 'my_projection_id',
            },
          },
        );
      } catch (err) {
        error = err;
      }

      expect(error.message).toEqual('Missing Datastores configuration name');
    });

    it('throws an error if the model configuration does not have a correlation_field', async () => {
      let error;

      try {
        await projections.initDestinationModel(
          new URL(
            '/?projection_id=projection_id&path=value',
            'ds://projections',
          ),
          datastores,
          {
            name: 'my_projections',
            model: {
              correlation_field: null,
            },
          },
        );
      } catch (err) {
        error = err;
      }

      expect(error.message).toEqual('Missing projection correlation_field');
    });

    it('initializes the destination model', async () => {
      destination.createModel = jest.fn();

      await projections.initDestinationModel(
        new URL('/?projection_id=projection_id&path=value', 'ds://projections'),
        datastores,
        {
          name: 'my_projections',
          model: {
            correlation_field: 'my_projection_id',
          },
        },
      );

      expect(destination.createModel).toHaveBeenCalledWith({
        ...thingsModelConfig,
        name: 'my_projections',
        correlation_field: 'my_projection_id',
      });
    });

    it('updates the destination model', async () => {
      destination.createModel = jest.fn().mockImplementation(() => {
        const err = new Error('Conflict');
        err.response = {
          status: 409,
        };

        throw err;
      });

      await projections.initDestinationModel(
        new URL('/?projection_id=projection_id&path=value', 'ds://projections'),
        datastores,
        {
          name: 'my_projections',
          model: {
            correlation_field: 'my_projection_id',
          },
        },
      );

      expect(destination.createModel).toHaveBeenCalledWith({
        ...thingsModelConfig,
        name: 'my_projections',
        correlation_field: 'my_projection_id',
      });

      expect(destination.updateModel).toHaveBeenCalledWith({
        ...thingsModelConfig,
        name: 'my_projections',
        correlation_field: 'my_projection_id',
      });
    });

    it('throws an error in case of model creation error not Conflict', async () => {
      destination.createModel = jest.fn().mockImplementation(() => {
        throw new Error('Ooops');
      });

      let error;
      try {
        await projections.initDestinationModel(
          new URL(
            '/?projection_id=projection_id&path=value',
            'ds://projections',
          ),
          datastores,
          {
            name: 'my_projections',
            model: {
              correlation_field: 'my_projection_id',
            },
          },
        );
      } catch (err) {
        error = err;
      }

      expect(error).toEqual(new Error('Ooops'));
    });

    it('updates the destination model indexes', async () => {
      destination.createModel = jest.fn();

      await projections.initDestinationModel(
        new URL('/?projection_id=projection_id&path=value', 'ds://projections'),
        datastores,
        {
          name: 'my_projections',
          model: {
            correlation_field: 'my_projection_id',
          },
        },
      );

      expect(destination.createModelIndexes).toHaveBeenCalledWith({
        ...thingsModelConfig,
        name: 'my_projections',
        correlation_field: 'my_projection_id',
      });
    });
  });

  describe('#getTriggers', () => {
    it('returns the most simple trigger from the aggregation map step', async () => {
      const triggers = await projections.getTriggers(
        new URL('/', 'ds://projections'),
        datastores,
        aggregator,
        {
          name: 'my_projections',
          trigger: {
            datastore: 'source',
            model: 'users',
            source: 'events',
          },
        },
      );

      expect(triggers).toEqual([
        {
          datastore: 'source',
          model: 'users',
          source: 'events',
          query: {},
        },
      ]);
    });

    it('returns multiple triggers for a given projection', async () => {
      const triggers = await projections.getTriggers(
        new URL('/', 'ds://projections'),
        datastores,
        aggregator,
        {
          name: 'my_projections',
          triggers: [
            {
              datastore: 'source',
              model: 'users',
              source: 'events',
            },
            {
              datastore: 'source',
              model: 'metrics',
              source: 'events',
            },
          ],
        },
      );

      expect(triggers).toEqual([
        {
          datastore: 'source',
          model: 'users',
          source: 'events',
          query: {},
        },
        {
          datastore: 'source',
          model: 'metrics',
          source: 'events',
          query: {},
        },
      ]);
    });

    it('returns the trigger from the aggregation map step', async () => {
      const triggers = await projections.getTriggers(
        new URL('/', 'ds://projections'),
        datastores,
        aggregator,
        {
          name: 'my_projections',
          trigger: {
            datastore: 'source',
            model: 'users',
            source: 'events',

            query: {
              is_enabled: true,
            },
          },
        },
      );

      expect(triggers).toEqual([
        {
          datastore: 'source',
          model: 'users',
          source: 'events',
          query: {
            is_enabled: true,
          },
        },
      ]);
    });

    it('returns the incremental trigger version', async () => {
      destination.find = jest.fn().mockImplementation(() => ({
        data: [
          {
            updated_at: '2021-01-02T00:11:22.333Z',
          },
        ],
      }));

      const triggers = await projections.getTriggers(
        new URL('/?is_incremental=true', 'ds://projections'),
        datastores,
        aggregator,
        {
          name: 'my_projections',
          trigger: {
            datastore: 'source',
            model: 'users',
            source: 'events',
            query: {
              is_enabled: true,
            },
          },
        },
      );

      expect(destination.find).toHaveBeenCalledWith(
        'my_projections',
        { _fields: { updated_at: 1 }, _sort: { updated_at: -1 } },
        0,
        1,
      );

      expect(triggers).toEqual([
        {
          datastore: 'source',
          model: 'users',
          source: 'events',
          query: {
            is_enabled: true,
            updated_at: {
              'date($gt)': '2021-01-02T00:11:22.333Z',
            },
          },
        },
      ]);
    });

    it('returns the incremental multi triggers version', async () => {
      destination.find = jest.fn().mockImplementation(() => ({
        data: [
          {
            updated_at: '2021-01-02T00:11:22.333Z',
          },
        ],
      }));

      const triggers = await projections.getTriggers(
        new URL('/?is_incremental=true', 'ds://projections'),
        datastores,
        aggregator,
        {
          name: 'my_projections',
          triggers: [
            {
              datastore: 'source',
              model: 'users',
              source: 'events',
              query: {
                is_enabled: true,
              },
            },
            {
              datastore: 'source',
              model: 'metrics',
              source: 'events',
            },
          ],
        },
      );

      expect(destination.find).toHaveBeenCalledWith(
        'my_projections',
        { _fields: { updated_at: 1 }, _sort: { updated_at: -1 } },
        0,
        1,
      );

      expect(triggers).toEqual([
        {
          datastore: 'source',
          model: 'users',
          source: 'events',
          query: {
            is_enabled: true,
            updated_at: {
              'date($gt)': '2021-01-02T00:11:22.333Z',
            },
          },
        },
        {
          datastore: 'source',
          model: 'metrics',
          source: 'events',
          query: {
            updated_at: {
              'date($gt)': '2021-01-02T00:11:22.333Z',
            },
          },
        },
      ]);
    });

    it('does not change the trigger if no document has been found', async () => {
      destination.find = jest.fn().mockImplementation(() => ({
        data: [],
      }));

      const triggers = await projections.getTriggers(
        new URL('/?is_incremental=true', 'ds://projections'),
        datastores,
        aggregator,
        {
          name: 'my_projections',
          trigger: {
            datastore: 'source',
            model: 'users',
            source: 'events',
            query: {
              is_enabled: true,
            },
          },
        },
      );

      expect(destination.find).toHaveBeenCalledWith(
        'my_projections',
        { _fields: { updated_at: 1 }, _sort: { updated_at: -1 } },
        0,
        1,
      );

      expect(triggers).toEqual([
        {
          datastore: 'source',
          model: 'users',
          source: 'events',
          query: { is_enabled: true },
        },
      ]);
    });

    it('returns the incremental trigger version on a specific incremental date', async () => {
      destination.find = jest.fn().mockImplementation(() => ({
        data: [
          {
            created_at: '2021-01-02T00:11:22.333Z',
          },
        ],
      }));

      const triggers = await projections.getTriggers(
        new URL(
          '/?is_incremental=true&incremental_field=created_at',
          'ds://projections',
        ),
        datastores,
        aggregator,
        {
          name: 'my_projections',
          trigger: {
            datastore: 'source',
            model: 'users',
            source: 'events',
            query: {
              is_enabled: true,
            },
          },
        },
      );

      expect(destination.find).toHaveBeenCalledWith(
        'my_projections',
        { _fields: { created_at: 1 }, _sort: { created_at: -1 } },
        0,
        1,
      );

      expect(triggers).toEqual([
        {
          datastore: 'source',
          model: 'users',
          source: 'events',
          query: {
            is_enabled: true,
            created_at: {
              'date($gt)': '2021-01-02T00:11:22.333Z',
            },
          },
        },
      ]);
    });

    it('returns the incremental trigger version on a specific incremental source date', async () => {
      destination.find = jest.fn().mockImplementation(() => ({
        data: [
          {
            updated_at: '2021-01-02T00:11:22.333Z',
          },
        ],
      }));

      const triggers = await projections.getTriggers(
        new URL(
          '/?is_incremental=true&incremental_field=updated_at&incremental_source_field=created_at',
          'ds://projections',
        ),
        datastores,
        aggregator,
        {
          name: 'my_projections',
          trigger: {
            datastore: 'source',
            model: 'users',
            source: 'events',
            query: {
              is_enabled: true,
            },
          },
        },
      );

      expect(destination.find).toHaveBeenCalledWith(
        'my_projections',
        { _fields: { updated_at: 1 }, _sort: { updated_at: -1 } },
        0,
        1,
      );

      expect(triggers).toEqual([
        {
          datastore: 'source',
          model: 'users',
          source: 'events',
          query: {
            is_enabled: true,
            created_at: {
              'date($gt)': '2021-01-02T00:11:22.333Z',
            },
          },
        },
      ]);
    });
  });
});