@getanthill/datastore/src/sdk/runner.integration.test.ts

Event-Sourced Datastore

import type { Datastore } from '../../dist/sdk';
import type App from '../App';

import setup from '../setup';
import { RunnerServices } from '../typings';

import * as runner from './runner';

describe('sdk/runner (integration)', () => {
  let mongodbSource;
  let source: Datastore;
  let sourceInstance: App;

  let uuid;

  const modelConfig = {
    is_enabled: true,
    db: 'datastore',
    name: 'accounts',
    correlation_field: 'account_id',
    schema: {
      model: {
        type: 'object',
        properties: {
          firstname: { type: 'string' },
          lastname: { type: 'string' },
          state: {
            type: 'string',
            enum: ['created', 'validating', 'validated', 'error'],
          },
        },
      },
    },
  };

  beforeAll(async () => {
    jest.setTimeout(30000);

    [, mongodbSource, , , , source, , sourceInstance] = await setup.startApi({
      mode: 'development',
      features: {
        api: {
          admin: true,
        },
        amqp: {
          isEnabled: true,
        },
      },
    });

    source.config.debug = false;
    source.streams.config.amqp = {
      ...sourceInstance.services.config.amqp,
      queue: {
        ...sourceInstance.services.config.amqp.queue,
        consumer: {
          ...sourceInstance.services.config.amqp.queue.consumer,
          name: 'worker',
        },
      },
    };

    source.streams.config.connector = 'amqp';

    sourceInstance.services.datastores = new Map([['source', source]]);

    await source.createModel(modelConfig);
    await source.createModelIndexes(modelConfig);
  });

  beforeEach(async () => {
    uuid = setup.uuid();

    jest
      .spyOn(process, 'exit')
      // @ts-ignore
      .mockImplementation(() => null);

    await Promise.all([
      mongodbSource.db('datastore_write').collection('accounts').deleteMany({}),
    ]);
  });

  afterEach(async () => {
    jest.restoreAllMocks();

    sourceInstance.services.datastores.source.streams.closeAll();
    source.streams.closeAll();
  });

  afterAll(async () => {
    sourceInstance.server.closeAllConnections();
    source.streams.closeAll();

    await setup.stopApi(sourceInstance);

    await setup.teardownDb(mongodbSource);
  });

  it('processes data multiple times if no `processing` configuration is provided (replay)', async () => {
    // Before
    const replay = runner.replay();

    const { data: account } = await source.create('accounts', {
      firstname: `alice:${uuid}`,
      state: 'created',
    });

    const handler = jest.fn();
    const runnerConfig = async (url: URL) => {
      return {
        datastore: 'source',
        model: 'accounts',
        query: {
          firstname: `alice:${uuid}`,
        },
        source: 'entities',
        raw: false,
        //processing: {...}, // <-- Missing
        start: () => {
          return sourceInstance.services;
        },
        handler,
      };
    };

    // When
    await Promise.all([
      replay(
        ['/handler#main'],
        {
          pageSize: 100,
          exitTimeout: 100,
          verbose: true,
          cwd: '',
          skipProcessBinding: true,
        },
        {},
        {
          main: runnerConfig,
        },
      ),
      replay(
        ['/handler#main'],
        {
          pageSize: 100,
          exitTimeout: 100,
          verbose: true,
          cwd: '',
          skipProcessBinding: true,
        },
        {},
        {
          main: runnerConfig,
        },
      ),
    ]);

    // Then
    expect(handler).toHaveBeenCalledTimes(2);
  });

  it('processes data multiple times if no `processing` configuration is provided (stream)', async () => {
    // Before
    const stream = runner.start();

    const handler = jest.fn();
    // When

    await stream(
      ['/handler?id=1#main'],
      {
        pageSize: 100,
        exitTimeout: 100,
        verbose: true,
        cwd: '',
        skipProcessBinding: true,
      },
      {},
      {
        main: async (url: URL) => {
          return {
            datastore: 'source',
            model: 'accounts',
            query: {
              firstname: `alice:${uuid}`,
              state: 'created',
            },
            source: 'entities',
            raw: false,
            //processing: {...}, // <-- Missing
            start: () => {
              return sourceInstance.services;
            },
            handler,
          };
        },
      },
    );
    await stream(
      ['/handler?id=2#main'],
      {
        pageSize: 100,
        exitTimeout: 100,
        verbose: true,
        cwd: '',
        skipProcessBinding: true,
      },
      {},
      {
        main: async (url: URL) => {
          return {
            datastore: 'source',
            model: 'accounts',
            query: {
              firstname: `alice:${uuid}`,
            },
            source: 'entities',
            raw: false,
            //processing: {...}, // <-- Missing
            start: () => {
              return sourceInstance.services;
            },
            handler,
          };
        },
      },
    );

    const { data: account } = await source.create('accounts', {
      firstname: `alice:${uuid}`,
      state: 'created',
    });

    await new Promise((resolve) => setTimeout(resolve, 500));

    // Then
    expect(handler).toHaveBeenCalledTimes(2);
  });

  it('skips the processing in case of concurrent execution (stream)', async () => {
    // Before
    const stream = runner.start();

    const handler = jest.fn();
    // When

    await stream(
      ['/handler?id=1#main'],
      {
        pageSize: 100,
        exitTimeout: 100,
        verbose: true,
        cwd: '',
        skipProcessBinding: true,
      },
      {},
      {
        main: async (url: URL) => {
          return {
            datastore: 'source',
            model: 'accounts',
            query: {
              firstname: `alice:${uuid}`,
              lastname: 'Doe',
            },
            source: 'entities',
            raw: false,
            processing: {
              correlation_field: 'account_id',
              field: 'state',
              states: ['created', 'validating'],
            },
            start: () => {
              return sourceInstance.services;
            },
            handler,
          };
        },
      },
    );

    await stream(
      ['/handler?id=2#main'],
      {
        pageSize: 100,
        exitTimeout: 100,
        verbose: true,
        cwd: '',
      },
      {},
      {
        main: async (url: URL) => {
          return {
            datastore: 'source',
            model: 'accounts',
            query: {
              firstname: `alice:${uuid}`,
            },
            source: 'entities',
            raw: false,
            processing: {
              correlation_field: 'account_id',
              field: 'state',
              states: ['created', 'validating'],
            },
            start: () => {
              return sourceInstance.services;
            },
            handler,
          };
        },
      },
    );

    const { data: account } = await source.create('accounts', {
      firstname: `alice:${uuid}`,
      lastname: 'Doe',
      state: 'created',
    });

    await new Promise((resolve) => setTimeout(resolve, 500));

    // Then
    expect(handler).toHaveBeenCalledTimes(1);
  });

  it('skips the processing in case of concurrent execution (replay)', async () => {
    // Before
    const replay = runner.replay();

    const { data: account } = await source.create('accounts', {
      firstname: `alice:${uuid}`,
      state: 'created',
    });

    const handler = jest.fn();
    const runnerConfig = async (url: URL) => {
      return {
        datastore: 'source',
        model: 'accounts',
        query: {
          firstname: `alice:${uuid}`,
        },
        source: 'entities',
        raw: false,
        processing: {
          correlation_field: 'account_id',
          field: 'state',
          states: ['created', 'validating'],
        },
        start: () => {
          return sourceInstance.services;
        },
        handler,
      };
    };

    // When
    await Promise.all([
      replay(
        ['/handler#main'],
        {
          pageSize: 100,
          exitTimeout: 100,
          verbose: true,
          cwd: '',
          skipProcessBinding: true,
        },
        {},
        {
          main: runnerConfig,
        },
      ),
      replay(
        ['/handler#main'],
        {
          pageSize: 100,
          exitTimeout: 100,
          verbose: true,
          cwd: '',
          skipProcessBinding: true,
        },
        {},
        {
          main: runnerConfig,
        },
      ),
    ]);

    // Then
    expect(handler).toHaveBeenCalledTimes(1);
  });

  it('skips reprocessing on replay', async () => {
    // Before
    const replay = runner.replay();

    const { data: account } = await source.create('accounts', {
      firstname: `alice:${uuid}`,
      state: 'created',
    });

    const handler = jest.fn();
    const runnerConfig = async (url: URL) => {
      return {
        datastore: 'source',
        model: 'accounts',
        query: {
          firstname: `alice:${uuid}`,
        },
        source: 'entities',
        raw: false,
        processing: {
          correlation_field: 'account_id',
          field: 'state',
          states: ['created', 'validating'],
        },
        start: () => {
          return sourceInstance.services;
        },
        handler,
      };
    };

    // When
    await replay(
      ['/handler#main'],
      {
        pageSize: 100,
        exitTimeout: 100,
        verbose: true,
        cwd: '',
        skipProcessBinding: true,
      },
      {},
      {
        main: runnerConfig,
      },
    );
    await replay(
      ['/handler#main'],
      {
        pageSize: 100,
        exitTimeout: 100,
        verbose: true,
        cwd: '',
      },
      {},
      {
        main: runnerConfig,
      },
    );

    // Then
    expect(handler).toHaveBeenCalledTimes(1);
  });

  it('sets the correct state on field on processing (replay)', async () => {
    // Before
    const replay = runner.replay();

    const { data: account } = await source.create('accounts', {
      firstname: `alice:${uuid}`,
      state: 'created',
    });

    const handler = jest.fn();
    const runnerConfig = async (url: URL) => {
      return {
        datastore: 'source',
        model: 'accounts',
        query: {
          firstname: `alice:${uuid}`,
        },
        source: 'entities',
        raw: false,
        processing: {
          correlation_field: 'account_id',
          field: 'state',
          states: ['created', 'validating', 'validated', 'error'],
        },
        start: () => {
          return sourceInstance.services;
        },
        handler,
      };
    };

    // When
    await replay(
      ['/handler#main'],
      {
        pageSize: 100,
        exitTimeout: 100,
        verbose: true,
        cwd: '',
        skipProcessBinding: true,
      },
      {},
      {
        main: runnerConfig,
      },
    );

    // Then
    const { data: events } = await source.events(
      'accounts',
      account.account_id,
    );

    expect(events.map((e) => ({ version: e.version, state: e.state }))).toEqual(
      [
        {
          version: 0,
          state: 'created',
        },
        {
          version: 1,
          state: 'validating',
        },
        {
          version: 2,
          state: 'validated',
        },
      ],
    );
  });

  it('sets the correct state on field on processing error (replay)', async () => {
    // Before
    const replay = runner.replay();

    const { data: account } = await source.create('accounts', {
      firstname: `alice:${uuid}`,
      state: 'created',
    });

    const handler = jest.fn().mockImplementation(() => {
      throw new Error('Ooops');
    });
    const runnerConfig = async (url: URL) => {
      return {
        datastore: 'source',
        model: 'accounts',
        query: {
          firstname: `alice:${uuid}`,
        },
        source: 'entities',
        raw: false,
        processing: {
          correlation_field: 'account_id',
          field: 'state',
          states: ['created', 'validating', 'validated', 'error'],
        },
        start: () => {
          return sourceInstance.services;
        },
        handler,
      };
    };

    // When
    await replay(
      ['/handler#main'],
      {
        pageSize: 100,
        exitTimeout: 100,
        verbose: true,
        cwd: '',
        skipProcessBinding: true,
      },
      {},
      {
        main: runnerConfig,
      },
    );

    // Then
    const { data: events } = await source.events(
      'accounts',
      account.account_id,
    );

    expect(events.map((e) => ({ version: e.version, state: e.state }))).toEqual(
      [
        {
          version: 0,
          state: 'created',
        },
        {
          version: 1,
          state: 'validating',
        },
        {
          version: 2,
          state: 'error',
        },
      ],
    );
  });

  it('sets the correct state on field on processing error (stream)', async () => {
    // Before
    const stream = runner.start();

    const handler = jest.fn().mockImplementation(() => {
      throw new Error('Ooops');
    });
    // When

    await stream(
      ['/handler?id=1#main'],
      {
        pageSize: 100,
        exitTimeout: 100,
        verbose: true,
        cwd: '',
        skipProcessBinding: true,
      },
      {},
      {
        main: async (url: URL) => {
          return {
            datastore: 'source',
            model: 'accounts',
            query: {
              firstname: `alice:${uuid}`,
              state: 'created',
            },
            source: 'entities',
            raw: false,
            processing: {
              correlation_field: 'account_id',
              field: 'state',
              states: ['created', 'validating', 'validated', 'error'],
            },
            start: () => {
              return sourceInstance.services;
            },
            handler,
          };
        },
      },
    );

    const { data: account } = await source.create('accounts', {
      firstname: `alice:${uuid}`,
      state: 'created',
    });

    await new Promise((resolve) => setTimeout(resolve, 500));

    // Then
    const { data: events } = await source.events(
      'accounts',
      account.account_id,
    );

    await new Promise((resolve) => setTimeout(resolve, 250));

    expect(events.map((e) => ({ version: e.version, state: e.state }))).toEqual(
      [
        {
          version: 0,
          state: 'created',
        },
        {
          version: 1,
          state: 'validating',
        },
        {
          version: 2,
          state: 'error',
        },
      ],
    );
  });

  describe('#shouldProcessData', () => {
    it('returns false if the entity does not have the correlation field', async () => {
      const status = await runner.shouldProcessData(
        // @ts-expect-error it is ok here
        sourceInstance.services,
        'source',
        'accounts',
        null,
        {
          correlation_field: 'account_id',
          field: 'state',
          states: ['created', 'validating', 'validated'],
        },
      );

      expect(status).toEqual(false);
    });

    it('returns false if the entity does not have the state field', async () => {
      const status = await runner.shouldProcessData(
        // @ts-expect-error it is ok here
        sourceInstance.services,
        'source',
        'accounts',
        {
          account_id: 'account_id',
        },
        {
          correlation_field: 'account_id',
          field: 'state',
          states: ['created', 'validating', 'validated'],
        },
      );

      expect(status).toEqual(false);
    });
  });

  describe('#isDataProcessed', () => {
    it('returns false if the entity does not have the correlation field', async () => {
      const status = await runner.isDataProcessed(
        // @ts-expect-error it is ok here
        sourceInstance.services,
        'source',
        'accounts',
        null,
        2,
        {
          correlation_field: 'account_id',
          field: 'state',
          states: ['created', 'validating', 'validated'],
        },
      );

      expect(status).toEqual(false);
    });

    it('returns false if the entity does not have the state field', async () => {
      const status = await runner.isDataProcessed(
        // @ts-expect-error it is ok here
        sourceInstance.services,
        'source',
        'accounts',
        {
          account_id: 'account_id',
        },
        2,
        {
          correlation_field: 'account_id',
          field: 'state',
          states: ['created', 'validating', 'validated'],
        },
      );

      expect(status).toEqual(false);
    });
  });
});