@getanthill/datastore/src/sdk/projections/validator.ts

Event-Sourced Datastore

import type { Ctx } from '../../typings';

import path from 'path';

import * as telemetry from '@getanthill/telemetry';

import { Command } from 'commander';
import get from 'lodash/get';
import cloneDeep from 'lodash/cloneDeep';
import merge from 'lodash/merge';

import setup from '../../setup';
import services from '../../services';
import * as runner from '../runner';
import * as utils from '../../utils';

import thingsModelConfig from '../../templates/examples/things.json';

import expect from 'expect';

const CLI_VERSION = '0.2.0';

const program = new Command();

program
  .storeOptionsAsProperties(false)
  .version(CLI_VERSION, '-v, --version', 'output the current version');

function mapEntityValues(
  ctx: Ctx,
  entity: any,
  iteration: number,
  cloneIndex: number,
) {
  const clonedEntity = cloneDeep(entity);

  utils.mapValuesDeep(clonedEntity, (v: any) => {
    if (typeof v !== 'string') {
      return v;
    }

    // Entity mapping
    const [id, ...fragments] = v.slice(1, -1).split('.');

    if (ctx.entities!.has(id)) {
      return get(ctx.entities!.get(id), fragments);
    }

    return v
      .replace(/\{(i|iteration)\}/g, iteration.toString())
      .replace(/\{(c|clone_index)\}/g, cloneIndex.toString())
      .replace(/\{(i|iteration)-1\}/g, `${Math.max(0, iteration - 1)}`)
      .replace(/\{uuid\}/g, setup.uuid());
  });

  return clonedEntity;
}

function mockDate(str?: string) {
  if (str === undefined) {
    return;
  }

  const constantDate = new Date(str);

  // @ts-ignore
  global._Date = Date;

  // @ts-ignore
  global.Date = class extends Date {
    constructor() {
      super();

      return constantDate;
    }

    static now() {
      return constantDate.getTime();
    }
  };
}

function unmockDate() {
  /* @ts-ignore */
  if (global._Date === undefined) {
    return;
  }
  /* @ts-ignore */
  global.Date = global._Date;
  /* @ts-ignore */
  delete global._Date;
}

export async function buildContext(initialContext: any) {
  let ctx = {
    telemetry,
    ...initialContext,
  };

  ctx = {
    ...ctx,
    ...(await prepare(ctx)),
  };

  ctx.projections = await init(ctx);

  await sendEvents(ctx, ctx.data!.imports, 0);

  return ctx;
}

export async function getHandler(ctx: Ctx, handlerType: 'start' | 'replay') {
  /* @ts-ignore */
  const handler = runner[handlerType]();

  return handler(
    ctx.projections!.map(
      (projection) =>
        `/projections?projection_id=${projection.projection_id}&${
          ctx.data!.config.runner_params || ''
        }`,
    ),
    {
      exitTimeout: 100,
      verbose: true,
      cwd: path.resolve(__dirname, '..'),
      maxReconnectionAttempts: 1000,
      reconnectionInterval: 100,
      connectionMaxLifeSpanInSeconds: 3600,
      pageSize: 10,
    },
    {},
  );
}

export async function load(dir: string) {
  return require(dir);
}

export async function prepare(ctx: Ctx) {
  ctx.telemetry.logger.info('[validator] Preparing datastores...');

  const instances = new Map();
  const entities = new Map();
  const modelConfigs = new Map();
  const datastores = Object.keys(ctx.data!.config.datastores);

  services.datastores = new Map();

  for (const d of datastores) {
    ctx.telemetry.logger.debug('[validator] Starting datastore...', {
      datastore: d,
    });
    const instance = await setup.startApi(
      merge(
        {
          security: {
            tokens: [
              {
                id: 'admin',
                level: 'admin',
                token: 'token',
              },
            ],
          },
          features: {
            api: {
              admin: true,
              updateSpecOnModelsChange: true,
            },
            cache: {
              isEnabled: false,
            },
          },
        },
        ctx.data!.config.datastores[d].config,
      ),
    );

    instances.set(d, instance);

    services.datastores.set(d, instance[5]);

    const _modelConfigs = ctx.data!.config.datastores[d].modelConfigs || [];

    for (const modelConfig of _modelConfigs) {
      ctx.telemetry.logger.debug('[validator] Importing model...', {
        datastore: d,
        model: modelConfig.name,
      });

      await instance[5].createModel(modelConfig);
    }

    await instance[7].restart();

    for (const modelConfig of _modelConfigs) {
      ctx.telemetry.logger.debug('[validator] Creating model indexes...', {
        datastore: d,
        model: modelConfig.name,
      });
      await instance[5].createModelIndexes(modelConfig);
    }

    const { data: initializedModelConfigs } = await instance[5].getModels();
    ctx.telemetry.logger.debug('[validator] Available models...', {
      datastore: d,
      models: Object.keys(initializedModelConfigs),
    });
    modelConfigs.set(d, initializedModelConfigs);
  }

  return {
    instances,
    entities,
    modelConfigs,
    datastores,
  };
}

export async function init(ctx: Ctx) {
  ctx.telemetry.logger.info('[validator] Initializing projections...');
  const sourceInstance = ctx.instances.get(
    ctx.data!.projections[0]?.from?.datastore ||
      ctx.data!.projections[0]?.triggers[0].datastore,
  );

  try {
    sourceInstance[5].config.debug = true;
    const { data: ddd } = await sourceInstance[5].createModel({
      ...thingsModelConfig,
      db: 'datastore',
      name: 'projections',
      correlation_field: 'projection_id',
      schema: {
        ...thingsModelConfig.schema,
        model: {
          additionalProperties: true,
          properties: {},
        },
      },
    });

    await sourceInstance[7].restart();
  } catch (err: any) {
    if (err?.response?.data?.status !== 409) {
      throw err;
    }

    // Model already initialized
  }

  const projections = [];

  for (const projection of ctx.data!.projections) {
    const { data: _projection } = await sourceInstance[5].create(
      'projections',
      projection,
    );

    projections.push(_projection);
  }

  return projections;
}

export async function sendEvents(ctx: Ctx, events: any[], waitTimeout = 3000) {
  for (const event of events) {
    const sdk = ctx.instances.get(event.datastore)[5];
    const modelConfigs = ctx.modelConfigs.get(event.datastore);

    const repeat: number = event.repeat || 1;
    const clone: number = event.clone || 1;
    const sleep: number = event.sleep || 100;

    const entities: any[] = cloneDeep(event.entities || []);
    for (let i = 1; i < clone; i++) {
      entities.push(...event.entities);
    }

    ctx.telemetry.logger.info('[validator] Sending events...', {
      entities_count: entities.length,
      repeat,
      clone,
      sleep,
    });

    mockDate(event.date ?? ctx.data!.config.imports?.date);

    for (let iteration = 0; iteration < repeat; iteration++) {
      const _entities = entities.map((entity, cloneIndex) => ({
        ...entity,
        idempotency: mapEntityValues(
          ctx,
          entity.idempotency,
          iteration,
          cloneIndex,
        ),
        entity: mapEntityValues(ctx, entity.entity, iteration, cloneIndex),
        ...mapEntityValues(
          ctx,
          {
            id: entity.id,
          },
          iteration,
          cloneIndex,
        ),
      }));

      await sdk.import(
        _entities,
        modelConfigs,
        {
          dryRun: false,
        },
        ctx.entities,
      );

      await new Promise((resolve) => setTimeout(resolve, sleep));
    }

    unmockDate();
  }

  await new Promise((resolve) => setTimeout(resolve, waitTimeout));
}

export async function run(ctx: Ctx, handlerType: 'start' | 'replay') {
  handlerType === 'replay' && (await sendEvents(ctx, ctx.data!.events, 0));

  await getHandler(ctx, handlerType);

  handlerType === 'start' &&
    (await sendEvents(ctx, ctx.data!.events, ctx.data!.config.exit_timeout));
}

export async function assert(ctx: Ctx, handlerType: string) {
  let withErrors = false;
  ctx.telemetry.logger.info('[validator] Checking results...');

  const assertions = ctx.data!.assertions.filter(
    (assertion) =>
      assertion.handler === undefined || assertion.handler === handlerType,
  );

  for (const assertion of assertions) {
    if (assertion.entities.length === 0) {
      continue;
    }

    const res = await ctx.instances
      .get(assertion.datastore)[5]
      .find(assertion.model, assertion.query || {});

    try {
      if (assertion.log === true) {
        ctx.telemetry.logger.info('[validator] Assertion', { data: res.data });
      }

      expect(res.data).toEqual(
        expect.arrayContaining([
          expect.objectContaining(
            mapEntityValues(ctx, assertion.entities[0], 0, 0),
          ),
        ]),
      );
    } catch (err: any) {
      console.error(err);
      withErrors = true;
      ctx.telemetry.logger.error('[validator] ❌ Assertion failure', {
        err,
        data: res.data,
        assertion,
        details: err?.response?.data,
      });
    }
  }

  if (withErrors === true) {
    throw new Error('Tests failed');
  }

  ctx.telemetry.logger.info('[validator] ✅ All checks passed');
}

export async function tearDown(ctx: Ctx) {
  ctx.telemetry.logger.info('[validator] Stopping...');
  for (const ds of services.datastores.values()) {
    ds.closeAll();
  }

  for (const d of ctx.datastores || []) {
    const instance = ctx.instances.get(d);

    ctx.telemetry.logger.debug('[validator] Dropping database...', {
      datastore: d,
    });
    await setup.teardownDb(instance[1]);

    ctx.telemetry.logger.debug('[validator] Stopping datastore...', {
      datastore: d,
    });

    await setup.stopApi(instance[7]);
  }
}

async function validator(
  dir: string,
  handlerType: 'start' | 'replay',
  cmd: any,
) {
  let ctx: Ctx = {
    telemetry,
  };

  try {
    const data = await load(dir);
    ctx = await buildContext({ data });

    await run(ctx, handlerType);

    await assert(ctx, handlerType);
  } catch (err: any) {
    if (err?.response?.data) {
      console.error(err?.response?.data);
    } else {
      console.error(err);
    }
  } finally {
    await tearDown(ctx);
  }
}

export async function cli(argv = process.argv) {
  program
    .argument('<path>', 'Path of the projections entries')
    .argument('[handler_type]', 'Handler type: start, replay', 'start')
    .option(
      '-v, --verbosity <verbosity>',
      'Logger level verbosity. 0: ALL 100: NOTHING]',
      '50',
    )
    .option('-p, --projections <projections...>', 'List of path projections')
    .action(validator);

  return program.parse(argv);
}

if (!module.parent) {
  cli();
}