@getanthill/datastore/src/cli/models.ts

Event-Sourced Datastore

import type { ModelConfig, ModelSchema, Services } from '../typings';

import { Argument, Command, Option } from 'commander';

import Ajv from 'ajv';
import addFormats from 'ajv-formats';

import merge from 'lodash/merge';
import omit from 'lodash/omit';
import pick from 'lodash/pick';

import * as utils from './utils';
import { REGEXP_DATE_ISO_STRING_8601 } from '../constants';

const validator = new Ajv({
  useDefaults: false,
  coerceTypes: true,
  strict: false,
});

// @ts-ignore
addFormats(validator);

function addSchemaFields(
  h: Command,
  {
    correlation_field,
    schema,
  }: { correlation_field: string; schema: ModelSchema },
) {
  h.option(
    `--${correlation_field} <${correlation_field}...>`,
    'Correlation field',
  );

  h.option(`--json <json>`, 'JSON Query', JSON.parse);

  for (const field in schema.model.properties) {
    try {
      const { type, description } = schema.model.properties[field];
      if (type === 'array') {
        h.option(`--${field} <${field}...>`, description);
      } else {
        h.option(`--${field} <${field}>`, description);
      }
    } catch (err) {
      // console.error(err);
      // ...
    }
  }
}

function buildQuery(config: ModelConfig, cmd: any, schema: any): any {
  const { debug, dryRun, json, ...args } = cmd;

  const query = merge(
    pick(
      args,
      Object.keys({
        [config.correlation_field]: {
          type: 'string',
          description: 'Correlation field',
        },
        ...schema.properties,
      }),
    ),
    json,
  );

  const isValid = validator.validate(omit(schema, 'required'), query);

  if (debug === true) {
    utils.log({ dryRun, args, json, query, isValid }, cmd.format);
  }

  if (cmd.skipValidation === false && !isValid) {
    utils.log(
      {
        err: 'Query is invalid',
        dryRun,
        args,
        json,
        query,
        isValid,
        details: validator.errors,
      },
      cmd.format,
    );

    throw new Error('Query is invalid');
  }

  return query;
}

export function find(services: Services, config: ModelConfig) {
  const model = config.name;

  return async (cmd: any) => {
    try {
      const datastore = services.datastores.get(config.datastore);

      if (!datastore) {
        return;
      }

      const { page, pageSize, ...args } = cmd;
      const query = buildQuery(config, args, config.schema.model);

      query._must_hash = cmd.skipHash === false;

      if (cmd.fields) {
        query._fields = cmd.fields.reduce(
          (s: any, c: string) => ({
            ...s,
            [c]: 1,
          }),
          {},
        );
      }

      if (cmd.sort) {
        query._sort = cmd.sort.reduce(
          (s: any, c: string) => ({
            [c.replace(/^[+-]/, '')]: c[0] === '-' ? -1 : 1,
          }),
          {},
        );
      }

      if (cmd.dryRun === true) {
        utils.log(query, cmd.format);

        return;
      }

      let entities = [];
      if (cmd.source === 'entities') {
        const { data } = await datastore.find(model, query, page, pageSize, {
          // @ts-ignore
          'with-response-validation': `${cmd.skipValidation === false}`,
        });

        entities = data;
      } else {
        const { data } = await datastore.allEvents(
          model,
          query,
          page,
          pageSize,
          {},
        );

        entities = data;
      }

      if (cmd.decrypt === true) {
        const { data: decryptedData } = await datastore.decrypt(
          model,
          entities,
        );

        entities = decryptedData;
      }

      utils.log(entities, cmd.format);
    } catch (err: any) {
      if (err.response) {
        utils.log(err.response.data, cmd.format);

        return;
      }

      utils.log(err, cmd.format);
    }
  };
}

export function get(services: Services, config: ModelConfig) {
  const model = config.name;

  return async (correlationId: string, cmd: any) => {
    try {
      const datastore = services.datastores.get(config.datastore);

      if (!datastore) {
        return;
      }

      if (cmd.dryRun === true) {
        utils.log(
          {
            [config.correlation_field]: correlationId,
          },
          cmd.format,
        );

        return;
      }

      let entity;
      const { data: obj } = await datastore.get(model, correlationId);

      entity = obj;

      if (cmd.decrypt === true) {
        const {
          data: [decrypted],
        } = await datastore.decrypt(model, [entity]);

        entity = decrypted;
      }

      utils.log(entity, cmd.format);
    } catch (err: any) {
      if (err.response) {
        utils.log(err.response.data, cmd.format);

        return;
      }

      utils.log(err, cmd.format);
    }
  };
}

export function create(services: Services, config: ModelConfig) {
  const model = config.name;
  const schema = config.schema;

  return async (cmd: any) => {
    const v = Object.keys(schema.events.CREATED).sort().pop();

    if (!v) {
      return;
    }

    try {
      const datastore = services.datastores.get(config.datastore);

      if (!datastore) {
        return;
      }

      const payload = buildQuery(config, cmd, schema.events.CREATED[v]);

      if (cmd.dryRun === false) {
        const { data } = await datastore.create(model, payload);
        utils.log(data, cmd.format);
      } else {
        utils.log(payload, cmd.format);
      }
    } catch (err: any) {
      if (err.response) {
        utils.log(err.response.data, cmd.format);

        return;
      }

      utils.log(err, cmd.format);
    }
  };
}

export function update(services: Services, config: ModelConfig) {
  const model = config.name;
  const schema = config.schema;

  return async (correlationId: string, cmd: any) => {
    const v = Object.keys(schema.events.CREATED).sort().pop();

    if (!v) {
      return;
    }

    try {
      const datastore = services.datastores.get(config.datastore);

      if (!datastore) {
        return;
      }

      const payload = buildQuery(config, cmd, schema.events.UPDATED[v]);

      if (cmd.dryRun === false) {
        // @ts-ignore
        const { data } = await datastore.update(model, correlationId, payload, {
          upsert: cmd.upsert,
        });
        utils.log(data, cmd.format);
      } else {
        utils.log(payload, cmd.format);
      }
    } catch (err: any) {
      if (err.response) {
        utils.log(err.response.data, cmd.format);

        return;
      }

      utils.log(err, cmd.format);
    }
  };
}

export function patch(services: Services, config: ModelConfig) {
  const model = config.name;

  return async (correlationId: string, _patches: string[], cmd: any) => {
    try {
      const datastore = services.datastores.get(config.datastore);

      if (!datastore) {
        return;
      }

      const patches = _patches.map((p) => JSON.parse(p));

      if (cmd.dryRun === false) {
        const { data } = await datastore.patch(model, correlationId, patches);
        utils.log(data, cmd.format);
      } else {
        utils.log(patches, cmd.format);
      }
    } catch (err: any) {
      if (err.response) {
        utils.log(err.response.data, cmd.format);

        return;
      }

      utils.log(err, cmd.format);
    }
  };
}

export function restore(services: Services, config: ModelConfig) {
  const model = config.name;

  return async (correlationId: string, version: number, cmd: any) => {
    try {
      const datastore = services.datastores.get(config.datastore);

      if (!datastore) {
        return;
      }

      if (cmd.dryRun === false) {
        const { data } = await datastore.restore(model, correlationId, version);
        utils.log(data, cmd.format);
      } else {
        utils.log({ correlation_id: correlationId, version }, cmd.format);
      }
    } catch (err: any) {
      if (err.response) {
        utils.log(err.response.data, cmd.format);

        return;
      }

      utils.log(err, cmd.format);
    }
  };
}

export function count(services: Services, config: ModelConfig) {
  const model = config.name;

  return async (cmd: any) => {
    try {
      const datastore = services.datastores.get(config.datastore);

      if (!datastore) {
        return;
      }

      const { page, pageSize, ...args } = cmd;
      const query = buildQuery(config, args, config.schema.model);

      if (cmd.dryRun === false) {
        const c = await datastore.count(
          model,
          {
            ...query,
            _must_hash: cmd.skipHash === false,
          },
          cmd.source,
        );
        utils.log(c, cmd.format);
      } else {
        utils.log(query, cmd.format);
      }
    } catch (err: any) {
      if (err.response) {
        utils.log(err.response.data, cmd.format);

        return;
      }

      utils.log(err, cmd.format);
    }
  };
}

export function events(services: Services, config: ModelConfig) {
  const model = config.name;

  return async (correlationId: string, cmd: any) => {
    try {
      const datastore = services.datastores.get(config.datastore);

      if (!datastore) {
        return;
      }

      const { page, pageSize } = cmd;

      if (cmd.dryRun === true) {
        utils.log(
          {
            [config.correlation_field]: correlationId,
          },
          cmd.format,
        );

        return;
      }

      let events = [];
      const { data } = await datastore.events(
        model,
        correlationId,
        page,
        pageSize,
      );

      events = data;

      if (cmd.decrypt === true) {
        const { data: decryptedEvents } = await datastore.decrypt(
          model,
          events,
        );

        events = decryptedEvents;
      }

      utils.log(events, cmd.format);
    } catch (err: any) {
      if (err.response) {
        utils.log(err.response.data, cmd.format);

        return;
      }

      utils.log(err, cmd.format);
    }
  };
}

export function entity(services: Services, config: ModelConfig) {
  const model = config.name;

  return async (verb: string, correlationId: string, cmd: any) => {
    try {
      const datastore = services.datastores.get(config.datastore);

      if (!datastore) {
        return;
      }

      if (cmd.dryRun === true) {
        utils.log(
          {
            [config.correlation_field]: correlationId,
          },
          cmd.format,
        );

        return;
      }

      let res;
      if (verb === 'data') {
        res = await datastore.data(model, correlationId, cmd.onlyModels);
      } else {
        /* @ts-ignore */
        res = await datastore[verb](model, correlationId, true, cmd.onlyModels);
      }

      utils.log(res.data, cmd.format);
    } catch (err: any) {
      if (err.response) {
        utils.log(err.response.data, cmd.format);

        return;
      }

      utils.log(err, cmd.format);
    }
  };
}

export function updateMany(services: Services, config: ModelConfig) {
  const model = config.name;

  return async (query: string, update: string, cmd: any) => {
    try {
      const datastore = services.datastores.get(config.datastore);
      const _update = JSON.parse(update);

      let stats: any;
      await datastore?.updateOverwhelmingly<string>(
        model,
        JSON.parse(query),
        async () => _update,
        (_stats: any) => {
          stats = _stats;
          Math.floor(stats.progress * 100) % 5 === 0 &&
            utils.log(stats, cmd.format);
        },
        100,
      );
    } catch (err: any) {
      if (err.response) {
        utils.log(err.response.data, cmd.format);

        return;
      }

      utils.log(err, cmd.format);
    }
  };
}

export function version(services: Services, config: ModelConfig) {
  const model = config.name;
  return async (correlationId: string, version: string, cmd: any) => {
    try {
      const datastore = services.datastores.get(config.datastore);
      if (!datastore) {
        return;
      }
      if (cmd.dryRun === true) {
        utils.log(
          {
            [config.correlation_field]: correlationId,
          },
          cmd.format,
        );
        return;
      }
      let res;

      const versionNumber = Number.parseInt(version, 10);
      const isVersionISOString = REGEXP_DATE_ISO_STRING_8601.test(version);
      if (isVersionISOString === true) {
        res = await datastore.at(model, correlationId, version);
      } else {
        res = await datastore.version(model, correlationId, versionNumber);
      }

      utils.log(res.data, cmd.format);
    } catch (err: any) {
      if (err.response) {
        utils.log(err.response.data, cmd.format);
        return;
      }
      utils.log(err, cmd.format);
    }
  };
}

export default function register(services: Services, config: ModelConfig) {
  const program = new Command(config.name);
  const summary = config.description?.split('\n', 2)[0] ?? '';

  program
    .summary(
      `${config.datastore}@${config.name} (v${config.version})${
        summary ? ' - ' + summary : ''
      }`,
    )
    .description(
      `datastore:\t${config.datastore}
version:\t${config.version}
created_at:\t${config.created_at}
updated_at:\t${config.updated_at}

description:
${(config.description || config.name || '').split('\n').shift()}
`,
    );

  let c = program
    .command('schema')
    .description(`Returns the JSON schema associated to ${config.name}`)
    .action(() => {
      utils.log(config.schema.model);
    });

  // Find
  c = program
    .command(`find`)
    .description(`Get available entities for ${config.name}`);

  addSchemaFields(c, config);
  utils.addStandardFields(c);

  try {
    c.addOption(
      new Option('-s, --source <source>', 'Count source: entities or events')
        .default('entities')
        .choices(['entities', 'events']),
    );
  } catch (err) {
    // console.error(err);
  }

  c.option('--fields <fields...>', 'Fields to return from the entity');
  c.option('--sort <sorts...>', 'Sort entities on some specific fields');
  c.option('--decrypt', 'Must try to decrypt entities', false);
  c.option('--skip-hash', 'Remove the hash request', false);
  c.option('--skip-validation', 'Skip query validation', false);

  utils.addPaginationFields(c);

  c.action(find(services, config));

  // Get
  c = program
    .command(`get`)
    .argument('<correlation_id>', 'Entity correlation ID')
    .description(`Get one entity for ${config.name}`);

  addSchemaFields(c, config);
  utils.addStandardFields(c);

  c.option('--decrypt', 'Must try to decrypt entities').action(
    get(services, config),
  );

  // Create
  c = program
    .command(`create`)
    .description(`Create a new entity for ${config.name}`);

  addSchemaFields(c, config);
  utils.addStandardFields(c);

  c.option('--skip-validation', 'Skip query validation', false);
  c.action(create(services, config));

  // Update
  c = program
    .command(`update`)
    .argument('<correlation_id>', 'Entity correlation ID')
    .description(`Update a single entity for ${config.name}`);

  c.option('--upsert', 'Upsert the entity if not exists', false);

  addSchemaFields(c, config);
  utils.addStandardFields(c);

  c.option('--skip-validation', 'Skip query validation', false);
  c.action(update(services, config));

  // UpdateMany
  c = program
    .command(`update:many`)
    .argument('<query>', 'Matching query')
    .argument('<update>', 'Update query')
    .description(`Update all entities for ${config.name} matching the query`);

  c.action(updateMany(services, config));

  // Patch
  c = program
    .command(`patch`)
    .argument('<correlation_id>', 'Entity correlation ID')
    .argument('<patches...>', 'JSON Patch')
    .description(`Patch a single entity for ${config.name}`);

  utils.addStandardFields(c);

  c.action(patch(services, config));

  // Restore
  c = program
    .command(`restore`)
    .argument('<correlation_id>', 'Entity correlation ID')
    .argument('<version>', 'Entity version to restore')
    .description(
      `Restore a single entity for ${config.name} to a given version`,
    );

  utils.addStandardFields(c);

  c.action(restore(services, config));

  // Count
  c = program
    .command(`count`)
    .description(
      `Get count of entities for ${config.name} matching your query`,
    );

  addSchemaFields(c, config);
  utils.addStandardFields(c);

  try {
    c.addOption(
      new Option('-s, --source <source>', 'Count source: entities or events')
        .default('entities')
        .choices(['entities', 'events']),
    );
  } catch (err) {
    // console.error(err);
  }

  c.option('--skip-hash', 'Remove the hash request', true);
  c.option('--skip-validation', 'Skip query validation', false);
  c.action(count(services, config));

  // Events
  c = program
    .command(`events`)
    .argument('<correlation_id>', 'Entity correlation ID')
    .description(`Get events associated to a ${config.name}`);

  utils.addPaginationFields(c);
  utils.addStandardFields(c);

  c.option('--decrypt', 'Must try to decrypt entities').action(
    events(services, config),
  );

  // Entity
  c = program
    .command('entity')
    .addArgument(
      new Argument('<verb>', 'Verb to apply on the entity').choices([
        'data',
        'archive',
        'unarchive',
        'delete',
      ]),
    )
    .argument('<correlation_id>', 'Entity correlation ID')
    .description(
      `Apply a specific method to data related to a single entity for ${config.name}`,
    );

  c.option('--only-models <models...>', 'Models to retrieve data from');

  utils.addStandardFields(c);
  c.action(entity(services, config));

  // Version
  c = program
    .command('version')
    .argument('<correlation_id>', 'Entity correlation ID')
    .addArgument(new Argument('<version>', 'Version'))
    .description(`Get entity at version`);

  utils.addStandardFields(c);
  c.action(version(services, config));

  return program;
}