@getanthill/datastore/src/cli/index.ts

Event-Sourced Datastore

import type { ModelConfig } from '../typings';

import { inspect } from 'util';
import fs from 'node:fs';
import os from 'node:os';
import { createInterface } from 'node:readline/promises';
import { stdin as input, stdout as output } from 'node:process';

import Ajv from 'ajv';
import addFormats from 'ajv-formats';

import yaml from 'js-yaml';
import { parse } from 'shell-quote';

import cloneDeep from 'lodash/cloneDeep';
import uniq from 'lodash/uniq';

import { Command, Option } from 'commander';

import { build } from '../services';
import { Datastore } from '../sdk';

import runner from '../sdk/runner';

import { ok } from 'node:assert';
import { DatastoreConfig } from '../sdk/Datastore';

// CLI sub commands
import admin from './admin';
import data from './data';
import models from './models';
import security from './security';
import { CompleterResult } from 'node:readline';

const { version: CLI_VERSION } = require('../../package.json');

const program = new Command('ds');

let isRun = false;

program
  .storeOptionsAsProperties(false)
  .version(CLI_VERSION, '-v, --vers', 'output the current version')
  .description(`Datastore API client

Gitlab:
> https://gitlab.com/getanthill/datastore

Documentation:
> https://datastore.getanthill.org/docs/tutorials/using-the-cli

Examples:
ds models accounts find
ds models accounts find --is_enabled true
ds models accounts find --is_enabled true --sort updated_at --fields email created_at
`);

const datastoreConfigs: { name: string; config: DatastoreConfig }[] =
  JSON.parse(
    process.env.DATASTORE_CONFIGS ||
      JSON.stringify([
        {
          name: 'default',
          config: {
            baseUrl: process.env.DATASTORE_API_URL || 'http://localhost:3001',
            token: process.env.DATASTORE_ACCESS_TOKEN || 'token',
            debug: false,
          },
        },
      ]),
  );

const services = build();
const datastores = new Map<string, Datastore>(
  datastoreConfigs.map((c) => {
    ok(!!c.name, 'Missing Datastore configuration name');

    return [
      c.name,
      new Datastore({ ...c.config, telemetry: services.telemetry }),
    ];
  }),
);

services.datastores = datastores;

const validator = new Ajv({
  useDefaults: false,
  coerceTypes: true,
  strict: false,
});

// @ts-ignore
addFormats(validator);

let modelConfigs: { [key: string]: ModelConfig } = {};

function log(obj: any, format = process.env.DATASTORE_CLI_FORMAT) {
  const _format = isRun === true ? 'cli' : format;

  if (_format === 'json') {
    console.log(JSON.stringify(obj));

    return;
  }

  if (_format === 'yaml') {
    console.log(yaml.dump(obj));

    return;
  }

  console.log(inspect(obj, false, null, true));
}

function addDatastoreOptions(h: Command) {
  const dsNames = Array.from(datastores.keys());

  h.addOption(
    new Option('--ds, --datastore <datastore>', 'Datastore to use')
      .default(dsNames[0] || 'default')
      .choices(dsNames.length === 0 ? ['default'] : dsNames),
  );
}

function stream() {
  const h = program.command('stream <model> <source>');

  addDatastoreOptions(h);

  const dsNames = Array.from(datastores.keys());

  h.addOption(
    new Option('--output <output>', 'Output')
      .default('entity')
      .choices(['entity', 'raw']),
  )
    .addOption(
      new Option('--format <format>', 'Response format').choices([
        'json',
        'yaml',
      ]),
    )
    .addOption(
      new Option('-c, --connector <connector>', 'Connector type').choices([
        'http',
        'amqp',
      ]),
    )
    .addOption(new Option('-q, --queue <queue>', 'Queue name'))
    .addOption(
      new Option('-s, --sync <datastore>', 'Sync event to a datastore').choices(
        dsNames.length === 0 ? ['default'] : dsNames,
      ),
    )
    .description('Stream entities changes or events');

  h.action(async (model, source, cmd) => {
    if (!datastores) {
      return;
    }

    try {
      const datastore = datastores.get(cmd.datastore);

      if (!datastore) {
        return;
      }

      let streamHandler = datastore.streams.streamHTTP.bind(datastore.streams);

      if (cmd.connector === 'amqp') {
        streamHandler = datastore.streams.streamAMQP.bind(datastore.streams);
      }

      await streamHandler(
        async (e, _route, _headers, opts) => {
          log(e, cmd.format);

          if (source === 'events' && cmd.sync) {
            const _model = model === 'all' ? e.model : model;
            const event = model === 'all' ? e.entity : e;
            const modelConfig = modelConfigs[_model];
            datastores
              .get(cmd.sync)
              ?.apply(
                e.model,
                event[modelConfig.correlation_field],
                event.type,
                event.v,
                event,
                { replay: 'true' },
              );
          }

          typeof opts?.ack === 'function' && (await opts.ack());
        },
        model,
        source,
      );
    } catch (err: any) {
      if (err.response) {
        log(err.response.data, cmd.format);

        return;
      }

      log(err, cmd.format);
    }
  });
}

function config() {
  const h = program
    .command('config')
    .addOption(
      new Option('--format <format>', 'Response format').choices([
        'json',
        'yaml',
      ]),
    )
    .description('Show cli configuration');

  h.action(async (cmd) => {
    try {
      log(
        {
          configs: datastoreConfigs,
          datastores: Array.from(datastores.keys()),
        },
        cmd.format,
      );
    } catch (err: any) {
      log(err, cmd.format);
    }
  });
}

function run() {
  const h = program.command('run').description('CLI infinite command');

  let cliName: string;
  const commands = new Map();
  const defaultOptions = new Map();
  function exitOverride(program: Command, sub = '') {
    const name = ((sub ? sub + ' ' : '') + program.name()).replace(
      cliName + ' ',
      '',
    );

    if (sub === '') {
      cliName = name;
    }

    program.exitOverride();

    commands.set(name, program);

    if (!defaultOptions.has(name)) {
      defaultOptions.set(name, {
        args: cloneDeep(program.args),
        // @ts-ignore
        _optionValues: cloneDeep(program._optionValues),
        // @ts-ignore
        _optionValueSources: cloneDeep(program._optionValueSources),
      });
    }

    for (const prog of program.commands) {
      exitOverride(prog, name);
    }
  }

  function reset(program: Command, sub = '') {
    const name = ((sub ? sub + ' ' : '') + program.name()).replace(
      cliName + ' ',
      '',
    );

    const c = defaultOptions.get(name);

    program.args = cloneDeep(c?.args ?? []);
    // @ts-ignore
    program._optionValues = cloneDeep(c?._optionValues ?? {});
    // @ts-ignore
    program._optionValueSources = cloneDeep(c?._optionValueSources ?? {});
    for (const prog of program.commands) {
      reset(prog, name);
    }
  }

  h.action(async (cmd) => {
    const history: string[] = [];
    const historyPath = os.homedir() + '/.ds_history';

    process.on('beforeExit', () => {
      console.log('\nBye!');
    });

    try {
      await fs.promises
        .readFile(historyPath)
        .then((h) => {
          history.push(...h.toString().split('\n').reverse());
        })
        .catch(() => {
          // ...
        });

      const writeStream = fs.createWriteStream(historyPath, { flags: 'a' });

      isRun = true;
      exitOverride(program);

      const completer = (input: string): CompleterResult => {
        const completions = [...history, ...Array.from(commands.keys())];
        const hits = uniq(
          completions
            .filter((c) => new RegExp('^' + input, 'i').test(c))
            .map((h) =>
              h.replace(new RegExp('^(' + input + '[^\\s]+).*', 'i'), '$1'),
            ),
        );

        return [hits.length ? hits : completions, input] as CompleterResult;
      };
      const readline = createInterface({ input, output, history, completer });

      let stdin;

      console.log(`
Datastore cli@${CLI_VERSION}`);

      do {
        stdin = await readline.question('> ');

        history.push(stdin);
        writeStream.write(stdin + '\n');

        if (stdin === 'exit') {
          break;
        }

        try {
          reset(program);

          console.log('');

          program.showHelpAfterError(false);
          await program.parseAsync(parse(stdin) as string[], {
            from: 'user',
          });
        } catch (err2: any) {
          // ...
        }

        console.log('');
      } while (stdin !== 'exit');

      readline.close();
    } catch (err: any) {
      log(err, cmd.format);
    }
  });
}

export async function cli(argv = process.argv) {
  const _argv = [...process.argv]; // clone argv.
  const cleanArgv = _argv.slice(2); // remove node path etc.
  const firstArg: string = cleanArgv // first command ignore lfags.
    .filter((v) => !/^--?/.test(v))
    .shift()!;

  const isRunCli = cleanArgv.length === 0;

  if (isRunCli === true) {
    argv.push('run');
  } else if (
    firstArg !== 'run' &&
    !program.commands.some((cmd) => cmd.name() === firstArg)
  ) {
    _argv.splice(2, 0, 'help');
  }

  program
    .command('heartbeat')
    .description('Check the availability of the service (/heartbeat)')
    .action(async (cmd) => {
      for (const [name, datastore] of datastores) {
        try {
          const { data } = await datastore.heartbeat();
          log({ datastore: name, response: data }, cmd.format);
        } catch (err: any) {
          console.error('Datastore heartbeat failed', {
            datastore: name,
            config: datastore.config,
          });
        }
      }
    });

  for (const [name, datastore] of datastores) {
    try {
      const dsProgram = new Command(name);

      dsProgram.summary(`datastore: ${datastore.config.baseUrl}`);

      const { data: _models } = await datastore.getModels();

      const modelConfigs: ModelConfig[] = _models;
      for (const model of Object.values(modelConfigs)) {
        try {
          model.datastore = name;

          dsProgram.addCommand(models(services, model));
        } catch (errCommand) {
          console.log('[Error] Sub command', {
            datastore: name,
            model: model.name,
          });
          console.error(errCommand);
        }
      }

      program.addCommand(dsProgram);
    } catch (err: any) {
      // console.error(err);
    }
  }

  config();
  program.addCommand(admin(services));
  program.addCommand(data(services));
  program.addCommand(security(services));

  isRunCli === false && program.addCommand(runner());
  isRunCli === false && stream();

  if (isRunCli === true) {
    console.log(program.helpInformation());
    run();
  }

  return program.parse(argv);
}

if (!module.parent) {
  cli();
}