@getanthill/datastore/src/services/amqp.ts

Event-Sourced Datastore

import type { JSONSchemaType } from 'ajv';
import type { Access, AnyObject, Telemetry } from '../typings';
import type { Route } from './broker';

import amqplib, { ConsumeMessage } from 'amqplib';

import merge from 'lodash/merge';
import omit from 'lodash/omit';

import Broker from './broker';

const REGEXP_MATCH_SLASHES = /\//g;
const REGEXP_MATCH_WILDCARD = /\*/g;

export default class AMQPClient extends Broker {
  static ERRORS = {
    NOT_CONNECTED: new Error('AMQP disconnected'),
  };

  private _connection: amqplib.Connection | null = null;
  private _channel: amqplib.Channel | null = null;
  private _queue: {
    topic: string;
    payload: object;
    options: object;
  }[] = [];
  private _alreadyConnected = false;
  private _connectionId = 0;
  private _connectionUrls: string[] = [];
  private _closing = false;
  private _consumerTag = '';
  private _consuming: Map<string, Promise<void>> = new Map();
  private _consumerTags: Map<
    string,
    {
      topic: string;
      schema: JSONSchemaType<AnyObject>;
      queueConfig: object;
    }
  > = new Map();

  constructor(config: any, telemetry?: Telemetry) {
    super(config, telemetry);

    this._connectionUrls =
      Array.isArray(config.url) === false ? [config.url] : config.url;
  }

  get connection() {
    if (this._connection === null) {
      throw AMQPClient.ERRORS.NOT_CONNECTED;
    }

    return this._connection;
  }

  get channel() {
    if (this._channel === null) {
      throw AMQPClient.ERRORS.NOT_CONNECTED;
    }

    return this._channel;
  }

  mapRoutingKey(
    topic: string,
    schema: JSONSchemaType<AnyObject>,
  ): Route & { routingKey: string } {
    const topicWithNamespace = this.topicWithNamespace(topic);

    return {
      original: topic,
      topic: topicWithNamespace.replace(
        Broker.REGEXP_MATCH_PATH_PARAMETERS,
        '+',
      ),
      routingKey: topicWithNamespace
        .replace(REGEXP_MATCH_SLASHES, '.')
        .replace(Broker.REGEXP_MATCH_PATH_PARAMETERS, '*'),
      regexp: new RegExp(
        topicWithNamespace
          .replace(Broker.REGEXP_MATCH_PATH_PARAMETERS, '([^\\/]+)')
          .replace(REGEXP_MATCH_WILDCARD, '.*'),
      ),
      paramNames: (Broker.REGEXP_MATCH_PATH_PARAMETERS.exec(topic) ?? []).map(
        (p) => p.slice(1, -1),
      ),
      params: {},
      validate: this.ajv.compile(schema),
    };
  }

  async connect(): Promise<AMQPClient> {
    if (this._connection !== null) {
      return this;
    }

    try {
      this.telemetry?.logger.debug('[AMQP] Connecting...');
      const connectionUrl = this._connectionUrls[this._connectionId];

      this._connectionId =
        (this._connectionId + 1) % this._connectionUrls.length;

      this._connection = await amqplib.connect(
        connectionUrl + '?heartbeat=10',
        this.config.options,
      );
      this._alreadyConnected = true;
      this.telemetry?.logger.debug('[AMQP] Connected ✅', {
        url: connectionUrl,
      });

      this._connection.on('error', (err) => {
        this.telemetry?.logger.error('[AMQP] Error', err);
      });

      this._connection.on('close', () => {
        if (this._closing === true) {
          return;
        }
        this.telemetry?.logger.debug('[AMQP] Reconnecting...');
        this._channel = null;
        this._connection = null;

        setTimeout(
          this.connect.bind(this),
          this.config.failover?.reconnectionTimeoutInMilliseconds,
        );
      });

      this._channel = await this._connection.createChannel();

      await this.channel.prefetch(this.config.channel.prefetch);

      await this.init();

      await this.resubscribe();

      await this.emptyQueue();
    } catch (err) {
      if (this._alreadyConnected === false && this._connectionId === 0) {
        throw err;
      }

      setTimeout(
        this.connect.bind(this),
        this.config.failover?.reconnectionTimeoutInMilliseconds,
      );
    }
    return this;
  }

  async init(): Promise<AMQPClient> {
    await Promise.all([
      this.channel.assertExchange(
        this.config.exchange.consumer.name,
        this.config.exchange.consumer.type,
        this.config.exchange.consumer.options,
      ),
      this.channel.assertExchange(
        this.config.exchange.producer.name,
        this.config.exchange.producer.type,
        this.config.exchange.producer.options,
      ),
      this.channel.assertQueue(
        this.config.queue.consumer.name,
        this.config.queue.consumer.options,
      ),
      this.config.queue.errors.isEnabled === true &&
        this.channel.assertQueue(
          this.config.queue.errors.name,
          this.config.queue.errors.options,
        ),
    ]);

    if (this.config.queue.errors.isEnabled === true) {
      await this.channel.bindQueue(
        this.config.queue.errors.name,
        this.config.exchange.producer.name,
        '*.*.errors',
      );
    }

    return this;
  }

  async end(): Promise<AMQPClient> {
    this._closing = true;
    this._channel !== null &&
      this._consumerTag &&
      (await this.channel.cancel(this._consumerTag));
    this._consumerTag = '';
    this._consuming = new Map();
    this._consumerTags = new Map();

    this._channel !== null && (await this._channel.close());
    this._channel = null;

    this._connection !== null && (await this._connection.close());
    this._connection = null;

    this._closing = false;

    return this;
  }

  authenticate(tokens: Access[], handler: (...args: any[]) => Promise<void>) {
    return (event: any, route: any, headers: any, opts: any) => {
      const token = headers?.['authorization'];

      if (tokens.find((t) => t.token === token)) {
        return handler(event, route, headers, opts);
      }

      typeof opts?.ack === 'function' && opts.ack();
      this.telemetry?.logger.debug(
        '[events#authenticate] Authentication failed',
        {
          event,
        },
      );
    };
  }

  nackIfFirstSeen(message: ConsumeMessage) {
    message?.fields?.redelivered === true
      ? this.channel.ack(message)
      : this.channel.nack(message);
  }

  onMessage(message: ConsumeMessage | null) {
    if (!message) {
      return;
    }

    try {
      const { fields, content, properties } = message;
      this.telemetry?.logger.debug('[amqp#onMessage] Received a new message', {
        fields,
        properties,
      });

      const routingKey = fields.routingKey.replace(/\./g, '/');
      const route = this.getRoute(routingKey);

      if (route === null) {
        this.telemetry?.logger.debug(
          '[services#amqp] No route matching the routing key',
          {
            routingKey,
            acked: false,
          },
        );

        return this.nackIfFirstSeen(message);
      }

      const event = JSON.parse(content.toString());

      const headers = properties.headers ?? {};

      this.telemetry?.logger.debug('[services#amqp] Handling event', {
        routingKey,
        event,
        headers: omit(headers, 'authorization'),
      });

      const isValid = route.validate(event);

      if (isValid === false) {
        this.logOnInvalid(event, route);

        return this.channel.ack(message);
      }

      this.emit(
        route.original,
        event,
        this.parseTopic(routingKey, route),
        headers,
        {
          source: 'amqp',
          original: message,
          delivery: fields.redelivered === true ? 1 : 0,
          ack: () => this.channel.ack(message),
          nack: () => this.channel.nack(message),
        },
      );
    } catch (err) {
      this.telemetry?.logger.warn(
        '[services#amqp] Failed processing message',
        err,
      );

      this.nackIfFirstSeen(message);
    }
  }

  async emptyQueue() {
    while (this._queue.length > 0) {
      const { topic, payload, options } = this._queue.shift()!;
      await this.publish(topic, payload, options);
    }
  }

  async publish(topic: string, payload: object, options: object = {}) {
    try {
      return await this.channel.publish(
        this.config.exchange.producer.name,
        this.topicWithNamespace(topic).replace(REGEXP_MATCH_SLASHES, '.'),
        Buffer.from(JSON.stringify(payload)),
        merge({}, options, {
          headers: this.config.headers,
        }),
      );
    } catch (err) {
      this._queue.push({
        topic,
        payload,
        options,
      });
    }
  }

  async consume(
    queueConfig: { name: string },
    consumeOptions: amqplib.Options.Consume = this.config.consume?.options,
  ) {
    if (this._consuming.has(queueConfig.name)) {
      return this._consuming.get(queueConfig.name);
    }

    const promise = new Promise<void>((resolve, reject) => {
      this.channel
        .consume(queueConfig.name, this.onMessage.bind(this), consumeOptions)
        .then(({ consumerTag }) => {
          this._consumerTag = consumerTag;

          resolve();
        })
        .catch(reject);
    });

    this._consuming.set(queueConfig.name, promise);

    return promise;
  }

  async resubscribe() {
    const subscriptions = Array.from(this._consumerTags.values());
    this._consumerTag = '';
    this._consuming = new Map();
    this._consumerTags = new Map();
    return Promise.all(
      subscriptions.map(({ topic, schema, queueConfig }) =>
        this.subscribe(topic, schema, queueConfig),
      ),
    );
  }

  async subscribe(
    topic: string,
    schema: JSONSchemaType<AnyObject>,
    queueConfig = this.config.queue.consumer,
  ) {
    const mappedTopic = this.mapRoutingKey(topic, schema);

    this.topics.set(mappedTopic.routingKey, mappedTopic);

    this.addChannelToSpec(topic, schema);

    await this.channel.bindQueue(
      queueConfig.name,
      this.config.exchange.consumer.name,
      mappedTopic.routingKey,
    );

    this._consumerTags.set(topic, {
      topic,
      schema,
      queueConfig,
    });

    await this.consume(queueConfig);

    return this;
  }

  async next(
    topic: string,
    timeout = 5000,
    cb = async (opts: any) => opts.ack(),
  ): Promise<string> {
    let resolve: (value: string | PromiseLike<string>) => void;
    let _timeout: NodeJS.Timeout;
    const promise = new Promise<string>((_resolve, _reject) => {
      resolve = _resolve;

      _timeout = setTimeout(
        () => _reject(new Error('[amqp#next] Message timeout')),
        timeout,
      );
    });

    const handler = async (event: any, route: any, headers: any, opts: any) => {
      clearTimeout(_timeout);

      resolve(event);

      this.removeListener(topic, handler);

      opts !== undefined && (await cb(opts));
    };

    this.on(topic, handler);

    return promise;
  }
}