@getanthill/datastore/src/sdk/Streams.ts

Event-Sourced Datastore

import type { JSONSchemaType } from 'ajv';
import type { AnyObject, Source, Telemetry } from '../typings';
import type Core from './Core';

import { EventEmitter } from 'events';
import { EventSource } from 'eventsource';

import merge from 'lodash/merge';

import AMQPClient from '../services/amqp';
import { objToJsonSchema } from './utils';
import { MessageOptions, Route } from '../services/broker';

export interface StreamConfig {
  baseUrl?: string;
  token?: string;
  debug?: boolean;
  telemetry?: Telemetry;
  connector: 'http' | 'amqp';

  amqp?: any;
}

export type StreamHandler = (
  event: AnyObject,
  route?: Route,
  headers?: AnyObject,
  opts?: MessageOptions,
) => Promise<void> | boolean;
export type StreamClose = () => void;

export interface StreamAMQPOptions {
  output?: string;
  reconnect?: boolean;
  reconnectionInterval?: number;
  reconnectionMaxAttempts?: number;
  connectionMaxLifeSpanInSeconds?: number;
  queueName?: string;
  queryAsJSONSchema?: boolean;
}

export default class Streams extends EventEmitter {
  static ERRORS = {
    ERROR_STREAM_MAX_RECONNECTION_ATTEMPTS_REACHED: new Error(
      'Max reconnection attempts reached for streaming',
    ),
  };

  public config: StreamConfig = {
    baseUrl: 'http://localhost:3001',
    token: 'token',
    debug: false,
    connector: 'http',
  };

  private _core: Core;
  private _telemetry?: Telemetry;
  private _streams: Map<string, StreamClose> = new Map();

  constructor(config: Partial<StreamConfig>, core: Core) {
    super();

    this.config = merge({}, this.config, config);
    this._core = core;

    this._telemetry = this.config.telemetry;
  }

  getEventSource(url: string, headers?: Record<string, any>) {
    return new EventSource(url, {
      fetch: (input: RequestInfo | URL, init?: RequestInit) =>
        fetch(input, { ...init, headers: { ...headers, ...init?.headers } }),
    });
  }

  getAMQPClient(options?: StreamAMQPOptions) {
    return new AMQPClient(
      merge({}, this.config.amqp, {
        queue: { consumer: { name: options?.queueName } },
      }),
      this._telemetry,
    );
  }

  // HTTP Stream
  getStreamId(model: string, source: string, query: object = {}): string {
    return `${model}:${source}:${JSON.stringify(query)}`;
  }

  private async getStreamCloseMethod(
    model: string,
    source: Source,
    streamId: string,
    query?: AnyObject,
    options?: AnyObject,
  ) {
    let streamHandler;

    if (this.config.connector === 'amqp') {
      streamHandler = this.streamAMQP.bind(this);
    } else {
      streamHandler = this.streamHTTP.bind(this);
    }

    return streamHandler(
      (data, route, headers, opts) =>
        this.emit(streamId, data, route, headers, opts),
      model,
      source,
      query,
      options,
    );
  }

  /**
   * Streaming API
   * @beta
   */
  async listen(
    model: string,
    source: Source,
    query?: AnyObject,
    options?: AnyObject,
  ): Promise<StreamClose> {
    const streamId = this.getStreamId(model, source, query);

    if (this._streams.has(streamId)) {
      this._telemetry?.logger.debug('[Datastore] Stream already registered');
      return Promise.resolve(this._streams.get(streamId)!);
    }

    const close = await this.getStreamCloseMethod(
      model,
      source,
      streamId,
      query,
      options,
    );

    this._streams.set(streamId, close);

    if (typeof options?.forward?.emit === 'function') {
      this.on(streamId, (event, route, headers, opts) =>
        options.forward.emit(streamId, event, route, headers, opts),
      );
    }

    /* istanbul ignore next */
    return () => this.close(streamId);
  }

  close(streamId: string) {
    const close = this._streams.get(streamId);
    typeof close === 'function' && close();

    this._streams.delete(streamId);
    this.removeAllListeners(streamId);
  }

  closeAll() {
    for (const streamId of this._streams.keys()) {
      this.close(streamId);
    }

    this._telemetry?.logger.info('[Datastore#closeAll] All streams closed', {
      url: this.config.baseUrl,
    });
  }

  async streamAMQP(
    handler: StreamHandler,
    model: string,
    source: Source,
    query: AnyObject = {},
    options?: StreamAMQPOptions,
  ): Promise<StreamClose> {
    this._telemetry?.logger.info('[Datastore#streamAMQP] Initialization', {
      url: this.config.baseUrl,
      model,
      source,
      query,
      options,
    });
    const amqp = this.getAMQPClient(options);

    await amqp.connect();

    const modelName = model === 'all' ? '*' : model;

    const topic =
      source === 'events'
        ? `${modelName}/*/events/*`
        : `${modelName}/*/success/*`;

    amqp.on(topic, async (event, route, headers, opts) => {
      await handler(event, route, headers, opts);
    });

    const schema =
      options?.queryAsJSONSchema === true ? query : objToJsonSchema(query);

    await amqp.subscribe(topic, schema as JSONSchemaType<AnyObject>);

    return () => {
      amqp.end().catch((err) => {
        this._telemetry?.logger.error('[Streams#streamAMQP] Error on closing', {
          err,
        });
      });
    };
  }

  /**
   * @deprecated in favor of streamHTTP
   */
  stream(...args: any): Promise<StreamClose> {
    /* @ts-ignore */
    return this.streamHTTP.call(this, ...args);
  }

  /**
   * Stream route to listen for specific documents update from the
   * Datastore
   */
  /* istanbul ignore next */
  async streamHTTP(
    handler: StreamHandler,
    model = 'all',
    source: Source = 'entities',
    query?: AnyObject,
  ): Promise<StreamClose> {
    return new Promise((resolve) => {
      const projections = Array.isArray(query) ? query : [];
      if (query && !Array.isArray(query) && Object.keys(query).length > 0) {
        const match: AnyObject = {};
        for (const key of Object.keys(query)) {
          match[`fullDocument.${key}`] = query[key];
        }
        projections.push({ $match: match });
      }

      const evtSource = this.getEventSource(
        this.config.baseUrl +
          this._core.getPath('stream', model, source, 'sse') +
          '?pipeline=' +
          JSON.stringify(projections),
        { authorization: this.config.token },
      );

      evtSource.addEventListener('message', async function (event) {
        if (!event.data) {
          return;
        }

        await handler(JSON.parse(event.data));
      });

      evtSource.addEventListener('error', (err: any) =>
        this._telemetry?.logger.warn('[Datastore#streamHTTP] Stream error', {
          url: this.config.baseUrl,
          model,
          source,
          err,
        }),
      );

      const _close = () => {
        evtSource.close();
      };

      /**
       * The event is emitted on first line reception. This is
       * possible thanks to the keep alive message
       * (#serverSentEventKeepAlive: :\n\n)
       */
      evtSource.addEventListener('open', (e) => {
        resolve(_close);
      });
    });
  }
}