@ocap/indexdb-elasticsearch/lib/table/base.js

OCAP indexdb adapter that uses elasticsearch as backend

/* eslint-disable no-await-in-loop */
/* eslint-disable no-underscore-dangle */
const { BN } = require('@ocap/util');
const get = require('lodash/get');
const { BaseIndex } = require('@ocap/indexdb');

const debug = require('debug')(require('../../package.json').name);

const formatEsResponseError = (err) => (get(err, 'meta.body') ? err.meta.body : err);

class ESIndex extends BaseIndex {
  constructor({ name, docId, client, http, indexParams }) {
    super(name, docId);

    this.docId = docId;
    this.client = client;
    this.http = http;
    this.indexParams = indexParams;

    this.ensureIndex(indexParams).then(() => {
      debug(`index ready ${name}`);
      this.markReady();
    });
  }

  async search(params = {}) {
    debug(`search ${this.name}`, params);
    try {
      const { body: result } = await this.client.search({ index: this.name, ...params });
      return result;
    } catch (err) {
      if (get(err, 'meta.body.error')) {
        console.error(`failed to search ${this.name}, error: ${err.message}`);
        console.error(err.meta.body.error);
      } else {
        console.error(`failed to search ${this.name}`, err);
      }

      return { hits: { hits: [], total: { value: 0 } } };
    }
  }

  async ensureIndex({ mappings = {}, settings = {} }) {
    const { name, http } = this;
    const endpoint = `/${name}`;
    const sleep = (timeout) =>
      new Promise((resolve) => {
        setTimeout(resolve, timeout);
      });

    try {
      const { data: exist } = await http.get(endpoint);
      debug('update index', exist);
      await http.put(`/${name}/_mappings`, mappings);
    } catch (err) {
      if (get(err, 'response.data.status') === 404) {
        debug('create index', name);
        await http.put(`/${name}?wait_for_active_shards=1`, { mappings, settings });

        // Wait for index active
        const maxRetry = 20;
        const retryDelay = 500;

        let created = null;
        let retryCount = 0;
        do {
          await sleep(retryDelay);
          try {
            const { data } = await http.get(endpoint);
            created = data;
            return created[name];
          } catch (e) {
            retryCount++;
          }
        } while (!created && retryCount < maxRetry);
      }

      console.error(`failed to update index ${name}`, err);
      if (err.response) {
        // eslint-disable-next-line
        console.log(err.response.data);
      }
      throw err;
    }

    const { data } = await http.get(endpoint);
    return data[name];
  }

  count() {
    return this.client.count({ index: this.name }).then((res) => res.body.count);
  }

  batchInsert(rows) {
    const body = rows.flatMap((row) => [{ index: { _index: this.name, _id: this.generatePrimaryKey(row) } }, row]);
    return this.client.bulk({ refresh: 'wait_for', body });
  }

  batchUpdate(rows) {
    const body = rows.flatMap((row) => [
      { update: { _index: this.name, _id: this.generatePrimaryKey(row) } },
      { doc: row },
    ]);
    return this.client.bulk({ refresh: 'wait_for', body });
  }

  async _insert(row) {
    const copy = { ...row };
    try {
      const id = this.generatePrimaryKey(copy);
      const doc = await this._get(id);
      if (doc) {
        return null;
      }

      const { body } = await this.client.create({
        index: this.name,
        id,
        body: copy,
        refresh: process.env.NODE_ENV === 'test' ? 'wait_for' : undefined,
      });
      debug(`insert ${this.name} ${body.result}`, copy);

      return row;
    } catch (err) {
      console.warn(`failed to insert ${this.name}`, formatEsResponseError(err));
      throw err;
    }
  }

  async _get(docId) {
    const id = this.generatePrimaryKey(docId);

    try {
      const { body } = await this.client.get({ index: this.name, id });
      return body ? body._source : null;
    } catch {
      return null;
    }
  }

  async _update(docId, updates) {
    const doc = await this._get(docId);
    if (!doc) {
      return null;
    }

    [].concat(this.docId).forEach((id) => {
      delete updates[id];
    });

    try {
      const id = this.generatePrimaryKey(docId);
      const { body } = await this.client.update({
        index: this.name,
        id,
        retry_on_conflict: 5,
        // partial doc merge is supported in opensearch, so we just pass the updates here
        // @link https://www.elastic.co/guide/en/elasticsearch/reference/7.16/docs-update.html#update-api-desc
        body: { doc: { ...updates } },
      });
      debug(`update ${this.name}`, { docId: id, result: body.result });
    } catch (err) {
      console.warn(`failed to update ${this.name}`, formatEsResponseError(err));
      throw err;
    }

    return Object.assign(doc, updates);
  }

  async _reset() {
    if (process.env.NODE_ENV !== 'production') {
      const { name, http } = this;
      const endpoint = `/${name}`;
      await http.delete(endpoint);
      await this.ensureIndex(this.indexParams);
    }
  }

  /**
   * Pad balance fields to fixed length string
   * This only exist because elasticsearch sorting issue
   *
   * @param {string} balance which number to be padded
   * @param {number} length how long is the padded string
   * @memberof ESIndex
   * @return string
   */
  static padBalance(balance, length) {
    return new BN(balance).toString(10, length);
  }

  static trimBalance(balance) {
    return new BN(balance).toString(10);
  }
}

module.exports = ESIndex;