@ocap/indexdb-elasticsearch/lib/db.js

OCAP indexdb adapter that uses elasticsearch as backend

const axios = require('axios');
const { BaseIndexDB } = require('@ocap/indexdb');
const { Client: ESClient } = require('@elastic/elasticsearch');
const { formatPagination } = require('@ocap/indexdb/lib/util');
const { formatTxAfterRead, formatDelegationAfterRead } = require('@ocap/indexdb/lib/util');
const { DEFAULT_TOKEN_DECIMAL } = require('@ocap/util/lib/constant');
const { toChecksumAddress } = require('@arcblock/did/lib/type');

const debug = require('debug')(require('../package.json').name);

const Account = require('./table/account'); // 这里命名为 Table 是为了保持命名一致, 不过我感觉 Table 表达的不准确
const Asset = require('./table/asset');
const Factory = require('./table/factory');
const Tx = require('./table/tx');
const Delegation = require('./table/delegation');
const Token = require('./table/token');
const Stake = require('./table/stake');
const Rollup = require('./table/rollup');
const RollupBlock = require('./table/rollup-block');
const RollupValidator = require('./table/rollup-validator');
const TokenDistribution = require('./table/token-distribution');
const TokenFactory = require('./table/token-factory');

const { name, version } = require('../package.json');
const { formatQueryResult, getAuthHeaders, getTimeCondition } = require('./util');

// https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-match-all-query.html
const FALLBACK_QUERY = { match_all: { boost: 1.0 } };
const REQUEST_TIMEOUT = 8000;
const MAX_REQUEST_FACTORY_ADDRESS_SIZE = 100;
const TRACK_TOTAL_HITS = +process.env.TRACK_TOTAL_HITS || 10000;

const getMigratedFrom = (account) => {
  if (account.migratedFrom) {
    return Array.isArray(account.migratedFrom) ? account.migratedFrom[0] : account.migratedFrom;
  }
  return '';
};

class ESIndexDB extends BaseIndexDB {
  /**
   * Creates an instance of ESIndexDB.
   * @param {object} config
   * @param {string} config.endpoint The elasticsearch endpoint
   * @param {string} config.prefix Elasticsearch index name prefix
   * @param {number} config.tokenLength How elasticsearch index account balance before indexing
   * @param {number} config.requestTimeout timeout in milliseconds for each request
   * @param {object} config.auth authentication config, https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/client-connecting.html#authentication
   * @memberof ESIndexDB
   */
  constructor(config) {
    super();

    if (!config.endpoint) {
      throw new Error('Elasticsearch indexdb requires config.endpoint to work');
    }

    if (!config.tokenLength) {
      throw new Error('Elasticsearch indexdb requires config.tokenLength to work');
    }

    if (config.prefix && typeof config.prefix !== 'string') {
      throw new Error('Elasticsearch indexdb requires config.prefix to be a string');
    }

    if (typeof config.tokenLength !== 'number' || config.tokenLength < 0) {
      throw new Error('Elasticsearch indexdb requires config.tokenLength to a number greater than 0');
    }

    this.config = config;

    this.name = name;
    this.version = version;

    this.http = axios.create({
      baseURL: config.endpoint,
      headers: getAuthHeaders(config.auth),
      timeout: config.requestTimeout || REQUEST_TIMEOUT,
    });

    this.client = new ESClient({
      node: config.endpoint,
      auth: config.auth,
      requestTimeout: config.requestTimeout || REQUEST_TIMEOUT,
    });

    this.account = new Account(this);
    this.asset = new Asset(this);
    this.factory = new Factory(this);
    this.tx = new Tx(this);
    this.delegation = new Delegation(this);
    this.token = new Token(this);
    this.stake = new Stake(this);
    this.rollup = new Rollup(this);
    this.rollupBlock = new RollupBlock(this);
    this.rollupValidator = new RollupValidator(this);
    this.tokenDistribution = new TokenDistribution(this);
    this.tokenFactory = new TokenFactory(this);

    this.attachReadyListeners();
  }

  async listTransactions({
    addressFilter = {},
    paging = {},
    timeFilter = {},
    typeFilter = {},
    assetFilter = {},
    factoryFilter = {},
    tokenFilter = {},
    accountFilter = {},
    validityFilter = {},
    txFilter = {},
    rollupFilter = {},
    stakeFilter = {},
    delegationFilter = {},
    tokenFactoryFilter = {},
  } = {}) {
    const pagination = formatPagination({ paging, defaultSortField: 'time' });

    const { sender, receiver, direction = 'UNION' } = addressFilter;
    const { types = [] } = typeFilter;
    const { factories = [] } = factoryFilter;
    const { assets = [] } = assetFilter;
    const { tokens = [] } = tokenFilter;
    const { accounts = [] } = accountFilter;
    const { txs = [] } = txFilter;
    const { rollups = [] } = rollupFilter;
    const { stakes = [] } = stakeFilter;
    const { delegations = [] } = delegationFilter;
    const { tokenFactories = [] } = tokenFactoryFilter;

    // OR is spelled should
    // AND is spelled must
    // NOR is spelled should_not
    const conditions = [];

    if (sender && receiver) {
      // 两个人之间的双向交易
      if (direction === 'MUTUAL') {
        conditions.push({
          bool: {
            should: [
              { bool: { must: [{ term: { sender } }, { term: { receiver } }] } },
              { bool: { must: [{ term: { sender: receiver } }, { term: { receiver: sender } }] } },
            ],
          },
        });
      } else if (direction === 'ONE_WAY') {
        // 两人之间的单向交易
        conditions.push({ term: { sender } }, { term: { receiver } });
      } else if (direction === 'UNION') {
        // 查询某个人收发的交易
        conditions.push({
          bool: {
            should: [{ term: { sender } }, { term: { receiver } }],
          },
        });
      }
    } else if (sender) {
      // 某个人发的交易
      conditions.push({ term: { sender } });
    } else if (receiver) {
      // 某个人收的交易
      conditions.push({ term: { receiver } });
    }

    if (types.length) {
      conditions.push({ terms: { type: types } });
    }

    if (assets.length) {
      conditions.push({ terms: { assets } });
    }

    if (factories.length) {
      conditions.push({ terms: { factories } });
    }

    if (tokens.length) {
      conditions.push({ terms: { tokens } });
    }

    if (accounts.length) {
      conditions.push({ terms: { accounts } });
    }

    if (txs.length) {
      conditions.push({ terms: { hash: txs } });
    }

    if (rollups.length) {
      conditions.push({ terms: { rollups } });
    }

    if (stakes.length) {
      conditions.push({ terms: { stakes } });
    }

    if (delegations.length) {
      conditions.push({ terms: { delegations } });
    }

    if (tokenFactories.length) {
      conditions.push({ terms: { tokenFactories } });
    }

    const condition = getTimeCondition(timeFilter, ['time'], 'time');
    if (condition) {
      conditions.push(condition);
    }

    if (validityFilter.validity && validityFilter.validity === 'VALID') {
      conditions.push({ term: { valid: true } });
    }
    if (validityFilter.validity && validityFilter.validity === 'INVALID') {
      conditions.push({ term: { valid: false } });
    }

    debug('listTransactions query conditions', conditions);

    const result = await this.tx.search({
      from: pagination.cursor,
      size: pagination.size,
      body: {
        track_total_hits: TRACK_TOTAL_HITS,
        query: conditions.length ? { bool: { must: conditions } } : FALLBACK_QUERY,
        sort: [{ [pagination.order.field]: { order: pagination.order.type } }],
        _source: { exclude: ['tx.itxJson.data', 'tx.itxJson.encoded_value'] }, // required to make api fast
      },
    });

    const { items, paging: nextPaging } = formatQueryResult(result, pagination);
    return { transactions: items.map((item) => formatTxAfterRead(item)), paging: nextPaging };
  }

  async getRelatedAddresses(address) {
    let account = await this.account.get(address);
    if (!account) {
      return [];
    }

    const related = [address];
    while (account && getMigratedFrom(account) && related.length <= 8) {
      related.push(getMigratedFrom(account));
      // eslint-disable-next-line no-await-in-loop
      account = await this.account.get(getMigratedFrom(account));
    }

    return related.filter(Boolean);
  }

  async listAssets({ ownerAddress, factoryAddress, paging, timeFilter = {} } = {}) {
    const conditions = [];
    if (ownerAddress) {
      const possibleOwners = await this.getRelatedAddresses(ownerAddress);
      if (possibleOwners.length) {
        conditions.push({ terms: { owner: possibleOwners } });
      } else {
        // If we dit not find any owner state, just return empty
        return { assets: [], account: null };
      }
    }
    if (factoryAddress) {
      conditions.push({ term: { parent: factoryAddress } });
    }
    const condition = getTimeCondition(timeFilter, ['genesisTime', 'renaissanceTime', 'consumedTime']);
    if (condition) {
      conditions.push(condition);
    }

    if (conditions.length === 0) {
      return { assets: [], account: null };
    }

    debug('listAssets query conditions', conditions);
    const pagination = formatPagination({ paging, defaultSortField: 'renaissanceTime' });

    const [result, account] = await Promise.all([
      this.asset.search({
        from: pagination.cursor,
        size: pagination.size,
        body: {
          track_total_hits: TRACK_TOTAL_HITS,
          query: { bool: { must: conditions } },
          sort: [{ [pagination.order.field]: { order: pagination.order.type } }],
        },
      }),
      ownerAddress ? this.account.get(ownerAddress) : null,
    ]);

    const { items, paging: nextPaging } = formatQueryResult(result, pagination);
    return { assets: items, account, paging: nextPaging };
  }

  async listTopAccounts({ tokenAddress, paging, timeFilter = {} } = {}) {
    const pagination = formatPagination({
      paging,
      defaultSortField: 'balance',
      supportedSortFields: ['genesisTime', 'renaissanceTime', 'balance'],
    });

    let sort = null;
    if (pagination.order.field === 'balance') {
      if (!tokenAddress) {
        throw new Error('tokenAddress is required when order.field is balance');
      }

      sort = [
        {
          'tokens.balance': {
            mode: 'max',
            order: pagination.order.type,
            nested: {
              path: 'tokens',
              filter: { term: { 'tokens.address': tokenAddress } },
            },
          },
        },
      ];
    } else {
      sort = [
        {
          [pagination.order.field]: {
            order: pagination.order.type,
          },
        },
      ];
    }

    const conditions = [];
    const condition = getTimeCondition(timeFilter, ['genesisTime', 'renaissanceTime']);
    if (condition) {
      conditions.push(condition);
    }
    if (tokenAddress) {
      conditions.push({
        nested: {
          path: 'tokens',
          query: {
            bool: {
              must: [{ term: { 'tokens.address': tokenAddress } }],
            },
          },
        },
      });
    }

    debug('listTopAccounts', { tokenAddress, paging, sort });

    const result = await this.account.search({
      from: pagination.cursor,
      size: pagination.size,
      body: {
        track_total_hits: TRACK_TOTAL_HITS,
        query: conditions.length ? { bool: { must: conditions } } : FALLBACK_QUERY,
        sort,
      },
    });

    const { items, paging: nextPaging } = formatQueryResult(result, pagination);

    items.forEach((account) => {
      if (Array.isArray(account.tokens) && account.tokens.length) {
        account.tokens = account.tokens.map((token) => {
          token.decimal = typeof token.decimal === 'undefined' ? DEFAULT_TOKEN_DECIMAL : token.decimal;
          return token;
        });
      }
    });

    return {
      accounts: await Promise.all(items.map((x) => Account.formatAfterRead(x))),
      paging: nextPaging,
    };
  }

  async listTokens({ issuerAddress, paging, timeFilter = {} } = {}) {
    debug('listTokens', { issuerAddress, paging });

    const conditions = [];

    if (issuerAddress) {
      conditions.push({ term: { issuer: issuerAddress } });
    }

    const condition = getTimeCondition(timeFilter, ['genesisTime', 'renaissanceTime']);
    if (condition) {
      conditions.push(condition);
    }

    const pagination = formatPagination({
      paging,
      defaultSortField: 'renaissanceTime',
      supportedSortFields: ['genesisTime', 'renaissanceTime', 'issuer', 'symbol', 'totalSupply'],
    });

    const result = await this.token.search({
      from: pagination.cursor,
      size: pagination.size,
      body: {
        track_total_hits: TRACK_TOTAL_HITS,
        query: conditions.length ? { bool: { must: conditions } } : FALLBACK_QUERY,
        sort: [{ [pagination.order.field]: { order: pagination.order.type } }],
      },
    });

    const { items, paging: nextPaging } = formatQueryResult(result, pagination);
    return { tokens: items.map((x) => Token.formatAfterRead(x)), paging: nextPaging };
  }

  async listTokenFactories({ owner, reserveAddress, tokenAddress, paging, timeFilter = {} } = {}) {
    debug('listTokenFactories', { owner, reserveAddress, tokenAddress, paging });

    const conditions = [];

    if (owner) {
      conditions.push({ term: { owner } });
    }

    if (reserveAddress) {
      conditions.push({ term: { reserveAddress } });
    }

    if (tokenAddress) {
      conditions.push({ term: { tokenAddress } });
    }

    const condition = getTimeCondition(timeFilter, ['genesisTime', 'renaissanceTime']);
    if (condition) {
      conditions.push(condition);
    }

    const pagination = formatPagination({
      paging,
      defaultSortField: 'renaissanceTime',
      supportedSortFields: ['genesisTime', 'renaissanceTime', 'reserveBalance', 'currentSupply'],
    });

    const result = await this.tokenFactory.search({
      from: pagination.cursor,
      size: pagination.size,
      body: {
        track_total_hits: TRACK_TOTAL_HITS,
        query: conditions.length ? { bool: { must: conditions } } : FALLBACK_QUERY,
        sort: [{ [pagination.order.field]: { order: pagination.order.type } }],
      },
    });

    const { items, paging: nextPaging } = formatQueryResult(result, pagination);
    return { tokenFactories: items.map((x) => TokenFactory.formatAfterRead(x)), paging: nextPaging };
  }

  async listFactories({ ownerAddress, addressList, paging, timeFilter = {} } = {}) {
    debug('listTokens', { ownerAddress, paging });
    const pagination = formatPagination({
      paging,
      defaultSortField: 'renaissanceTime',
      supportedSortFields: ['genesisTime', 'renaissanceTime', 'owner', 'numMinted', 'limit', 'name'],
    });

    const conditions = [];
    if (ownerAddress) {
      conditions.push({ term: { owner: ownerAddress } });
    }

    const condition = getTimeCondition(timeFilter, ['genesisTime', 'renaissanceTime']);
    if (condition) {
      conditions.push(condition);
    }

    if (Array.isArray(addressList) && addressList.length > 0) {
      if (addressList.length > MAX_REQUEST_FACTORY_ADDRESS_SIZE) {
        throw new Error(`The length of 'addressList' cannot exceed the length of ${MAX_REQUEST_FACTORY_ADDRESS_SIZE}`);
      }

      conditions.push({ terms: { address: addressList } });
    }

    const sortFields = {
      name: 'name.raw',
    };

    const assetsResult = await this.factory.search({
      from: pagination.cursor,
      size: pagination.size,
      body: {
        track_total_hits: TRACK_TOTAL_HITS,
        query: conditions.length > 0 ? { bool: { must: conditions } } : FALLBACK_QUERY,
        sort: [{ [sortFields[pagination.order.field] || pagination.order.field]: { order: pagination.order.type } }],
      },
    });

    const { items, paging: nextPaging } = formatQueryResult(assetsResult, pagination);
    return { factories: items.map((x) => Factory.formatAfterRead(x)), paging: nextPaging };
  }

  async listStakes({ paging = {}, addressFilter = {}, timeFilter = {}, assetFilter = {} } = {}) {
    const pagination = formatPagination({
      paging,
      defaultSortField: 'renaissanceTime',
      supportedSortFields: ['genesisTime', 'renaissanceTime'],
    });

    const { sender, receiver } = addressFilter;
    const { assets = [] } = assetFilter;

    // OR is spelled should
    // AND is spelled must
    // NOR is spelled should_not
    const conditions = [];

    if (sender && receiver) {
      conditions.push({
        bool: {
          should: [
            { bool: { must: [{ term: { sender } }, { term: { receiver } }] } },
            { bool: { must: [{ term: { sender: receiver } }, { term: { receiver: sender } }] } },
          ],
        },
      });
    } else if (sender) {
      conditions.push({ term: { sender } });
    } else if (receiver) {
      conditions.push({ term: { receiver } });
    }

    if (assets.length) {
      conditions.push({ terms: { assets } });
    }

    const condition = getTimeCondition(timeFilter, ['genesisTime', 'renaissanceTime']);
    if (condition) {
      conditions.push(condition);
    }

    debug('listStakes query conditions', conditions);

    const result = await this.stake.search({
      from: pagination.cursor,
      size: pagination.size,
      body: {
        track_total_hits: TRACK_TOTAL_HITS,
        query: conditions.length ? { bool: { must: conditions } } : FALLBACK_QUERY,
        sort: [{ [pagination.order.field]: { order: pagination.order.type } }],
        _source: { exclude: [] }, // to make api fast
      },
    });

    const { items, paging: nextPaging } = formatQueryResult(result, pagination);
    return { stakes: items.map((item) => Stake.formatAfterRead(item)), paging: nextPaging };
  }

  async listRollups({
    paging,
    tokenAddress = '',
    erc20TokenAddress = '',
    foreignTokenAddress = '',
    timeFilter = {},
  } = {}) {
    const pagination = formatPagination({
      paging,
      defaultSortField: 'renaissanceTime',
      supportedSortFields: ['genesisTime', 'renaissanceTime'],
    });

    const conditions = [];
    if (tokenAddress) {
      conditions.push({ term: { tokenAddress } });
    }
    const condition = getTimeCondition(timeFilter, ['genesisTime', 'renaissanceTime']);
    if (condition) {
      conditions.push(condition);
    }
    const foreignTokenAddr = foreignTokenAddress || erc20TokenAddress;
    if (foreignTokenAddr) {
      conditions.push({
        nested: {
          path: 'foreignToken',
          query: {
            bool: {
              should: [
                { match: { 'foreignToken.contractAddress': toChecksumAddress(foreignTokenAddr) } },
                { match: { 'foreignToken.contractAddress': foreignTokenAddr.toLowerCase() } },
              ],
            },
          },
        },
      });
    }

    debug('listRollups', { paging, pagination, conditions });

    const result = await this.rollup.search({
      from: pagination.cursor,
      size: pagination.size,
      body: {
        track_total_hits: TRACK_TOTAL_HITS,
        query: conditions.length ? { bool: { must: conditions } } : FALLBACK_QUERY,
        sort: [{ [pagination.order.field]: { order: pagination.order.type } }],
      },
    });

    const { items, paging: nextPaging } = formatQueryResult(result, pagination);
    return { rollups: items, paging: nextPaging };
  }

  async listRollupBlocks({
    paging = {},
    rollupAddress = '',
    tokenAddress = '',
    proposer = '',
    height = '',
    timeFilter = {},
    txFilter = {},
    validatorFilter = {},
  } = {}) {
    const pagination = formatPagination({ paging, defaultSortField: 'genesisTime' });

    const { txs = [] } = txFilter;
    const { validators = [] } = validatorFilter;

    const conditions = [];

    if (rollupAddress) {
      conditions.push({ term: { rollup: rollupAddress } });
    }

    if (Number(height) > 0) {
      conditions.push({ term: { height: Number(height) } });
    }

    if (proposer) {
      conditions.push({ term: { proposer } });
    }

    if (tokenAddress) {
      conditions.push({
        nested: {
          path: 'tokenInfo',
          query: { bool: { must: { term: { 'tokenInfo.address': tokenAddress } } } },
        },
      });
    }

    if (txs.length) {
      conditions.push({ terms: { txs } });
    }

    if (validators.length) {
      conditions.push({ terms: { validators } });
    }

    const condition = getTimeCondition(timeFilter, ['genesisTime', 'renaissanceTime']);
    if (condition) {
      conditions.push(condition);
    }

    debug('listRollupBlocks query conditions', conditions);

    const result = await this.rollupBlock.search({
      from: pagination.cursor,
      size: pagination.size,
      body: {
        track_total_hits: TRACK_TOTAL_HITS,
        query: conditions.length ? { bool: { must: conditions } } : FALLBACK_QUERY,
        sort: [{ [pagination.order.field]: { order: pagination.order.type } }],
      },
    });

    const { items, paging: nextPaging } = formatQueryResult(result, pagination);
    return { blocks: items, paging: nextPaging };
  }

  async listRollupValidators({ paging = {}, rollupAddress = '' } = {}) {
    const pagination = formatPagination({
      paging,
      defaultSortField: 'genesisTime',
      supportedSortFields: ['joinTime', 'leaveTime', 'genesisTime', 'renaissanceTime'],
    });

    const conditions = [];

    if (rollupAddress) {
      conditions.push({ term: { rollup: rollupAddress } });
    }

    debug('listRollupValidators query conditions', conditions);

    const result = await this.rollupValidator.search({
      from: pagination.cursor,
      size: pagination.size,
      body: {
        track_total_hits: TRACK_TOTAL_HITS,
        query: conditions.length ? { bool: { must: conditions } } : FALLBACK_QUERY,
        sort: [{ [pagination.order.field]: { order: pagination.order.type } }],
      },
    });

    const { items, paging: nextPaging } = formatQueryResult(result, pagination);
    return { validators: items.map((x) => RollupValidator.formatAfterRead(x)), paging: nextPaging };
  }

  async listDelegations({ from, to, paging, timeFilter = {} } = {}) {
    debug('listDelegations', { from, to, paging, timeFilter });
    if (!from && !to) {
      return { delegations: [], paging: { cursor: 0, next: false, total: 0 } };
    }

    const conditions = [];

    if (from) {
      conditions.push({ term: { from } });
    }
    if (to) {
      conditions.push({ term: { to } });
    }

    const condition = getTimeCondition(timeFilter, ['genesisTime', 'renaissanceTime']);
    if (condition) {
      conditions.push(condition);
    }

    const pagination = formatPagination({
      paging,
      defaultSortField: 'renaissanceTime',
      supportedSortFields: ['genesisTime', 'renaissanceTime'],
    });

    const result = await this.delegation.search({
      from: pagination.cursor,
      size: pagination.size,
      body: {
        track_total_hits: TRACK_TOTAL_HITS,
        query: conditions.length ? { bool: { must: conditions } } : FALLBACK_QUERY,
        sort: [{ [pagination.order.field]: { order: pagination.order.type } }],
      },
    });

    const { items, paging: nextPaging } = formatQueryResult(result, pagination);
    return { delegations: items.map(formatDelegationAfterRead), paging: nextPaging };
  }
}

module.exports = ESIndexDB;