@sugarcube/plugin-elasticsearch/_dist/elastic.js

Use [Elasticsearch](https://www.elastic.co/products/elasticsearch) for SugarCube data.

"use strict";

Object.defineProperty(exports, "__esModule", {
  value: true
});
exports.Elastic = exports.queryExisting = exports.queryOne = exports.queryByIds = exports.bulk = exports.query = exports.reindex = exports.toMsg = exports.toHeader = exports.connect = void 0;

var _fp = require("lodash/fp");

var _dashp = require("dashp");

var _es = require("es6");

var _es2 = require("es7");

var _nodeFetch = _interopRequireDefault(require("node-fetch"));

var _core = require("@sugarcube/core");

var _utils = require("./utils");

var _mappings = _interopRequireDefault(require("./mappings"));

var _queries = _interopRequireDefault(require("./queries"));

function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }

/* eslint-disable no-plusplus */
const {
  curry2,
  curry3,
  curry4,
  curry6
} = _core.utils;

const esVersion = async node => {
  const {
    version
  } = await (0, _nodeFetch.default)(node).then(resp => resp.json());
  return parseInt(version.number[0], 10);
};

const mergeMappings = async (client, customMappings = {}) => {
  const {
    body
  } = await client.info();
  const version = parseInt(body.version.number[0], 10);

  switch (version) {
    case 6:
      {
        return Object.assign(_mappings.default, customMappings);
      }

    case 7:
      {
        const {
          mappings,
          ...rest
        } = _mappings.default;
        return Object.assign(rest, {
          mappings: mappings._doc
        }, customMappings);
      }

    default:
      {
        throw new Error(`Elasticsearch version ${version} not supported.`);
      }
  }
}; // ES6 returns total as number, ES7 as an object with a value sttribute.


const normalizeTotal = body => {
  return (0, _fp.isPlainObject)(body.hits.total) ? body.hits.total.value : body.hits.total;
};

const connect = async node => {
  let Client;
  const version = await esVersion(node);

  switch (version) {
    case 6:
      {
        Client = _es.Client;
        break;
      }

    case 7:
      {
        Client = _es2.Client;
        break;
      }

    default:
      {
        throw new Error(`Elasticsearch version ${version} not supported.`);
      }
  }

  return new Client({
    node,
    requestTimeout: 60 * 1000
  });
};

exports.connect = connect;

const toHeader = (index, unit) => ({
  _index: index,
  _id: unit._sc_id_hash
});

exports.toHeader = toHeader;

const toMsg = (index, unit) => (0, _utils.stripUnderscores)((toHeader(index, unit), {
  body: unit
}));

exports.toMsg = toMsg;
const createIndex = curry3("createIndex", async (index, mapping, client) => {
  const body = await mergeMappings(client, mapping);
  const {
    body: aliasExists
  } = await client.indices.existsAlias({
    name: index
  });
  const {
    body: indexExists
  } = await client.indices.exists({
    index
  });
  if (aliasExists || indexExists) return (0, _dashp.ofP)(null);
  await client.indices.create({
    index: `${index}-1`,
    body
  });
  return client.indices.putAlias({
    index: `${index}-1`,
    name: index
  });
});
/*
 * Public API
 */

const reindex = curry6("reindex", async (index, host, port, toIndex, client, customMappings) => {
  await createIndex(toIndex, customMappings, client);
  await client.reindex({
    body: _queries.default.reindex(index, host, port, toIndex),
    refresh: true,
    waitForCompletion: false
  });
  return [null, {}];
});
exports.reindex = reindex;
const query = curry4("query", async (index, reqBody, amount, client, customMappings) => {
  await createIndex(index, customMappings, client);
  const allData = [];
  let meta = {
    took: 0,
    total: 0
  };
  const responseQueue = [];
  responseQueue.push(await client.search({
    index,
    body: reqBody,
    size: 2000,
    scroll: "60s"
  }));

  while (responseQueue.length) {
    const {
      body
    } = responseQueue.shift(); // ES7 changed the return value of body.hits.total

    const total = normalizeTotal(body);

    while (body.hits.hits.length > 0) {
      const unit = body.hits.hits.shift();
      allData.push(Object.assign((0, _utils.unstripify)(unit._source), {
        _sc_elastic_score: unit.score
      }, unit.highlight != null ? {
        _sc_elastic_highlights: (0, _utils.unstripify)(unit.highlight)
      } : {}));
    }

    meta = Object.assign({}, meta, body.timed_out ? {
      timedOut: true
    } : {}, {
      took: meta.took + body.took,
      total
    });

    if (amount != null && allData.length >= amount || total === allData.length) {
      break;
    }

    responseQueue.push( // eslint-disable-next-line no-await-in-loop
    await client.scroll({
      scrollId: body._scroll_id,
      scroll: "30s"
    }));
  }

  return [allData, meta];
});
exports.query = query;
const bulk = curry4("bulk", async (index, ops, client, customMappings) => {
  const batchSize = 500;
  const toIndex = [];
  const toUpdate = [];

  for (let i = 0; i < (ops.index || []).length; i++) {
    const unit = ops.index[i];
    toIndex.push({
      index: toHeader(index, unit)
    });
    toIndex.push((0, _utils.stripUnderscores)(unit));
  }

  for (let i = 0; i < (ops.update || []).length; i++) {
    const unit = ops.update[i];
    toUpdate.push({
      update: toHeader(index, unit)
    });
    toUpdate.push({
      doc: (0, _utils.stripUnderscores)(unit)
    });
  } // Ensure the index is created.


  await createIndex(index, customMappings, client); // Run the bulk requests

  const responseQueue = [];
  const errors = [];
  let meta = {
    took: 0,
    batches: 0,
    batchSize
  };
  const [firstChunk, ...chunks] = (0, _fp.chunk)(batchSize, toIndex.concat(toUpdate));
  responseQueue.push(await client.bulk({
    index,
    body: firstChunk,
    type: "_doc",
    refresh: true
  }));

  while (responseQueue.length) {
    const {
      body
    } = responseQueue.shift();
    const {
      took,
      items
    } = body;

    for (let i = 0; i < (items || []).length; i++) {
      const item = items[i];
      const op = item.index || item.update || item.create || item.delete;
      const result = op.result || "error";
      const count = meta[result] || 0;
      if (op.error != null) errors.push({
        id: op._id,
        error: `${op.error.type}: ${op.error.reason} (${op.error.caused_by != null ? op.error.caused_by.reason : ""})`
      });
      meta = Object.assign({}, meta, {
        took: meta.took + took,
        [result]: count + 1
      });
    }

    if (chunks.length === 0) {
      break;
    }

    const nextChunk = chunks.shift();
    responseQueue.push( // eslint-disable-next-line no-await-in-loop
    await client.bulk({
      index,
      body: nextChunk,
      type: "_doc",
      refresh: true
    }));
  }

  return [errors, meta];
});
exports.bulk = bulk;
const queryByIds = curry3("queryByIds", async (index, ids, client, customMappings) => {
  const batchSize = 2500;
  const responses = await (0, _dashp.collectP)(idsChunk => {
    const reqBody = _queries.default.byIds(idsChunk);

    return query(index, reqBody, null, client, customMappings);
  }, (0, _fp.chunk)(batchSize, ids));
  return responses.reduce(([data, meta], response) => {
    const {
      took,
      total
    } = response[1];
    return [data.concat(response[0]), Object.assign({}, meta, {
      took: took + meta.took,
      total: total + meta.total
    })];
  }, [[], {
    took: 0,
    total: 0,
    batches: responses.length,
    batchSize
  }]);
});
exports.queryByIds = queryByIds;
const queryOne = curry3("queryOne", async (index, id, client, customMappings) => {
  // Ensure the index is created.
  await createIndex(index, customMappings, client);
  const {
    _version: version,
    _type: type,
    _source: data
  } = await client.get({
    index,
    id,
    type: "_all"
  }).body;
  return [data, {
    version,
    type
  }];
});
exports.queryOne = queryOne;
const queryExisting = curry3("queryExisting", async (index, ids, client, customMappings) => {
  const batchSize = 5000;
  await createIndex(index, customMappings, client);
  const responses = await (0, _dashp.collectP)(idsChunk => {
    const reqBody = _queries.default.existing(idsChunk);

    return query(index, reqBody, null, client, customMappings);
  }, (0, _fp.chunk)(batchSize, ids));
  return responses.reduce(([data, meta], response) => {
    const {
      took,
      total
    } = response[1];
    return [data.concat(response[0].map(unit => unit._sc_id_hash)), Object.assign({}, meta, {
      took: took + meta.took,
      total: total + meta.total
    })];
  }, [[], {
    took: 0,
    total: 0,
    batches: responses.length,
    batchSize
  }]);
});
exports.queryExisting = queryExisting;
const Elastic = {
  Do: curry2("ElasticDo", async (G, {
    host,
    port,
    mappings
  }) => {
    const node = `http://${host}:${port}`;
    const client = await connect(node);
    const api = {
      bulk,
      query,
      queryByIds,
      queryOne,
      queryExisting,
      reindex
    };
    const customMappings = (0, _utils.stripUnderscores)(mappings || {});
    const generator = G(api);
    let data;
    let history = [];

    const chain = async nextG => {
      const {
        done,
        value
      } = await nextG.next(data);
      if (done) return (0, _dashp.ofP)([value || data, history]); // All curried function names have the format of <name>-<int> where
      // <int> is the number of missing arguments. For a prettier output in
      // the history strip -<int> from the name.

      const prettyName = value.name.replace(/-.*$/, "");
      let result;
      let meta;

      try {
        [result, meta] = await value(client, customMappings);
      } catch (e) {
        const error = JSON.stringify({
          status: (0, _fp.get)("meta.statusCode", e),
          body: (0, _fp.get)("meta.meta.request.params.body", e),
          href: (0, _fp.get)("meta.meta.request.params.url.href", e),
          type: (0, _fp.get)("meta.body.error.type", e),
          reason: (0, _fp.get)("meta.body.error.reason", e),
          line: (0, _fp.get)("meta.body.error.line", e),
          col: (0, _fp.get)("meta.body.error.col", e),
          method: (0, _fp.get)("meta.meta.request.params.method", e),
          path: (0, _fp.get)("meta.meta.request.params.path", e),
          qs: (0, _fp.get)("meta.meta.request.params.querystring", e)
        });
        e.message = `\`${prettyName}\` in \`${G.name}\`: ${error}`;
        throw e;
      }

      history = history.concat([[prettyName, meta]]);
      data = result;
      return chain(nextG);
    };

    return (0, _dashp.ofP)(chain(generator));
  })
};
exports.Elastic = Elastic;