@sugarcube/plugin-elasticsearch/_dist/plugins/export.js

Use [Elasticsearch](https://www.elastic.co/products/elasticsearch) for SugarCube data.

"use strict";

Object.defineProperty(exports, "__esModule", {
  value: true
});
exports.default = void 0;

var _fp = require("lodash/fp");

var _fs = _interopRequireDefault(require("fs"));

var _core = require("@sugarcube/core");

var _elastic = require("../elastic");

var _utils = require("../utils");

function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }

const {
  sToA
} = _core.utils;

const plugin = (envelope, {
  cfg,
  log,
  stats
}) => {
  const host = (0, _fp.get)("elastic.host", cfg);
  const port = (0, _fp.get)("elastic.port", cfg);
  const index = (0, _fp.get)("elastic.index", cfg);
  const omitFields = sToA(",", (0, _fp.get)("elastic.omit_fields", cfg));
  const mappings = (0, _fp.get)("elastic.mappings", cfg) ? JSON.parse(_fs.default.readFileSync((0, _fp.get)("elastic.mappings", cfg))) : {};
  if (envelope.data.length === 0) return envelope;
  log.info(`Using ${host}:${port}/${index}.`);
  return _elastic.Elastic.Do(function* indexUnits({
    bulk,
    queryExisting
  }) {
    const ids = envelope.data.map(u => u._sc_id_hash);
    const existingIds = yield queryExisting(index, ids);

    const dataToIndex = _core.envelope.filterData(u => !existingIds.includes(u._sc_id_hash), envelope);

    const dataToUpdate = _core.envelope.filterData(u => existingIds.includes(u._sc_id_hash), envelope);

    const toIndex = (0, _utils.omitFromData)(omitFields, dataToIndex.data);
    const toUpdate = (0, _utils.omitFromData)(omitFields, dataToUpdate.data);
    const sourceStatsNew = dataToIndex.data.reduce((memo, {
      _sc_source: source
    }) => Object.assign(memo, {
      [source]: (0, _fp.getOr)(0, source, memo) + 1
    }), {});
    const sourceStatsExisting = dataToUpdate.data.reduce((memo, {
      _sc_source: source
    }) => Object.assign(memo, {
      [source]: (0, _fp.getOr)(0, source, memo) + 1
    }), {});
    stats.count("total", ids.length);
    stats.count("new", toIndex.length);
    Object.entries(sourceStatsNew).forEach(([source, count]) => stats.count(`${source}_new`, count));
    stats.count("existing", toUpdate.length);
    Object.entries(sourceStatsExisting).forEach(([source, count]) => stats.count(`${source}_existing`, count));
    log.info(`Indexing ${(0, _fp.size)(toIndex)} units.`);
    if ((0, _fp.size)(toIndex) > 0) log.debug(`  counts new: ${Object.entries(sourceStatsNew).map(([source, count]) => `${source}=${count}`).join(", ")}`);
    log.info(`Updating ${(0, _fp.size)(toUpdate)} units.`);
    if ((0, _fp.size)(toUpdate) > 0) log.debug(`  counts existing: ${Object.entries(sourceStatsExisting).map(([source, count]) => `${source}=${count}`).join(", ")}`);
    const errors = yield bulk(index, {
      index: toIndex,
      update: toUpdate
    });

    if ((0, _fp.size)(errors) > 0) {
      errors.forEach(e => stats.fail({
        type: "any_unit",
        term: e.id,
        reason: e.error
      }));
    }
  }, {
    host,
    port,
    mappings
  }).then(([, history]) => {
    history.forEach(([k, meta]) => log.debug(`${k}: ${JSON.stringify(meta)}.`));
    return envelope;
  });
};

plugin.desc = "Store data in Elasticsearch.";
plugin.argv = {
  "elastic.omit_fields": {
    type: "string",
    nargs: 1,
    desc: "Omit those fields when exporting."
  },
  "elastic.mappings": {
    type: "string",
    nargs: 1,
    desc: "Load custom index mappings from a JSON file."
  }
};
var _default = plugin;
exports.default = _default;