@sugarcube/plugin-http/_dist/plugins/get.js

HTTP related plugins for sugarcube.

"use strict";

Object.defineProperty(exports, "__esModule", {
  value: true
});
exports.default = void 0;

var _fp = require("lodash/fp");

var _dashp = require("dashp");

var _pify = _interopRequireDefault(require("pify"));

var _fs = _interopRequireDefault(require("fs"));

var _url = _interopRequireDefault(require("url"));

var _path = require("path");

var _core = require("@sugarcube/core");

var _pluginFs = require("@sugarcube/plugin-fs");

var _utils = require("../utils");

function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }

const {
  sToA
} = _core.utils;
const accessAsync = (0, _pify.default)(_fs.default.access);
const unlinkAsync = (0, _pify.default)(_fs.default.unlink);

const cleanUp = async location => {
  try {
    await accessAsync(location);
    await unlinkAsync(location); // eslint-disable-next-line no-empty
  } catch (e) {}
};

const downloadExists = async location => {
  try {
    await accessAsync(location);
  } catch (e) {
    // telegram_channels use filenames that throw a ENAMETOOLONG.
    if (e.code === "ENOENT" || e.code === "ENAMETOOLONG") return false;
    throw e;
  }

  return true;
};

const curlGet = async (envelope, {
  log,
  cfg,
  stats
}) => {
  const dataDir = (0, _fp.get)("http.data_dir", cfg);
  const getTypes = sToA(",", (0, _fp.get)("http.get_types", cfg));
  return _core.envelope.fmapDataAsync(async unit => {
    const downloads = await (0, _dashp.collectP)(async media => {
      if (!(0, _fp.includes)(media.type, getTypes)) return null;
      stats.count("total");
      const {
        type,
        term,
        href
      } = media;
      const source = href || term;
      const idHash = media._sc_id_hash; // We maintain backwards compatibility with the old location where to
      // store files. If the new style location fails, try the old style
      // location as well. And only if that one fails as well download the
      // file to the new style location.
      //
      // Old style locations could look like this:
      //   - data/{unit_hash}/image/{media_hash}/{filename}
      //   - data/{unit_hash}/image/{media_hash}/{media_hash}.ext
      //
      // New style location is aligned with the handling of other types of
      // media.
      //   - data/{unit_hash}/image/{media_hash}.ext

      const dir = (0, _path.join)(dataDir, unit._sc_id_hash, type);
      const oldDir = (0, _path.join)(dataDir, unit._sc_id_hash, type, idHash);
      const location = (0, _path.join)(dir, `${idHash}${(0, _path.extname)(_url.default.parse(source).pathname)}`);
      const oldLocation = (0, _path.join)(oldDir, `${idHash}${(0, _path.extname)(_url.default.parse(source).pathname)}`);
      const oldLocation2 = (0, _path.join)(oldDir, (0, _path.basename)(_url.default.parse(source).pathname));

      try {
        const locations = [location, oldLocation, oldLocation2];
        const locationsExists = await Promise.all(locations.map(l => downloadExists(l)));

        for (let i = 0; i < locationsExists.length; i += 1) {
          if (locationsExists[i]) {
            stats.count("existing");
            log.info(`Media ${source} exists at ${locations[i]}.`);
            return null;
          }
        }
      } catch (e) {
        const reason = `Failed to access ${e.path}: ${e.message}.`;
        stats.fail({
          type: unit._sc_source,
          term: source,
          reason
        });
        return null;
      }

      await (0, _pluginFs.mkdirP)(dir);
      let md5;
      let sha256;

      try {
        await (0, _utils.download)(source, location);
        stats.count("success");
        [md5, sha256] = await Promise.all([(0, _pluginFs.md5sum)(location), (0, _pluginFs.sha256sum)(location)]);
      } catch (e) {
        if (e.code !== "ENOENT") {
          const reason = `Failed to download ${media.type} to ${location}: ${e.message}. Cleaning up stale artifact.`;
          stats.fail({
            type: unit._sc_source,
            term: source,
            reason
          });
          await cleanUp(location);
          return null;
        }
      }

      log.info(`Fetched ${source} to ${location}.`);
      return {
        location,
        md5,
        sha256,
        type,
        term,
        ...(href ? {
          href
        } : {})
      };
    }, unit._sc_media);
    return Object.assign(unit, {
      _sc_downloads: unit._sc_downloads.concat(downloads.filter(d => d != null))
    });
  }, envelope);
};

const plugin = _core.plugin.liftManyA2([_utils.assertDir, curlGet]);

plugin.desc = "Fetch images from the web.";
plugin.argv = {
  "http.get_types": {
    type: "string",
    nargs: 1,
    default: "image,file,pdf",
    desc: "Fetch files of those media types."
  }
};
var _default = plugin;
exports.default = _default;