@lyra/export/lib/AssetHandler.js

Export Lyra documents and assets

'use strict';

var _slicedToArray = function () { function sliceIterator(arr, i) { var _arr = []; var _n = true; var _d = false; var _e = undefined; try { for (var _i = arr[Symbol.iterator](), _s; !(_n = (_s = _i.next()).done); _n = true) { _arr.push(_s.value); if (i && _arr.length === i) break; } } catch (err) { _d = true; _e = err; } finally { try { if (!_n && _i["return"]) _i["return"](); } finally { if (_d) throw _e; } } return _arr; } return function (arr, i) { if (Array.isArray(arr)) { return arr; } else if (Symbol.iterator in Object(arr)) { return sliceIterator(arr, i); } else { throw new TypeError("Invalid attempt to destructure non-iterable instance"); } }; }();

var _extends = Object.assign || function (target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) { if (Object.prototype.hasOwnProperty.call(source, key)) { target[key] = source[key]; } } } return target; };

function _objectWithoutProperties(obj, keys) { var target = {}; for (var i in obj) { if (keys.indexOf(i) >= 0) continue; if (!Object.prototype.hasOwnProperty.call(obj, i)) continue; target[i] = obj[i]; } return target; }

function _asyncToGenerator(fn) { return function () { var gen = fn.apply(this, arguments); return new Promise(function (resolve, reject) { function step(key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { return Promise.resolve(value).then(function (value) { step("next", value); }, function (err) { step("throw", err); }); } } return step("next"); }); }; }

const path = require('path');
const crypto = require('crypto');
const fse = require('fs-extra');
const miss = require('mississippi');
const PQueue = require('p-queue');

var _require = require('lodash');

const omit = _require.omit;

const pkg = require('../package.json');
const requestStream = require('./requestStream');
const debug = require('./debug');

const EXCLUDE_PROPS = ['_id', '_type', 'assetId', 'extension', 'mimeType', 'path', 'url'];
const ACTION_REMOVE = 'remove';
const ACTION_REWRITE = 'rewrite';

class AssetHandler {
  constructor(options) {
    var _this = this;

    this.rewriteAssets = miss.through.obj((() => {
      var _ref = _asyncToGenerator(function* (doc, enc, callback) {
        if (['lyra.imageAsset', 'lyra.fileAsset'].includes(doc._type)) {
          const type = doc._type === 'lyra.imageAsset' ? 'image' : 'file';
          const filePath = `${type}s/${generateFilename(doc._id)}`;
          _this.assetsSeen.set(doc._id, type);
          _this.queueAssetDownload(doc, filePath, type);
          callback();
          return;
        }

        callback(null, (yield _this.findAndModify(doc, ACTION_REWRITE)));
      });

      return function (_x, _x2, _x3) {
        return _ref.apply(this, arguments);
      };
    })());
    this.stripAssets = miss.through.obj((() => {
      var _ref2 = _asyncToGenerator(function* (doc, enc, callback) {
        if (['lyra.imageAsset', 'lyra.fileAsset'].includes(doc._type)) {
          callback();
          return;
        }

        callback(null, (yield _this.findAndModify(doc, ACTION_REMOVE)));
      });

      return function (_x4, _x5, _x6) {
        return _ref2.apply(this, arguments);
      };
    })());
    this.skipAssets = miss.through.obj((doc, enc, callback) => {
      const isAsset = ['lyra.imageAsset', 'lyra.fileAsset'].includes(doc._type);
      if (isAsset) {
        callback();
        return;
      }

      callback(null, doc);
    });
    this.noop = miss.through.obj((doc, enc, callback) => callback(null, doc));

    this.findAndModify = (() => {
      var _ref3 = _asyncToGenerator(function* (item, action) {
        if (Array.isArray(item)) {
          const children = yield Promise.all(item.map(function (child) {
            return _this.findAndModify(child, action);
          }));
          return children.filter(Boolean);
        }

        if (!item || typeof item !== 'object') {
          return item;
        }

        const isAsset = isAssetField(item);
        if (isAsset && action === ACTION_REMOVE) {
          return undefined;
        }

        if (isAsset && action === ACTION_REWRITE) {
          const asset = item.asset,
                other = _objectWithoutProperties(item, ['asset']);

          const assetId = asset._ref;
          if (isModernAsset(assetId)) {
            const assetType = getAssetType(item);
            const filePath = `${assetType}s/${generateFilename(assetId)}`;
            return _extends({ _lyraAsset: `${assetType}@file://./${filePath}` }, other);
          }

          // Legacy asset
          const type = _this.assetsSeen.get(assetId) || (yield _this.lookupAssetType(assetId));
          const filePath = `${type}s/${generateFilename(assetId)}`;
          return _extends({ _lyraAsset: `${type}@file://./${filePath}` }, other);
        }

        const newItem = {};
        const keys = Object.keys(item);
        for (let i = 0; i < keys.length; i++) {
          const key = keys[i];
          const value = item[key];

          // eslint-disable-next-line no-await-in-loop
          newItem[key] = yield _this.findAndModify(value, action);

          if (typeof newItem[key] === 'undefined') {
            delete newItem[key];
          }
        }

        return newItem;
      });

      return function (_x7, _x8) {
        return _ref3.apply(this, arguments);
      };
    })();

    this.lookupAssetType = (() => {
      var _ref4 = _asyncToGenerator(function* (assetId) {
        const docType = yield _this.client.fetch('*[_id == $id][0]._type', {
          id: assetId
        });
        return docType === 'lyra.imageAsset' ? 'image' : 'file';
      });

      return function (_x9) {
        return _ref4.apply(this, arguments);
      };
    })();

    this.client = options.client;
    this.tmpDir = options.tmpDir;
    this.assetDirsCreated = false;

    this.assetsSeen = new Map();
    this.assetMap = {};
    this.filesWritten = 0;
    this.queueSize = 0;
    this.queue = options.queue || new PQueue({ concurrency: 3 });
    this.reject = () => {
      throw new Error('Asset handler errored before `finish()` was called');
    };
  }

  clear() {
    this.assetsSeen.clear();
    this.queue.clear();
    this.queueSize = 0;
  }

  finish() {
    return new Promise((resolve, reject) => {
      this.reject = reject;
      this.queue.onIdle().then(() => resolve(this.assetMap));
    });
  }

  // Called when we want to download all assets to local filesystem and rewrite documents to hold
  // placeholder asset references (_lyraAsset: 'image@file:///local/path')


  // Called in the case where we don't _want_ assets, so basically just remove all asset documents
  // as well as references to assets (*.asset._ref ^= (image|file)-)


  // Called when we are using raw export mode along with `assets: false`, where we simply
  // want to skip asset documents but retain asset references (useful for data mangling)


  queueAssetDownload(assetDoc, dstPath, type) {
    if (!assetDoc.url) {
      debug('Asset document "%s" does not have a URL property, skipping', assetDoc._id);
      return;
    }

    debug('Adding download task for %s (destination: %s)', assetDoc._id, dstPath);
    this.queueSize++;
    this.queue.add(() => this.downloadAsset(assetDoc, dstPath));
  }

  downloadAsset(assetDoc, dstPath) {
    var _this2 = this;

    return _asyncToGenerator(function* () {
      const url = assetDoc.url;

      const headers = { 'User-Agent': `${pkg.name}@${pkg.version}` };
      const stream = yield requestStream({ url, headers });

      if (stream.statusCode !== 200) {
        _this2.queue.clear();
        _this2.reject(new Error(`Referenced asset URL "${url}" returned HTTP ${stream.statusCode}`));
        return;
      }

      if (!_this2.assetDirsCreated) {
        /* eslint-disable no-sync */
        fse.ensureDirSync(path.join(_this2.tmpDir, 'files'));
        fse.ensureDirSync(path.join(_this2.tmpDir, 'images'));
        /* eslint-enable no-sync */
        _this2.assetDirsCreated = true;
      }

      debug('Asset stream ready, writing to filesystem at %s', dstPath);
      const hash = yield writeHashedStream(path.join(_this2.tmpDir, dstPath), stream);
      const type = assetDoc._type === 'lyra.imageAsset' ? 'image' : 'file';
      const id = `${type}-${hash}`;

      const metaProps = omit(assetDoc, EXCLUDE_PROPS);
      if (Object.keys(metaProps).length > 0) {
        _this2.assetMap[id] = metaProps;
      }

      _this2.filesWritten++;
    })();
  }

  // eslint-disable-next-line complexity
}

function isAssetField(item) {
  return item.asset && item.asset._ref;
}

function getAssetType(item) {
  if (!item.asset || typeof item.asset._ref !== 'string') {
    return null;
  }

  var _ref5 = item.asset._ref.match(/^(image|file)-/) || [],
      _ref6 = _slicedToArray(_ref5, 2);

  const type = _ref6[1];

  return type || null;
}

function isModernAsset(assetId) {
  return (/^(image|file)/.test(assetId)
  );
}

function generateFilename(assetId) {
  var _ref7 = assetId.match(/^(image|file)-(.*?)(-[a-z]+)?$/) || [],
      _ref8 = _slicedToArray(_ref7, 4);

  const asset = _ref8[2],
        ext = _ref8[3];

  const extension = (ext || 'bin').replace(/^-/, '');
  return asset ? `${asset}.${extension}` : `${assetId}.bin`;
}

function writeHashedStream(filePath, stream) {
  const hash = crypto.createHash('sha1');
  const hasher = miss.through((chunk, enc, cb) => {
    hash.update(chunk);
    cb(null, chunk);
  });

  return new Promise((resolve, reject) => miss.pipe(stream, hasher, fse.createWriteStream(filePath), err => {
    return err ? reject(err) : resolve(hash.digest('hex'));
  }));
}

module.exports = AssetHandler;