@lyra/export/lib/export.js

Export Lyra documents and assets

'use strict';

function _asyncToGenerator(fn) { return function () { var gen = fn.apply(this, arguments); return new Promise(function (resolve, reject) { function step(key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { return Promise.resolve(value).then(function (value) { step("next", value); }, function (err) { step("throw", err); }); } } return step("next"); }); }; }

const os = require('os');
const path = require('path');
const zlib = require('zlib');
const fse = require('fs-extra');
const miss = require('mississippi');
const split = require('split2');
const archiver = require('archiver');
const debug = require('./debug');
const AssetHandler = require('./AssetHandler');
const stringifyStream = require('./stringifyStream');
const validateOptions = require('./validateOptions');
const rejectOnApiError = require('./rejectOnApiError');
const getDocumentsStream = require('./getDocumentsStream');
const filterSystemDocuments = require('./filterSystemDocuments');
const filterDocumentTypes = require('./filterDocumentTypes');
const filterDrafts = require('./filterDrafts');

const noop = () => null;

function exportDataset(opts) {
  const options = validateOptions(opts);
  const onProgress = options.onProgress || noop;
  const archive = archiver('tar', {
    gzip: true,
    gzipOptions: {
      level: options.compress ? zlib.Z_DEFAULT_COMPRESSION : zlib.Z_NO_COMPRESSION
    }
  });

  const slugDate = new Date().toISOString().replace(/[^a-z0-9]/gi, '-').toLowerCase();

  const prefix = `${opts.dataset}-export-${slugDate}`;
  const tmpDir = path.join(os.tmpdir(), prefix);
  const assetHandler = new AssetHandler({
    client: options.client,
    tmpDir,
    prefix
  });

  debug('Outputting assets (temporarily) to %s', tmpDir);
  debug('Outputting to %s', options.outputPath === '-' ? 'stdout' : options.outputPath);
  const outputStream = options.outputPath === '-' ? process.stdout : fse.createWriteStream(options.outputPath);

  let assetStreamHandler = assetHandler.noop;
  if (!options.raw) {
    assetStreamHandler = options.assets ? assetHandler.rewriteAssets : assetHandler.stripAssets;
  }

  return new Promise((() => {
    var _ref = _asyncToGenerator(function* (resolve, reject) {
      let onComplete = (() => {
        var _ref3 = _asyncToGenerator(function* (err) {
          onProgress({ step: 'Clearing temporary files...' });
          yield fse.remove(tmpDir);

          if (!err) {
            resolve();
            return;
          }

          debug('Error during streaming: %s', err.stack);
          assetHandler.clear();
          reject(err);
        });

        return function onComplete(_x4) {
          return _ref3.apply(this, arguments);
        };
      })();

      miss.finished(archive, function (archiveErr) {
        if (archiveErr) {
          debug('Archiving errored! %s', archiveErr.stack);
          reject(archiveErr);
          return;
        }

        debug('Archive finished!');
      });

      debug('Getting dataset export stream');
      onProgress({ step: 'Exporting documents...' });

      let documentCount = 0;
      let lastReported = Date.now();
      const reportDocumentCount = function reportDocumentCount(chunk, enc, cb) {
        ++documentCount;

        const now = Date.now();
        if (now - lastReported > 50) {
          onProgress({
            step: 'Exporting documents...',
            current: documentCount,
            total: '?',
            update: true
          });

          lastReported = now;
        }

        cb(null, chunk);
      };

      const inputStream = yield getDocumentsStream(options.client, options.dataset);
      const jsonStream = miss.pipeline(inputStream, split(JSON.parse), rejectOnApiError, filterSystemDocuments, assetStreamHandler, filterDocumentTypes(options.types), options.drafts ? miss.through.obj() : filterDrafts, stringifyStream, miss.through(reportDocumentCount));

      miss.finished(jsonStream, (() => {
        var _ref2 = _asyncToGenerator(function* (err) {
          if (err) {
            return;
          }

          onProgress({
            step: 'Exporting documents...',
            current: documentCount,
            total: documentCount,
            update: true
          });

          if (!options.raw && options.assets) {
            onProgress({ step: 'Downloading assets...' });
          }

          let prevCompleted = 0;
          const progressInterval = setInterval(function () {
            const completed = assetHandler.queueSize - assetHandler.queue.size;
            if (prevCompleted === completed) {
              return;
            }

            prevCompleted = completed;
            onProgress({
              step: 'Downloading assets...',
              current: completed,
              total: assetHandler.queueSize,
              update: true
            });
          }, 500);

          debug('Waiting for asset handler to complete downloads');
          try {
            const assetMap = yield assetHandler.finish();
            archive.append(JSON.stringify(assetMap), { name: 'assets.json', prefix });
            clearInterval(progressInterval);
          } catch (assetErr) {
            clearInterval(progressInterval);
            reject(assetErr);
            return;
          }

          // Add all downloaded assets to archive
          archive.directory(path.join(tmpDir, 'files'), `${prefix}/files`, {
            store: true
          });
          archive.directory(path.join(tmpDir, 'images'), `${prefix}/images`, {
            store: true
          });

          debug('Finalizing archive, flushing streams');
          onProgress({ step: 'Adding assets to archive...' });
          archive.finalize();
        });

        return function (_x3) {
          return _ref2.apply(this, arguments);
        };
      })());

      archive.on('warning', function (err) {
        debug('Archive warning: %s', err.message);
      });

      archive.append(jsonStream, { name: 'data.ndjson', prefix });
      miss.pipe(archive, outputStream, onComplete);
    });

    return function (_x, _x2) {
      return _ref.apply(this, arguments);
    };
  })());
}

module.exports = exportDataset;