contentful-export/dist/tasks/download-assets.js

this tool allows you to export a space to a JSON dump

"use strict";

Object.defineProperty(exports, "__esModule", {
  value: true
});
exports.default = downloadAssets;
var _bluebird = _interopRequireDefault(require("bluebird"));
var _contentfulBatchLibs = require("contentful-batch-libs");
var _figures = _interopRequireDefault(require("figures"));
var _fs = require("fs");
var _path = _interopRequireDefault(require("path"));
var _stream = require("stream");
var _util = require("util");
var _embargoedAssets = require("../utils/embargoedAssets");
var _axios = _interopRequireDefault(require("axios"));
function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
const streamPipeline = (0, _util.promisify)(_stream.pipeline);

/**
 * @param {Object} options - The options for downloading the asset.
 * @param {string} options.url - The URL of the asset to download.
 * @param {string} options.directory - The directory where the asset should be saved.
 * @param {import('axios').AxiosInstance} options.httpClient - The HTTP client to use for downloading the asset.
 */
async function downloadAsset({
  url,
  directory,
  httpClient
}) {
  // handle urls without protocol
  if (url.startsWith('//')) {
    url = 'https:' + url;
  }

  // build local file path from the url for the download
  const parsedUrl = new URL(url);
  const decodedPathname = decodeURIComponent(parsedUrl.pathname);
  const localFile = _path.default.join(directory, parsedUrl.host, decodedPathname);

  // ensure directory exists and create file stream
  await _fs.promises.mkdir(_path.default.dirname(localFile), {
    recursive: true
  });
  const file = (0, _fs.createWriteStream)(localFile);
  try {
    // download asset
    const assetRequest = await httpClient.get(url, {
      responseType: 'stream',
      transformResponse: [data => data]
    });

    // Wait for stream to be consumed before returning local file
    await streamPipeline(assetRequest.data, file);
    return localFile;
  } catch (e) {
    /**
     * @type {import('axios').AxiosError}
     */
    const axiosError = e;
    throw new Error(`error response status: ${axiosError.response.status}`);
  }
}
function downloadAssets(options) {
  return (ctx, task) => {
    let successCount = 0;
    let warningCount = 0;
    let errorCount = 0;
    const httpClient = _axios.default.create({
      headers: options.headers,
      timeout: options.timeout,
      httpAgent: options.httpAgent,
      httpsAgent: options.httpsAgent,
      proxy: options.proxy
    });
    return _bluebird.default.map(ctx.data.assets, asset => {
      const entityName = (0, _contentfulBatchLibs.getEntityName)(asset);
      if (!asset.fields.file) {
        task.output = `${_figures.default.warning} asset ${entityName} has no file(s)`;
        warningCount++;
        return;
      }
      const locales = Object.keys(asset.fields.file);
      return _bluebird.default.mapSeries(locales, locale => {
        const url = asset.fields.file[locale].url;
        if (!url) {
          task.output = `${_figures.default.cross} asset '${entityName}' doesn't contain an url in path asset.fields.file[${locale}].url`;
          errorCount++;
          return _bluebird.default.resolve();
        }
        let startingPromise = _bluebird.default.resolve({
          url,
          directory: options.exportDir,
          httpClient
        });
        if ((0, _embargoedAssets.isEmbargoedAsset)(url)) {
          const {
            host,
            accessToken,
            spaceId,
            environmentId
          } = options;
          const expiresAtMs = (0, _embargoedAssets.calculateExpiryTimestamp)();
          startingPromise = (0, _embargoedAssets.signUrl)(host, accessToken, spaceId, environmentId, url, expiresAtMs, httpClient).then(signedUrl => ({
            url: signedUrl,
            directory: options.exportDir,
            httpClient
          }));
        }
        return startingPromise.then(downloadAsset).then(() => {
          task.output = `${_figures.default.tick} downloaded ${entityName} (${url})`;
          successCount++;
        }).catch(error => {
          task.output = `${_figures.default.cross} error downloading ${url}: ${error.message}`;
          errorCount++;
        });
      });
    }, {
      concurrency: 6
    }).then(() => {
      ctx.assetDownloads = {
        successCount,
        warningCount,
        errorCount
      };
    });
  };
}
module.exports = exports.default;