s3-batch-upload/lib/Uploader.js

Super fast batched S3 folder uploads from CLI or API.

"use strict";

var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");

exports.__esModule = true;
exports.default = void 0;

var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));

var _extends2 = _interopRequireDefault(require("@babel/runtime/helpers/extends"));

var _batch = _interopRequireDefault(require("./batch"));

const glob = require('glob');

const minimatch = require('minimatch');

const path = require('path');

const AWS = require('aws-sdk');

const ProgressBar = require('progress');

const ora = require('ora');

const chalk = require('chalk');

const fs = require('fs');

const mime = require('mime');

const defaultOptions = {
  dryRun: false,
  concurrency: 100,
  glob: '*.*',
  globOptions: {},
  overwrite: true
};

class Uploader {
  constructor(options) {
    Object.defineProperty(this, "s3", {
      configurable: true,
      enumerable: true,
      writable: true,
      value: void 0
    });
    Object.defineProperty(this, "options", {
      configurable: true,
      enumerable: true,
      writable: true,
      value: void 0
    });
    Object.defineProperty(this, "bar", {
      configurable: true,
      enumerable: true,
      writable: true,
      value: void 0
    });
    this.options = (0, _extends2.default)({}, defaultOptions, options);

    if (this.options.config) {
      if (typeof this.options.config === 'string') {
        AWS.config.loadFromPath(this.options.config);
      } else if (typeof this.options.config === 'object') {
        AWS.config.constructor(this.options.config);
      } else {
        throw new Error('unsupported config is passed as a argument.');
      }
    } // TODO: more checks on other options?


    if (!this.options.bucket) {
      throw new Error('No bucket defined!');
    }

    this.s3 = this.options.s3Client || new AWS.S3();
  }
  /**
   * Executes the upload operation based on the provided options in the Uploader constructor.
   * @returns A list of paths of the upload files relative to the bucket.
   */


  upload() {
    return this.run();
  }

  run() {
    var _this = this;

    return (0, _asyncToGenerator2.default)(function* () {
      const files = yield _this.getFiles();
      const _this$options = _this.options,
            concurrency = _this$options.concurrency,
            localPath = _this$options.localPath,
            remotePath = _this$options.remotePath; // a nice progress bar to show during upload

      _this.bar = new ProgressBar('[:bar] :percent | :etas | :current / :total | :rate/fps ', {
        total: files.length,
        complete: '=',
        incomplete: ' ',
        width: 20
      }); // do the work!

      const results = yield (0, _batch.default)({
        files,
        concurrency,
        processItem: file => {
          const key = path.join(remotePath, file);
          return _this.uploadFile(path.resolve(localPath, file), key);
        },
        onProgress: () => _this.bar.tick()
      }); // tslint:disable-next-line no-console

      console.log('Upload complete!');
      return results;
    })();
  }
  /**
   * Based on the local path and the provided glob pattern, this util function will find all relevant
   * files, which will be used to upload in another step.
   * @returns A list of resolved files based on the glob pattern
   */


  getFiles() {
    const _options = this.options,
          localPath = _options.localPath,
          globPath = _options.glob,
          globOptions = _options.globOptions;
    const gatheringSpinner = ora(`Gathering files from ${chalk.blue(localPath)} (please wait) ...`);
    gatheringSpinner.start();
    return new Promise((resolve, reject) => {
      glob(`**/${globPath}`, (0, _extends2.default)({}, globOptions, {
        cwd: path.resolve(localPath)
      }), (err, files) => {
        if (err) {
          gatheringSpinner.fail(err);
          reject(err);
        }

        gatheringSpinner.succeed(`Found ${chalk.green(files.length)} files at ${chalk.blue(localPath)}, starting upload:`);
        resolve(files);
      });
    });
  }
  /**
   * Uploads a single file to S3 from the local to the remote path with the available options,
   * and returns the uploaded location.
   *
   * @param localFilePath Path to the local file, either relative to cwd, or absolute
   * @param remotePath The path to upload the file to in the bucket
   * @returns The remote path upload location relative to the bucket
   */


  uploadFile(localFilePath, remotePath) {
    var _this2 = this;

    return (0, _asyncToGenerator2.default)(function* () {
      const _this2$options = _this2.options,
            dryRun = _this2$options.dryRun,
            Bucket = _this2$options.bucket,
            ACL = _this2$options.accessControlLevel;
      const baseParams = {
        Bucket,
        Key: remotePath.replace(/\\/g, '/')
      };

      if (!_this2.options.overwrite) {
        try {
          yield _this2.s3.headObject(baseParams).promise(); // tslint:disable-next-line no-console

          console.log('File exists, skipping: ', baseParams.Key);
          return baseParams.Key;
        } catch (err) {
          if (err.code !== 'NotFound') {
            // tslint:disable-next-line no-console
            console.error('err:', err);
            throw err;
          }
        }
      }

      const body = fs.createReadStream(localFilePath);
      const params = (0, _extends2.default)({}, baseParams, {
        Body: body,
        ContentType: mime.getType(localFilePath),
        CacheControl: _this2.getCacheControlValue(localFilePath)
      });

      if (ACL) {
        params.ACL = ACL;
      }

      if (!dryRun) {
        try {
          yield _this2.s3.upload(params).promise();
        } catch (err) {
          // tslint:disable-next-line no-console
          console.error('err:', err);
          throw err;
        }
      }

      return params.Key;
    })();
  }
  /**
   *
   * @param file Path to a local file, either relative to cwd, or absolute
   * @return The resolved CacheControl value based on the provided settings
   */


  getCacheControlValue(file) {
    const cacheControl = this.options.cacheControl;

    if (cacheControl) {
      // return single option for all files
      if (typeof cacheControl === 'string') {
        return cacheControl;
      } // find match in glob patterns


      const match = Object.keys(cacheControl).find(key => minimatch(file, key));
      return match && cacheControl[match] || '';
    } // return default value


    return '';
  }

}

exports.default = Uploader;