s3-tus-store/lib/index.js

[](https://travis-ci.org/blockai/s3-tus-store)

'use strict';

Object.defineProperty(exports, "__esModule", {
  value: true
});

var _slicedToArray = function () { function sliceIterator(arr, i) { var _arr = []; var _n = true; var _d = false; var _e = undefined; try { for (var _i = arr[Symbol.iterator](), _s; !(_n = (_s = _i.next()).done); _n = true) { _arr.push(_s.value); if (i && _arr.length === i) break; } } catch (err) { _d = true; _e = err; } finally { try { if (!_n && _i["return"]) _i["return"](); } finally { if (_d) throw _e; } } return _arr; } return function (arr, i) { if (Array.isArray(arr)) { return arr; } else if (Symbol.iterator in Object(arr)) { return sliceIterator(arr, i); } else { throw new TypeError("Invalid attempt to destructure non-iterable instance"); } }; }();
// import { SizeStream } from 'common-streams'

// import { inspect } from 'util'

var _debug = require('debug');

var _debug2 = _interopRequireDefault(_debug);

var _toObjectReducer = require('to-object-reducer');

var _toObjectReducer2 = _interopRequireDefault(_toObjectReducer);

var _meterstream = require('meterstream');

var _meterstream2 = _interopRequireDefault(_meterstream);

var _stream = require('stream');

var _duplexify = require('duplexify');

var _duplexify2 = _interopRequireDefault(_duplexify);

var _abstractTusStore = require('abstract-tus-store');

var _eos = require('./eos');

var _eos2 = _interopRequireDefault(_eos);

var _writePartByPart = require('./write-part-by-part');

var _writePartByPart2 = _interopRequireDefault(_writePartByPart);

function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }

function _asyncToGenerator(fn) { return function () { var gen = fn.apply(this, arguments); return new Promise(function (resolve, reject) { function step(key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { return Promise.resolve(value).then(function (value) { step("next", value); }, function (err) { step("throw", err); }); } } return step("next"); }); }; }

// TODO: docs below slightly outdated!
// Inspired by https://github.com/tus/tusd/blob/master/s3store/s3store.go
//
// Configuration
//
// In order to allow this backend to function properly, the user accessing the
// bucket must have at least following AWS IAM policy permissions for the
// bucket and all of its subresources:
// 	s3:AbortMultipartUpload
// 	s3:DeleteObject
// 	s3:GetObject
// 	s3:ListMultipartUploadParts
// 	s3:PutObject//
// Implementation
//
// Once a new tus upload is initiated, multiple objects in S3 are created:
//
// First of all, a new info object is stored which contains a JSON-encoded blob
// of general information about the upload including its size and meta data.
// This kind of objects have the suffix ".info" in their key.
//
// In addition a new multipart upload
// (http://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html) is
// created. Whenever a new chunk is uploaded to tusd using a PATCH request, a
// new part is pushed to the multipart upload on S3.
//
// If meta data is associated with the upload during creation, it will be added
// to the multipart upload and after finishing it, the meta data will be passed
// to the final object. However, the metadata which will be attached to the
// final object can only contain ASCII characters and every non-ASCII character
// will be replaced by a question mark (for example, "Menü" will be "Men?").
// However, this does not apply for the metadata returned by the GetInfo
// function since it relies on the info object for reading the metadata.
// Therefore, HEAD responses will always contain the unchanged metadata, Base64-
// encoded, even if it contains non-ASCII characters.

const debug = (0, _debug2.default)('s3-tus-store:index');

const defaults = {
  // MaxPartSize specifies the maximum size of a single part uploaded to S3
  // in bytes. This value must be bigger than minPartSize! In order to
  // choose the correct number, two things have to be kept in mind:
  //
  // If this value is too big and uploading the part to S3 is interrupted
  // unexpectedly, the entire part is discarded and the end user is required
  // to resume the upload and re-upload the entire big part.
  //
  // If this value is too low, a lot of requests to S3 may be made, depending
  // on how fast data is coming in. This may result in an eventual overhead.
  maxPartSize: 6 * 1024 * 1024, // 6 MB
  //
  // @oli we set maxPartSize to minPartSize so we dont need
  // to know content length in advance
  //
  // MinPartSize specifies the minimum size of a single part uploaded to S3
  // in bytes. This number needs to match with the underlying S3 backend or else
  // uploaded parts will be reject. AWS S3, for example, uses 5MB for this value.
  minPartSize: 5 * 1024 * 1024
};

const asciiOnly = str => str.replace(/[^\x00-\x7F]/g, '');

// TODO: optional TTL?
// TODO: MAKE SURE UPLOADID IS UNIQUE REGARDLESS OF KEY

exports.default = (_ref) => {
  let client = _ref.client,
      bucket = _ref.bucket;
  var _ref$minPartSize = _ref.minPartSize;
  let minPartSize = _ref$minPartSize === undefined ? defaults.minPartSize : _ref$minPartSize;
  var _ref$maxPartSize = _ref.maxPartSize;
  let maxPartSize = _ref$maxPartSize === undefined ? defaults.maxPartSize : _ref$maxPartSize;

  const buildParams = (key, extra) => Object.assign({
    Key: key,
    Bucket: bucket
  }, extra);

  const buildS3Metadata = function () {
    let uploadMetadata = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : {};

    debug('buildS3Metadata');
    const metadata = uploadMetadata;
    // Values must be strings... :(
    // TODO: test what happens with non ASCII keys/values
    const validMetadata = Object.keys(metadata).map(key => [key, `${ metadata[key] }`])
    // strip non US ASCII characters
    .map((_ref2) => {
      var _ref3 = _slicedToArray(_ref2, 2);

      let key = _ref3[0],
          str = _ref3[1];
      return [key, asciiOnly(str)];
    }).reduce(_toObjectReducer2.default, {});
    return validMetadata;
  };

  const getUploadKey = uploadId => `tus-uploads/${ uploadId }`;
  const getUploadKeyForKey = key => `tus-uploads/finished/${ key }.upload`;

  const getUploadForKey = (() => {
    var _ref4 = _asyncToGenerator(function* (key) {
      debug('buildS3Metadata', { key });

      var _ref5 = yield client.getObject(buildParams(getUploadKeyForKey(key))).promise();

      const Body = _ref5.Body;

      return JSON.parse(Body);
    });

    return function getUploadForKey(_x2) {
      return _ref4.apply(this, arguments);
    };
  })();

  const getUpload = (() => {
    var _ref6 = _asyncToGenerator(function* (uploadId) {
      debug('getUpload', { uploadId });

      var _ref7 = yield client.getObject(buildParams(getUploadKey(uploadId))).promise().catch(function (err) {
        if (err.code === 'NoSuchUpload') {
          throw new _abstractTusStore.errors.UploadNotFound(uploadId);
        }
        throw err;
      });

      const Body = _ref7.Body;

      return JSON.parse(Body);
    });

    return function getUpload(_x3) {
      return _ref6.apply(this, arguments);
    };
  })();

  const saveUpload = (() => {
    var _ref8 = _asyncToGenerator(function* (uploadId, upload) {
      debug('saveUpload', { uploadId, upload });
      const key = getUploadKey(uploadId);
      const json = JSON.stringify(upload);
      yield client.putObject(buildParams(key, {
        Body: json,
        ContentLength: Buffer.byteLength(json)
      })).promise();
    });

    return function saveUpload(_x4, _x5) {
      return _ref8.apply(this, arguments);
    };
  })();

  const saveUploadForKey = (() => {
    var _ref9 = _asyncToGenerator(function* (uploadId, upload) {
      debug('saveUploadForKey', { uploadId, upload });
      const key = getUploadKeyForKey(upload.key);
      const json = JSON.stringify(Object.assign({}, upload, {
        uploadId
      }));
      yield client.putObject(buildParams(key, {
        Body: json,
        ContentLength: Buffer.byteLength(json)
      })).promise();
    });

    return function saveUploadForKey(_x6, _x7) {
      return _ref9.apply(this, arguments);
    };
  })();

  const getParts = (() => {
    var _ref10 = _asyncToGenerator(function* (uploadId, key) {
      var _ref11 = yield client.listParts(buildParams(key, {
        UploadId: uploadId
      })).promise(),
          _ref11$Parts = _ref11.Parts;

      const Parts = _ref11$Parts === undefined ? [] : _ref11$Parts;

      debug('getParts', Parts);
      return Parts;
    });

    return function getParts(_x8, _x9) {
      return _ref10.apply(this, arguments);
    };
  })();

  const countSizeFromParts = parts => parts.map((_ref12) => {
    let Size = _ref12.Size;
    return Size;
  }).reduce((total, size) => total + size, 0);

  const getUploadOffset = (() => {
    var _ref13 = _asyncToGenerator(function* (uploadIdOrParts, key) {
      debug('getUploadOffset', { uploadIdOrParts, key });
      if (Array.isArray(uploadIdOrParts)) {
        debug('parts', uploadIdOrParts);
        return countSizeFromParts(uploadIdOrParts);
      }
      const parts = yield getParts(uploadIdOrParts, key);
      if (!Array.isArray(parts)) {
        throw new Error('this should never happen');
      }
      return getUploadOffset(parts);
    });

    return function getUploadOffset(_x10, _x11) {
      return _ref13.apply(this, arguments);
    };
  })();

  const create = (() => {
    var _ref15 = _asyncToGenerator(function* (key, _ref14) {
      let uploadLength = _ref14.uploadLength;
      var _ref14$metadata = _ref14.metadata;
      let metadata = _ref14$metadata === undefined ? {} : _ref14$metadata;

      debug('create', { key });

      var _ref16 = yield client.createMultipartUpload(buildParams(key, {
        Metadata: buildS3Metadata(metadata)
      })).promise();

      const UploadId = _ref16.UploadId;

      const uploadId = UploadId;
      const upload = {
        key,
        uploadLength,
        metadata
      };
      yield saveUpload(uploadId, upload);
      return { uploadId };
    });

    return function create(_x12, _x13) {
      return _ref15.apply(this, arguments);
    };
  })();

  const info = (() => {
    var _ref17 = _asyncToGenerator(function* (uploadId) {
      debug('info', { uploadId });
      const upload = yield getUpload(uploadId);
      debug('got upload', upload);
      const offset = yield getUploadOffset(uploadId, upload.key).then(function (uploadOffset) {
        if (uploadOffset === upload.uploadLength) {
          // upload is completed but completeMultipartUpload has not been
          // called for some reason...
          // force a last call to append :/
          return uploadOffset - 1;
        }
        return uploadOffset;
      }).catch(function (err) {
        // we got the upload file but upload part does not exist
        // that means the upload is actually completed.
        if (err.code === 'NoSuchUpload') {
          return upload.uploadLength;
        }
        throw err;
      });
      return Object.assign({
        offset
      }, upload);
    });

    return function info(_x14) {
      return _ref17.apply(this, arguments);
    };
  })();

  const createLimitStream = (uploadLength, offset) => {
    if (typeof uploadLength === 'undefined') {
      return new _stream.PassThrough();
    }
    const meterStream = new _meterstream2.default(uploadLength - offset);
    return meterStream;
  };

  const afterWrite = (() => {
    var _ref18 = _asyncToGenerator(function* (uploadId, upload, beforeComplete, parts) {
      const offset = yield getUploadOffset(parts);
      // Upload complete!
      debug(`offset = ${ offset }`);
      // TODO: what happens if process crashes here? multipart upload never completes...
      // So, when we get HEAD (info()) request and the offset === uploadLength, we have to
      // check that the multipart upload was really completed and if not, complete it before
      // returning a response
      if (offset === upload.uploadLength) {
        debug('Completing upload!');
        yield beforeComplete(upload, uploadId);
        const MultipartUpload = {
          Parts: parts.map(function (_ref19) {
            let ETag = _ref19.ETag,
                PartNumber = _ref19.PartNumber;
            return { ETag, PartNumber };
          })
        };
        const completeUploadParams = buildParams(null, {
          MultipartUpload,
          UploadId: uploadId,
          Key: upload.key
        });
        yield saveUploadForKey(uploadId, upload);
        debug('s3.completeMultipartUpload start', { completeUploadParams });
        yield client.completeMultipartUpload(completeUploadParams).promise();
        debug('s3.completeMultipartUpload end');
        // TODO: remove upload file?
        return {
          offset,
          complete: true,
          upload: Object.assign({}, upload, {
            offset
          })
        };
      }
      return { offset };
    });

    return function afterWrite(_x15, _x16, _x17, _x18) {
      return _ref18.apply(this, arguments);
    };
  })();

  const append = (() => {
    var _ref20 = _asyncToGenerator(function* (uploadId, rs, arg3, arg4) {
      // guess arg by type
      var _ref21 = function () {
        if (typeof arg3 === 'object') {
          return { opts: arg3 };
        }
        return { expectedOffset: arg3, opts: arg4 };
      }();

      const expectedOffset = _ref21.expectedOffset;
      var _ref21$opts = _ref21.opts;
      const opts = _ref21$opts === undefined ? {} : _ref21$opts;

      debug('append', { uploadId, expectedOffset });
      var _opts$beforeComplete = opts.beforeComplete;
      const beforeComplete = _opts$beforeComplete === undefined ? _asyncToGenerator(function* () {}) : _opts$beforeComplete;

      // need to do this asap to make sure we don't miss reads

      const through = rs.pipe(new _stream.PassThrough());
      const rsEos = (0, _eos2.default)(rs, { writable: false });

      debug('append opts', opts);

      const upload = yield getUpload(uploadId);
      const parts = yield getParts(uploadId, upload.key);
      const offset = yield getUploadOffset(parts);

      // For some reason, upload is finished but not completed yet
      if (offset === upload.uploadLength) {
        if (!Number.isInteger(expectedOffset) || expectedOffset === upload.uploadLength - 1) {
          return afterWrite(uploadId, upload, beforeComplete, parts);
        }
      }

      if (Number.isInteger(expectedOffset)) {
        // check if offset is right
        if (offset !== expectedOffset) {
          throw new _abstractTusStore.errors.OffsetMismatch(offset, expectedOffset);
        }
      }

      const limitStream = createLimitStream(upload.uploadLength, offset);
      const limitStreamEos = (0, _eos2.default)(limitStream, { writable: false });

      // Parts are 1-indexed
      const nextPartNumber = parts.length ? parts[parts.length - 1].PartNumber + 1 : 1;

      const bytesLimit = Number.isInteger(upload.uploadLength) ? upload.uploadLength - offset : Infinity;

      const newParts = yield (0, _writePartByPart2.default)({
        client,
        bucket,
        uploadId,
        nextPartNumber,
        maxPartSize,
        minPartSize,
        // Number of bytes remaining to complete upload
        bytesLimit,
        key: upload.key,
        body: through.pipe(limitStream) });
      debug('writePartByPart done');

      // This ensures that if either stream emitted an error,
      // our promise will throw
      yield Promise.all([rsEos, limitStreamEos]);

      return afterWrite(uploadId, upload, beforeComplete, [...parts, ...newParts]);
    });

    return function append(_x19, _x20, _x21, _x22) {
      return _ref20.apply(this, arguments);
    };
  })();

  const createReadStream = (key, onInfo) => {
    debug('createReadStream', { key });
    const rs = (0, _duplexify2.default)();
    const letsgo = (() => {
      var _ref23 = _asyncToGenerator(function* () {
        const body = client.getObject(buildParams(key)).createReadStream();
        rs.setReadable(body);
        // https://github.com/aws/aws-sdk-js/issues/1153
        if (onInfo) {
          debug('onInfo', { key });
          // const data = await client.headObject(buildParams(key)).promise()
          // S3 metadata sucks...

          var _ref24 = yield getUploadForKey(key);

          const uploadLength = _ref24.uploadLength,
                metadata = _ref24.metadata;

          onInfo({
            metadata,
            contentLength: uploadLength
          });
        }
      });

      return function letsgo() {
        return _ref23.apply(this, arguments);
      };
    })();
    letsgo();
    return rs;
  };

  return {
    info,
    create,
    append,
    createReadStream,
    minChunkSize: minPartSize
  };
};