@uppy/aws-s3
Version:
Upload to Amazon S3 with Uppy
431 lines (429 loc) • 17.2 kB
JavaScript
function _classPrivateFieldLooseBase(e, t) { if (!{}.hasOwnProperty.call(e, t)) throw new TypeError("attempted to use private field on non-instance"); return e; }
var id = 0;
function _classPrivateFieldLooseKey(e) { return "__private_" + id++ + "_" + e; }
import { pausingUploadReason } from './MultipartUploader.js';
import { throwIfAborted } from './utils.js';
function removeMetadataFromURL(urlString) {
const urlObject = new URL(urlString);
urlObject.search = '';
urlObject.hash = '';
return urlObject.href;
}
var _abortMultipartUpload = /*#__PURE__*/_classPrivateFieldLooseKey("abortMultipartUpload");
var _cache = /*#__PURE__*/_classPrivateFieldLooseKey("cache");
var _createMultipartUpload = /*#__PURE__*/_classPrivateFieldLooseKey("createMultipartUpload");
var _fetchSignature = /*#__PURE__*/_classPrivateFieldLooseKey("fetchSignature");
var _getUploadParameters = /*#__PURE__*/_classPrivateFieldLooseKey("getUploadParameters");
var _listParts = /*#__PURE__*/_classPrivateFieldLooseKey("listParts");
var _previousRetryDelay = /*#__PURE__*/_classPrivateFieldLooseKey("previousRetryDelay");
var _requests = /*#__PURE__*/_classPrivateFieldLooseKey("requests");
var _retryDelays = /*#__PURE__*/_classPrivateFieldLooseKey("retryDelays");
var _sendCompletionRequest = /*#__PURE__*/_classPrivateFieldLooseKey("sendCompletionRequest");
var _setS3MultipartState = /*#__PURE__*/_classPrivateFieldLooseKey("setS3MultipartState");
var _uploadPartBytes = /*#__PURE__*/_classPrivateFieldLooseKey("uploadPartBytes");
var _getFile = /*#__PURE__*/_classPrivateFieldLooseKey("getFile");
var _shouldRetry = /*#__PURE__*/_classPrivateFieldLooseKey("shouldRetry");
var _nonMultipartUpload = /*#__PURE__*/_classPrivateFieldLooseKey("nonMultipartUpload");
export class HTTPCommunicationQueue {
constructor(_requests2, options, setS3MultipartState, getFile) {
Object.defineProperty(this, _nonMultipartUpload, {
value: _nonMultipartUpload2
});
Object.defineProperty(this, _shouldRetry, {
value: _shouldRetry2
});
Object.defineProperty(this, _abortMultipartUpload, {
writable: true,
value: void 0
});
Object.defineProperty(this, _cache, {
writable: true,
value: new WeakMap()
});
Object.defineProperty(this, _createMultipartUpload, {
writable: true,
value: void 0
});
Object.defineProperty(this, _fetchSignature, {
writable: true,
value: void 0
});
Object.defineProperty(this, _getUploadParameters, {
writable: true,
value: void 0
});
Object.defineProperty(this, _listParts, {
writable: true,
value: void 0
});
Object.defineProperty(this, _previousRetryDelay, {
writable: true,
value: void 0
});
Object.defineProperty(this, _requests, {
writable: true,
value: void 0
});
Object.defineProperty(this, _retryDelays, {
writable: true,
value: void 0
});
Object.defineProperty(this, _sendCompletionRequest, {
writable: true,
value: void 0
});
Object.defineProperty(this, _setS3MultipartState, {
writable: true,
value: void 0
});
Object.defineProperty(this, _uploadPartBytes, {
writable: true,
value: void 0
});
Object.defineProperty(this, _getFile, {
writable: true,
value: void 0
});
_classPrivateFieldLooseBase(this, _requests)[_requests] = _requests2;
_classPrivateFieldLooseBase(this, _setS3MultipartState)[_setS3MultipartState] = setS3MultipartState;
_classPrivateFieldLooseBase(this, _getFile)[_getFile] = getFile;
this.setOptions(options);
}
setOptions(options) {
const requests = _classPrivateFieldLooseBase(this, _requests)[_requests];
if ('abortMultipartUpload' in options) {
_classPrivateFieldLooseBase(this, _abortMultipartUpload)[_abortMultipartUpload] = requests.wrapPromiseFunction(options.abortMultipartUpload, {
priority: 1
});
}
if ('createMultipartUpload' in options) {
_classPrivateFieldLooseBase(this, _createMultipartUpload)[_createMultipartUpload] = requests.wrapPromiseFunction(options.createMultipartUpload, {
priority: -1
});
}
if ('signPart' in options) {
_classPrivateFieldLooseBase(this, _fetchSignature)[_fetchSignature] = requests.wrapPromiseFunction(options.signPart);
}
if ('listParts' in options) {
_classPrivateFieldLooseBase(this, _listParts)[_listParts] = requests.wrapPromiseFunction(options.listParts);
}
if ('completeMultipartUpload' in options) {
_classPrivateFieldLooseBase(this, _sendCompletionRequest)[_sendCompletionRequest] = requests.wrapPromiseFunction(options.completeMultipartUpload, {
priority: 1
});
}
if ('retryDelays' in options) {
var _options$retryDelays;
_classPrivateFieldLooseBase(this, _retryDelays)[_retryDelays] = (_options$retryDelays = options.retryDelays) != null ? _options$retryDelays : [];
}
if ('uploadPartBytes' in options) {
_classPrivateFieldLooseBase(this, _uploadPartBytes)[_uploadPartBytes] = requests.wrapPromiseFunction(options.uploadPartBytes, {
priority: Infinity
});
}
if ('getUploadParameters' in options) {
_classPrivateFieldLooseBase(this, _getUploadParameters)[_getUploadParameters] = requests.wrapPromiseFunction(options.getUploadParameters);
}
}
async getUploadId(file, signal) {
let cachedResult;
// As the cache is updated asynchronously, there could be a race condition
// where we just miss a new result so we loop here until we get nothing back,
// at which point it's out turn to create a new cache entry.
// eslint-disable-next-line no-cond-assign
while ((cachedResult = _classPrivateFieldLooseBase(this, _cache)[_cache].get(file.data)) != null) {
try {
return await cachedResult;
} catch {
// In case of failure, we want to ignore the cached error.
// At this point, either there's a new cached value, or we'll exit the loop a create a new one.
}
}
const promise = _classPrivateFieldLooseBase(this, _createMultipartUpload)[_createMultipartUpload](_classPrivateFieldLooseBase(this, _getFile)[_getFile](file), signal);
const abortPromise = () => {
promise.abort(signal.reason);
_classPrivateFieldLooseBase(this, _cache)[_cache].delete(file.data);
};
signal.addEventListener('abort', abortPromise, {
once: true
});
_classPrivateFieldLooseBase(this, _cache)[_cache].set(file.data, promise);
promise.then(async result => {
signal.removeEventListener('abort', abortPromise);
_classPrivateFieldLooseBase(this, _setS3MultipartState)[_setS3MultipartState](file, result);
_classPrivateFieldLooseBase(this, _cache)[_cache].set(file.data, result);
}, () => {
signal.removeEventListener('abort', abortPromise);
_classPrivateFieldLooseBase(this, _cache)[_cache].delete(file.data);
});
return promise;
}
async abortFileUpload(file) {
const result = _classPrivateFieldLooseBase(this, _cache)[_cache].get(file.data);
if (result == null) {
// If the createMultipartUpload request never was made, we don't
// need to send the abortMultipartUpload request.
return;
}
// Remove the cache entry right away for follow-up requests do not try to
// use the soon-to-be aborted cached values.
_classPrivateFieldLooseBase(this, _cache)[_cache].delete(file.data);
_classPrivateFieldLooseBase(this, _setS3MultipartState)[_setS3MultipartState](file, Object.create(null));
let awaitedResult;
try {
awaitedResult = await result;
} catch {
// If the cached result rejects, there's nothing to abort.
return;
}
await _classPrivateFieldLooseBase(this, _abortMultipartUpload)[_abortMultipartUpload](_classPrivateFieldLooseBase(this, _getFile)[_getFile](file), awaitedResult);
}
async uploadFile(file, chunks, signal) {
throwIfAborted(signal);
if (chunks.length === 1 && !chunks[0].shouldUseMultipart) {
return _classPrivateFieldLooseBase(this, _nonMultipartUpload)[_nonMultipartUpload](file, chunks[0], signal);
}
const {
uploadId,
key
} = await this.getUploadId(file, signal);
throwIfAborted(signal);
try {
const parts = await Promise.all(chunks.map((chunk, i) => this.uploadChunk(file, i + 1, chunk, signal)));
throwIfAborted(signal);
return await _classPrivateFieldLooseBase(this, _sendCompletionRequest)[_sendCompletionRequest](_classPrivateFieldLooseBase(this, _getFile)[_getFile](file), {
key,
uploadId,
parts,
signal
}, signal).abortOn(signal);
} catch (err) {
if ((err == null ? void 0 : err.cause) !== pausingUploadReason && (err == null ? void 0 : err.name) !== 'AbortError') {
// We purposefully don't wait for the promise and ignore its status,
// because we want the error `err` to bubble up ASAP to report it to the
// user. A failure to abort is not that big of a deal anyway.
this.abortFileUpload(file);
}
throw err;
}
}
restoreUploadFile(file, uploadIdAndKey) {
_classPrivateFieldLooseBase(this, _cache)[_cache].set(file.data, uploadIdAndKey);
}
async resumeUploadFile(file, chunks, signal) {
throwIfAborted(signal);
if (chunks.length === 1 && chunks[0] != null && !chunks[0].shouldUseMultipart) {
return _classPrivateFieldLooseBase(this, _nonMultipartUpload)[_nonMultipartUpload](file, chunks[0], signal);
}
const {
uploadId,
key
} = await this.getUploadId(file, signal);
throwIfAborted(signal);
const alreadyUploadedParts = await _classPrivateFieldLooseBase(this, _listParts)[_listParts](_classPrivateFieldLooseBase(this, _getFile)[_getFile](file), {
uploadId,
key,
signal
}, signal).abortOn(signal);
throwIfAborted(signal);
const parts = await Promise.all(chunks.map((chunk, i) => {
const partNumber = i + 1;
const alreadyUploadedInfo = alreadyUploadedParts.find(_ref => {
let {
PartNumber
} = _ref;
return PartNumber === partNumber;
});
if (alreadyUploadedInfo == null) {
return this.uploadChunk(file, partNumber, chunk, signal);
}
// Already uploaded chunks are set to null. If we are restoring the upload, we need to mark it as already uploaded.
chunk == null || chunk.setAsUploaded == null || chunk.setAsUploaded();
return {
PartNumber: partNumber,
ETag: alreadyUploadedInfo.ETag
};
}));
throwIfAborted(signal);
return _classPrivateFieldLooseBase(this, _sendCompletionRequest)[_sendCompletionRequest](_classPrivateFieldLooseBase(this, _getFile)[_getFile](file), {
key,
uploadId,
parts,
signal
}, signal).abortOn(signal);
}
async uploadChunk(file, partNumber, chunk, signal) {
throwIfAborted(signal);
const {
uploadId,
key
} = await this.getUploadId(file, signal);
const signatureRetryIterator = _classPrivateFieldLooseBase(this, _retryDelays)[_retryDelays].values();
const chunkRetryIterator = _classPrivateFieldLooseBase(this, _retryDelays)[_retryDelays].values();
const shouldRetrySignature = () => {
const next = signatureRetryIterator.next();
if (next == null || next.done) {
return null;
}
return next.value;
};
for (;;) {
throwIfAborted(signal);
const chunkData = chunk.getData();
const {
onProgress,
onComplete
} = chunk;
let signature;
try {
signature = await _classPrivateFieldLooseBase(this, _fetchSignature)[_fetchSignature](_classPrivateFieldLooseBase(this, _getFile)[_getFile](file), {
// Always defined for multipart uploads
uploadId: uploadId,
key,
partNumber,
body: chunkData,
signal
}).abortOn(signal);
} catch (err) {
const timeout = shouldRetrySignature();
if (timeout == null || signal.aborted) {
throw err;
}
await new Promise(resolve => setTimeout(resolve, timeout));
// eslint-disable-next-line no-continue
continue;
}
throwIfAborted(signal);
try {
return {
PartNumber: partNumber,
...(await _classPrivateFieldLooseBase(this, _uploadPartBytes)[_uploadPartBytes]({
signature,
body: chunkData,
size: chunkData.size,
onProgress,
onComplete,
signal
}).abortOn(signal))
};
} catch (err) {
if (!(await _classPrivateFieldLooseBase(this, _shouldRetry)[_shouldRetry](err, chunkRetryIterator))) throw err;
}
}
}
}
async function _shouldRetry2(err, retryDelayIterator) {
var _err$source;
const requests = _classPrivateFieldLooseBase(this, _requests)[_requests];
const status = err == null || (_err$source = err.source) == null ? void 0 : _err$source.status;
// TODO: this retry logic is taken out of Tus. We should have a centralized place for retrying,
// perhaps the rate limited queue, and dedupe all plugins with that.
if (status == null) {
return false;
}
if (status === 403 && err.message === 'Request has expired') {
if (!requests.isPaused) {
// We don't want to exhaust the retryDelayIterator as long as there are
// more than one request in parallel, to give slower connection a chance
// to catch up with the expiry set in Companion.
if (requests.limit === 1 || _classPrivateFieldLooseBase(this, _previousRetryDelay)[_previousRetryDelay] == null) {
const next = retryDelayIterator.next();
if (next == null || next.done) {
return false;
}
// If there are more than 1 request done in parallel, the RLQ limit is
// decreased and the failed request is requeued after waiting for a bit.
// If there is only one request in parallel, the limit can't be
// decreased, so we iterate over `retryDelayIterator` as we do for
// other failures.
// `#previousRetryDelay` caches the value so we can re-use it next time.
_classPrivateFieldLooseBase(this, _previousRetryDelay)[_previousRetryDelay] = next.value;
}
// No need to stop the other requests, we just want to lower the limit.
requests.rateLimit(0);
await new Promise(resolve => setTimeout(resolve, _classPrivateFieldLooseBase(this, _previousRetryDelay)[_previousRetryDelay]));
}
} else if (status === 429) {
// HTTP 429 Too Many Requests => to avoid the whole download to fail, pause all requests.
if (!requests.isPaused) {
const next = retryDelayIterator.next();
if (next == null || next.done) {
return false;
}
requests.rateLimit(next.value);
}
} else if (status > 400 && status < 500 && status !== 409) {
// HTTP 4xx, the server won't send anything, it's doesn't make sense to retry
return false;
} else if (typeof navigator !== 'undefined' && navigator.onLine === false) {
// The navigator is offline, let's wait for it to come back online.
if (!requests.isPaused) {
requests.pause();
window.addEventListener('online', () => {
requests.resume();
}, {
once: true
});
}
} else {
// Other error code means the request can be retried later.
const next = retryDelayIterator.next();
if (next == null || next.done) {
return false;
}
await new Promise(resolve => setTimeout(resolve, next.value));
}
return true;
}
async function _nonMultipartUpload2(file, chunk, signal) {
var _ref3;
const {
method = 'POST',
url,
fields,
headers
} = await _classPrivateFieldLooseBase(this, _getUploadParameters)[_getUploadParameters](_classPrivateFieldLooseBase(this, _getFile)[_getFile](file), {
signal
}).abortOn(signal);
let body;
const data = chunk.getData();
if (method.toUpperCase() === 'POST') {
const formData = new FormData();
Object.entries(fields).forEach(_ref2 => {
let [key, value] = _ref2;
return formData.set(key, value);
});
formData.set('file', data);
body = formData;
} else {
body = data;
}
const {
onProgress,
onComplete
} = chunk;
const result = await _classPrivateFieldLooseBase(this, _uploadPartBytes)[_uploadPartBytes]({
signature: {
url,
headers,
method
},
body,
size: data.size,
onProgress,
onComplete,
signal
}).abortOn(signal); // todo this doesn't make sense
// Note: `fields.key` is not returned by old Companion versions.
// See https://github.com/transloadit/uppy/pull/5602
const key = fields == null ? void 0 : fields.key;
_classPrivateFieldLooseBase(this, _setS3MultipartState)[_setS3MultipartState](file, {
key: key
});
return {
...result,
location: (_ref3 = result.location) != null ? _ref3 : removeMetadataFromURL(url),
bucket: fields == null ? void 0 : fields.bucket,
key
};
}