UNPKG

@uppy/aws-s3

Version:

Upload to Amazon S3 with Uppy

691 lines (685 loc) 27 kB
function _classPrivateFieldLooseBase(e, t) { if (!{}.hasOwnProperty.call(e, t)) throw new TypeError("attempted to use private field on non-instance"); return e; } var id = 0; function _classPrivateFieldLooseKey(e) { return "__private_" + id++ + "_" + e; } import { BasePlugin } from '@uppy/core'; import { RequestClient } from '@uppy/companion-client'; import EventManager from '@uppy/core/lib/EventManager.js'; import { RateLimitedQueue } from '@uppy/utils/lib/RateLimitedQueue'; import { filterNonFailedFiles, filterFilesToEmitUploadStarted } from '@uppy/utils/lib/fileFilters'; import { createAbortError } from '@uppy/utils/lib/AbortController'; import getAllowedMetaFields from '@uppy/utils/lib/getAllowedMetaFields'; import MultipartUploader from './MultipartUploader.js'; import { throwIfAborted } from './utils.js'; import createSignedURL from './createSignedURL.js'; import { HTTPCommunicationQueue } from './HTTPCommunicationQueue.js'; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore We don't want TS to generate types for the package.json const packageJson = { "version": "4.2.3" }; function assertServerError(res) { if (res != null && res.error) { const error = new Error(res.message); Object.assign(error, res.error); throw error; } return res; } /** * Computes the expiry time for a request signed with temporary credentials. If * no expiration was provided, or an invalid value (e.g. in the past) is * provided, undefined is returned. This function assumes the client clock is in * sync with the remote server, which is a requirement for the signature to be * validated for AWS anyway. */ function getExpiry(credentials) { const expirationDate = credentials.Expiration; if (expirationDate) { const timeUntilExpiry = Math.floor((new Date(expirationDate) - Date.now()) / 1000); if (timeUntilExpiry > 9) { return timeUntilExpiry; } } return undefined; } function getAllowedMetadata(_ref) { let { meta, allowedMetaFields, querify = false } = _ref; const metaFields = allowedMetaFields != null ? allowedMetaFields : Object.keys(meta); if (!meta) return {}; return Object.fromEntries(metaFields.filter(key => meta[key] != null).map(key => { const realKey = querify ? `metadata[${key}]` : key; const value = String(meta[key]); return [realKey, value]; })); } const defaultOptions = { allowedMetaFields: true, limit: 6, getTemporarySecurityCredentials: false, shouldUseMultipart: file => (file.size || 0) > 100 * 1024 * 1024, retryDelays: [0, 1000, 3000, 5000] }; var _companionCommunicationQueue = /*#__PURE__*/_classPrivateFieldLooseKey("companionCommunicationQueue"); var _client = /*#__PURE__*/_classPrivateFieldLooseKey("client"); var _setClient = /*#__PURE__*/_classPrivateFieldLooseKey("setClient"); var _assertHost = /*#__PURE__*/_classPrivateFieldLooseKey("assertHost"); var _cachedTemporaryCredentials = /*#__PURE__*/_classPrivateFieldLooseKey("cachedTemporaryCredentials"); var _getTemporarySecurityCredentials = /*#__PURE__*/_classPrivateFieldLooseKey("getTemporarySecurityCredentials"); var _setS3MultipartState = /*#__PURE__*/_classPrivateFieldLooseKey("setS3MultipartState"); var _getFile = /*#__PURE__*/_classPrivateFieldLooseKey("getFile"); var _uploadLocalFile = /*#__PURE__*/_classPrivateFieldLooseKey("uploadLocalFile"); var _getCompanionClientArgs = /*#__PURE__*/_classPrivateFieldLooseKey("getCompanionClientArgs"); var _upload = /*#__PURE__*/_classPrivateFieldLooseKey("upload"); var _setCompanionHeaders = /*#__PURE__*/_classPrivateFieldLooseKey("setCompanionHeaders"); var _setResumableUploadsCapability = /*#__PURE__*/_classPrivateFieldLooseKey("setResumableUploadsCapability"); var _resetResumableCapability = /*#__PURE__*/_classPrivateFieldLooseKey("resetResumableCapability"); export default class AwsS3Multipart extends BasePlugin { constructor(uppy, _opts) { var _rateLimitedQueue; super(uppy, { ...defaultOptions, uploadPartBytes: AwsS3Multipart.uploadPartBytes, createMultipartUpload: null, listParts: null, abortMultipartUpload: null, completeMultipartUpload: null, signPart: null, getUploadParameters: null, ..._opts }); // We need the `as any` here because of the dynamic default options. // eslint-disable-next-line class-methods-use-this Object.defineProperty(this, _getCompanionClientArgs, { value: _getCompanionClientArgs2 }); Object.defineProperty(this, _uploadLocalFile, { value: _uploadLocalFile2 }); Object.defineProperty(this, _getTemporarySecurityCredentials, { value: _getTemporarySecurityCredentials2 }); Object.defineProperty(this, _assertHost, { value: _assertHost2 }); Object.defineProperty(this, _setClient, { value: _setClient2 }); Object.defineProperty(this, _companionCommunicationQueue, { writable: true, value: void 0 }); Object.defineProperty(this, _client, { writable: true, value: void 0 }); Object.defineProperty(this, _cachedTemporaryCredentials, { writable: true, value: void 0 }); Object.defineProperty(this, _setS3MultipartState, { writable: true, value: (file, _ref2) => { let { key, uploadId } = _ref2; const cFile = this.uppy.getFile(file.id); if (cFile == null) { // file was removed from store return; } this.uppy.setFileState(file.id, { s3Multipart: { ...cFile.s3Multipart, key, uploadId } }); } }); Object.defineProperty(this, _getFile, { writable: true, value: file => { return this.uppy.getFile(file.id) || file; } }); Object.defineProperty(this, _upload, { writable: true, value: async fileIDs => { if (fileIDs.length === 0) return undefined; const files = this.uppy.getFilesByIds(fileIDs); const filesFiltered = filterNonFailedFiles(files); const filesToEmit = filterFilesToEmitUploadStarted(filesFiltered); this.uppy.emit('upload-start', filesToEmit); const promises = filesFiltered.map(file => { if (file.isRemote) { const getQueue = () => this.requests; _classPrivateFieldLooseBase(this, _setResumableUploadsCapability)[_setResumableUploadsCapability](false); const controller = new AbortController(); const removedHandler = removedFile => { if (removedFile.id === file.id) controller.abort(); }; this.uppy.on('file-removed', removedHandler); const uploadPromise = this.uppy.getRequestClientForFile(file).uploadRemoteFile(file, _classPrivateFieldLooseBase(this, _getCompanionClientArgs)[_getCompanionClientArgs](file), { signal: controller.signal, getQueue }); this.requests.wrapSyncFunction(() => { this.uppy.off('file-removed', removedHandler); }, { priority: -1 })(); return uploadPromise; } return _classPrivateFieldLooseBase(this, _uploadLocalFile)[_uploadLocalFile](file); }); const upload = await Promise.allSettled(promises); // After the upload is done, another upload may happen with only local files. // We reset the capability so that the next upload can use resumable uploads. _classPrivateFieldLooseBase(this, _setResumableUploadsCapability)[_setResumableUploadsCapability](true); return upload; } }); Object.defineProperty(this, _setCompanionHeaders, { writable: true, value: () => { var _classPrivateFieldLoo; (_classPrivateFieldLoo = _classPrivateFieldLooseBase(this, _client)[_client]) == null || _classPrivateFieldLoo.setCompanionHeaders(this.opts.headers); } }); Object.defineProperty(this, _setResumableUploadsCapability, { writable: true, value: boolean => { const { capabilities } = this.uppy.getState(); this.uppy.setState({ capabilities: { ...capabilities, resumableUploads: boolean } }); } }); Object.defineProperty(this, _resetResumableCapability, { writable: true, value: () => { _classPrivateFieldLooseBase(this, _setResumableUploadsCapability)[_setResumableUploadsCapability](true); } }); this.type = 'uploader'; this.id = this.opts.id || 'AwsS3Multipart'; _classPrivateFieldLooseBase(this, _setClient)[_setClient](_opts); const dynamicDefaultOptions = { createMultipartUpload: this.createMultipartUpload, listParts: this.listParts, abortMultipartUpload: this.abortMultipartUpload, completeMultipartUpload: this.completeMultipartUpload, signPart: _opts != null && _opts.getTemporarySecurityCredentials ? this.createSignedURL : this.signPart, getUploadParameters: _opts != null && _opts.getTemporarySecurityCredentials ? this.createSignedURL : this.getUploadParameters }; for (const key of Object.keys(dynamicDefaultOptions)) { if (this.opts[key] == null) { this.opts[key] = dynamicDefaultOptions[key].bind(this); } } /** * Simultaneous upload limiting is shared across all uploads with this plugin. * * @type {RateLimitedQueue} */ this.requests = (_rateLimitedQueue = this.opts.rateLimitedQueue) != null ? _rateLimitedQueue : new RateLimitedQueue(this.opts.limit); _classPrivateFieldLooseBase(this, _companionCommunicationQueue)[_companionCommunicationQueue] = new HTTPCommunicationQueue(this.requests, this.opts, _classPrivateFieldLooseBase(this, _setS3MultipartState)[_setS3MultipartState], _classPrivateFieldLooseBase(this, _getFile)[_getFile]); this.uploaders = Object.create(null); this.uploaderEvents = Object.create(null); } [Symbol.for('uppy test: getClient')]() { return _classPrivateFieldLooseBase(this, _client)[_client]; } setOptions(newOptions) { _classPrivateFieldLooseBase(this, _companionCommunicationQueue)[_companionCommunicationQueue].setOptions(newOptions); super.setOptions(newOptions); _classPrivateFieldLooseBase(this, _setClient)[_setClient](newOptions); } /** * Clean up all references for a file's upload: the MultipartUploader instance, * any events related to the file, and the Companion WebSocket connection. * * Set `opts.abort` to tell S3 that the multipart upload is cancelled and must be removed. * This should be done when the user cancels the upload, not when the upload is completed or errored. */ resetUploaderReferences(fileID, opts) { if (this.uploaders[fileID]) { this.uploaders[fileID].abort({ really: (opts == null ? void 0 : opts.abort) || false }); this.uploaders[fileID] = null; } if (this.uploaderEvents[fileID]) { this.uploaderEvents[fileID].remove(); this.uploaderEvents[fileID] = null; } } createMultipartUpload(file, signal) { _classPrivateFieldLooseBase(this, _assertHost)[_assertHost]('createMultipartUpload'); throwIfAborted(signal); const allowedMetaFields = getAllowedMetaFields(this.opts.allowedMetaFields, file.meta); const metadata = getAllowedMetadata({ meta: file.meta, allowedMetaFields }); return _classPrivateFieldLooseBase(this, _client)[_client].post('s3/multipart', { filename: file.name, type: file.type, metadata }, { signal }).then(assertServerError); } listParts(file, _ref3, oldSignal) { var _signal; let { key, uploadId, signal } = _ref3; (_signal = signal) != null ? _signal : signal = oldSignal; // eslint-disable-line no-param-reassign _classPrivateFieldLooseBase(this, _assertHost)[_assertHost]('listParts'); throwIfAborted(signal); const filename = encodeURIComponent(key); return _classPrivateFieldLooseBase(this, _client)[_client].get(`s3/multipart/${encodeURIComponent(uploadId)}?key=${filename}`, { signal }).then(assertServerError); } completeMultipartUpload(file, _ref4, oldSignal) { var _signal2; let { key, uploadId, parts, signal } = _ref4; (_signal2 = signal) != null ? _signal2 : signal = oldSignal; // eslint-disable-line no-param-reassign _classPrivateFieldLooseBase(this, _assertHost)[_assertHost]('completeMultipartUpload'); throwIfAborted(signal); const filename = encodeURIComponent(key); const uploadIdEnc = encodeURIComponent(uploadId); return _classPrivateFieldLooseBase(this, _client)[_client].post(`s3/multipart/${uploadIdEnc}/complete?key=${filename}`, { parts: parts.map(_ref5 => { let { ETag, PartNumber } = _ref5; return { ETag, PartNumber }; }) }, { signal }).then(assertServerError); } async createSignedURL(file, options) { const data = await _classPrivateFieldLooseBase(this, _getTemporarySecurityCredentials)[_getTemporarySecurityCredentials](options); const expires = getExpiry(data.credentials) || 604800; // 604 800 is the max value accepted by AWS. const { uploadId, key, partNumber } = options; // Return an object in the correct shape. return { method: 'PUT', expires, fields: {}, url: `${await createSignedURL({ accountKey: data.credentials.AccessKeyId, accountSecret: data.credentials.SecretAccessKey, sessionToken: data.credentials.SessionToken, expires, bucketName: data.bucket, Region: data.region, Key: key != null ? key : `${crypto.randomUUID()}-${file.name}`, uploadId, partNumber })}`, // Provide content type header required by S3 headers: { 'Content-Type': file.type } }; } signPart(file, _ref6) { let { uploadId, key, partNumber, signal } = _ref6; _classPrivateFieldLooseBase(this, _assertHost)[_assertHost]('signPart'); throwIfAborted(signal); if (uploadId == null || key == null || partNumber == null) { throw new Error('Cannot sign without a key, an uploadId, and a partNumber'); } const filename = encodeURIComponent(key); return _classPrivateFieldLooseBase(this, _client)[_client].get(`s3/multipart/${encodeURIComponent(uploadId)}/${partNumber}?key=${filename}`, { signal }).then(assertServerError); } abortMultipartUpload(file, _ref7) { let { key, uploadId, signal } = _ref7; _classPrivateFieldLooseBase(this, _assertHost)[_assertHost]('abortMultipartUpload'); const filename = encodeURIComponent(key); const uploadIdEnc = encodeURIComponent(uploadId); return _classPrivateFieldLooseBase(this, _client)[_client].delete(`s3/multipart/${uploadIdEnc}?key=${filename}`, undefined, { signal }).then(assertServerError); } getUploadParameters(file, options) { _classPrivateFieldLooseBase(this, _assertHost)[_assertHost]('getUploadParameters'); const { meta } = file; const { type, name: filename } = meta; const allowedMetaFields = getAllowedMetaFields(this.opts.allowedMetaFields, file.meta); const metadata = getAllowedMetadata({ meta, allowedMetaFields, querify: true }); const query = new URLSearchParams({ filename, type, ...metadata }); return _classPrivateFieldLooseBase(this, _client)[_client].get(`s3/params?${query}`, options); } static async uploadPartBytes(_ref8) { let { signature: { url, expires, headers, method = 'PUT' }, body, size = body.size, onProgress, onComplete, signal } = _ref8; throwIfAborted(signal); if (url == null) { throw new Error('Cannot upload to an undefined URL'); } return new Promise((resolve, reject) => { const xhr = new XMLHttpRequest(); xhr.open(method, url, true); if (headers) { Object.keys(headers).forEach(key => { xhr.setRequestHeader(key, headers[key]); }); } xhr.responseType = 'text'; if (typeof expires === 'number') { xhr.timeout = expires * 1000; } function onabort() { xhr.abort(); } function cleanup() { signal == null || signal.removeEventListener('abort', onabort); } signal == null || signal.addEventListener('abort', onabort); xhr.upload.addEventListener('progress', ev => { onProgress(ev); }); xhr.addEventListener('abort', () => { cleanup(); reject(createAbortError()); }); xhr.addEventListener('timeout', () => { cleanup(); const error = new Error('Request has expired'); error.source = { status: 403 }; reject(error); }); xhr.addEventListener('load', () => { cleanup(); if (xhr.status === 403 && xhr.responseText.includes('<Message>Request has expired</Message>')) { const error = new Error('Request has expired'); error.source = xhr; reject(error); return; } if (xhr.status < 200 || xhr.status >= 300) { const error = new Error('Non 2xx'); error.source = xhr; reject(error); return; } onProgress == null || onProgress({ loaded: size, lengthComputable: true }); // https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/getAllResponseHeaders#examples const arr = xhr.getAllResponseHeaders().trim().split(/[\r\n]+/); // @ts-expect-error null is allowed to avoid inherited properties const headersMap = { __proto__: null }; for (const line of arr) { const parts = line.split(': '); const header = parts.shift(); const value = parts.join(': '); headersMap[header] = value; } const { etag, location } = headersMap; // More info bucket settings when this is not present: // https://github.com/transloadit/uppy/issues/5388#issuecomment-2464885562 if (method.toUpperCase() === 'POST' && location == null) { // Not being able to read the Location header is not a fatal error. // eslint-disable-next-line no-console console.error('@uppy/aws-s3: Could not read the Location header. This likely means CORS is not configured correctly on the S3 Bucket. See https://uppy.io/docs/aws-s3/#setting-up-your-s3-bucket'); } if (etag == null) { // eslint-disable-next-line no-console console.error('@uppy/aws-s3: Could not read the ETag header. This likely means CORS is not configured correctly on the S3 Bucket. See https://uppy.io/docs/aws-s3/#setting-up-your-s3-bucket'); return; } onComplete == null || onComplete(etag); resolve({ ...headersMap, ETag: etag // keep capitalised ETag for backwards compatiblity }); }); xhr.addEventListener('error', ev => { cleanup(); const error = new Error('Unknown error'); error.source = ev.target; reject(error); }); xhr.send(body); }); } install() { _classPrivateFieldLooseBase(this, _setResumableUploadsCapability)[_setResumableUploadsCapability](true); this.uppy.addPreProcessor(_classPrivateFieldLooseBase(this, _setCompanionHeaders)[_setCompanionHeaders]); this.uppy.addUploader(_classPrivateFieldLooseBase(this, _upload)[_upload]); this.uppy.on('cancel-all', _classPrivateFieldLooseBase(this, _resetResumableCapability)[_resetResumableCapability]); } uninstall() { this.uppy.removePreProcessor(_classPrivateFieldLooseBase(this, _setCompanionHeaders)[_setCompanionHeaders]); this.uppy.removeUploader(_classPrivateFieldLooseBase(this, _upload)[_upload]); this.uppy.off('cancel-all', _classPrivateFieldLooseBase(this, _resetResumableCapability)[_resetResumableCapability]); } } function _setClient2(opts) { if (opts == null || !('endpoint' in opts || 'companionUrl' in opts || 'headers' in opts || 'companionHeaders' in opts || 'cookiesRule' in opts || 'companionCookiesRule' in opts)) return; if ('companionUrl' in opts && !('endpoint' in opts)) { this.uppy.log('`companionUrl` option has been removed in @uppy/aws-s3, use `endpoint` instead.', 'warning'); } if ('companionHeaders' in opts && !('headers' in opts)) { this.uppy.log('`companionHeaders` option has been removed in @uppy/aws-s3, use `headers` instead.', 'warning'); } if ('companionCookiesRule' in opts && !('cookiesRule' in opts)) { this.uppy.log('`companionCookiesRule` option has been removed in @uppy/aws-s3, use `cookiesRule` instead.', 'warning'); } if ('endpoint' in opts) { _classPrivateFieldLooseBase(this, _client)[_client] = new RequestClient(this.uppy, { pluginId: this.id, provider: 'AWS', companionUrl: this.opts.endpoint, companionHeaders: this.opts.headers, companionCookiesRule: this.opts.cookiesRule }); } else { if ('headers' in opts) { _classPrivateFieldLooseBase(this, _setCompanionHeaders)[_setCompanionHeaders](); } if ('cookiesRule' in opts) { _classPrivateFieldLooseBase(this, _client)[_client].opts.companionCookiesRule = opts.cookiesRule; } } } function _assertHost2(method) { if (!_classPrivateFieldLooseBase(this, _client)[_client]) { throw new Error(`Expected a \`endpoint\` option containing a URL, or if you are not using Companion, a custom \`${method}\` implementation.`); } } async function _getTemporarySecurityCredentials2(options) { throwIfAborted(options == null ? void 0 : options.signal); if (_classPrivateFieldLooseBase(this, _cachedTemporaryCredentials)[_cachedTemporaryCredentials] == null) { const { getTemporarySecurityCredentials } = this.opts; // We do not await it just yet, so concurrent calls do not try to override it: if (getTemporarySecurityCredentials === true) { _classPrivateFieldLooseBase(this, _assertHost)[_assertHost]('getTemporarySecurityCredentials'); _classPrivateFieldLooseBase(this, _cachedTemporaryCredentials)[_cachedTemporaryCredentials] = _classPrivateFieldLooseBase(this, _client)[_client].get('s3/sts', options).then(assertServerError); } else { _classPrivateFieldLooseBase(this, _cachedTemporaryCredentials)[_cachedTemporaryCredentials] = getTemporarySecurityCredentials(options); } _classPrivateFieldLooseBase(this, _cachedTemporaryCredentials)[_cachedTemporaryCredentials] = await _classPrivateFieldLooseBase(this, _cachedTemporaryCredentials)[_cachedTemporaryCredentials]; setTimeout(() => { // At half the time left before expiration, we clear the cache. That's // an arbitrary tradeoff to limit the number of requests made to the // remote while limiting the risk of using an expired token in case the // clocks are not exactly synced. // The HTTP cache should be configured to ensure a client doesn't request // more tokens than it needs, but this timeout provides a second layer of // security in case the HTTP cache is disabled or misconfigured. _classPrivateFieldLooseBase(this, _cachedTemporaryCredentials)[_cachedTemporaryCredentials] = null; }, (getExpiry(_classPrivateFieldLooseBase(this, _cachedTemporaryCredentials)[_cachedTemporaryCredentials].credentials) || 0) * 500); } return _classPrivateFieldLooseBase(this, _cachedTemporaryCredentials)[_cachedTemporaryCredentials]; } function _uploadLocalFile2(file) { var _this = this; return new Promise((resolve, reject) => { const onProgress = (bytesUploaded, bytesTotal) => { var _latestFile$progress$; const latestFile = this.uppy.getFile(file.id); this.uppy.emit('upload-progress', latestFile, { uploadStarted: (_latestFile$progress$ = latestFile.progress.uploadStarted) != null ? _latestFile$progress$ : 0, bytesUploaded, bytesTotal }); }; const onError = err => { this.uppy.log(err); this.uppy.emit('upload-error', file, err); this.resetUploaderReferences(file.id); reject(err); }; const onSuccess = result => { const uploadResp = { body: { ...result }, status: 200, uploadURL: result.location }; this.resetUploaderReferences(file.id); this.uppy.emit('upload-success', _classPrivateFieldLooseBase(this, _getFile)[_getFile](file), uploadResp); if (result.location) { this.uppy.log(`Download ${file.name} from ${result.location}`); } resolve(); }; const upload = new MultipartUploader(file.data, { // .bind to pass the file object to each handler. companionComm: _classPrivateFieldLooseBase(this, _companionCommunicationQueue)[_companionCommunicationQueue], log: function () { return _this.uppy.log(...arguments); }, getChunkSize: this.opts.getChunkSize ? this.opts.getChunkSize.bind(this) : undefined, onProgress, onError, onSuccess, onPartComplete: part => { this.uppy.emit('s3-multipart:part-uploaded', _classPrivateFieldLooseBase(this, _getFile)[_getFile](file), part); }, file, shouldUseMultipart: this.opts.shouldUseMultipart, ...file.s3Multipart }); this.uploaders[file.id] = upload; const eventManager = new EventManager(this.uppy); this.uploaderEvents[file.id] = eventManager; eventManager.onFileRemove(file.id, removed => { upload.abort(); this.resetUploaderReferences(file.id, { abort: true }); resolve(`upload ${removed} was removed`); }); eventManager.onCancelAll(file.id, () => { upload.abort(); this.resetUploaderReferences(file.id, { abort: true }); resolve(`upload ${file.id} was canceled`); }); eventManager.onFilePause(file.id, isPaused => { if (isPaused) { upload.pause(); } else { upload.start(); } }); eventManager.onPauseAll(file.id, () => { upload.pause(); }); eventManager.onResumeAll(file.id, () => { upload.start(); }); upload.start(); }); } function _getCompanionClientArgs2(file) { var _file$remote; return { ...((_file$remote = file.remote) == null ? void 0 : _file$remote.body), protocol: 's3-multipart', size: file.data.size, metadata: file.meta }; } AwsS3Multipart.VERSION = packageJson.version;