UNPKG

s3-bucket-multipart-toolkit

Version:
845 lines (738 loc) 24.9 kB
var fs = require('fs'); var AWS = require('aws-sdk'); var Promise = require('bluebird'); var promisify = Promise.promisify; const COPY_PART_SIZE_MINIMUM_BYTES = 5242880; // 5MB in bytes const DEFAULT_COPY_PART_SIZE_BYTES = 500000000; // 500 MB in bytes function calculatePartitionsRangeArray(object_size) { const partitions = []; const copy_part_size = DEFAULT_COPY_PART_SIZE_BYTES; const numOfPartitions = Math.floor(object_size / copy_part_size); const remainder = object_size % copy_part_size; let index, partition; for (index = 0; index < numOfPartitions; index++) { const nextIndex = index + 1; if (nextIndex === numOfPartitions && remainder < COPY_PART_SIZE_MINIMUM_BYTES) { partition = (index * copy_part_size) + '-' + ((nextIndex) * copy_part_size + remainder - 1); } else { partition = (index * copy_part_size) + '-' + ((nextIndex) * copy_part_size - 1); } partitions.push(partition); } if (remainder >= COPY_PART_SIZE_MINIMUM_BYTES) { partition = (index * copy_part_size) + '-' + (index * copy_part_size + remainder - 1); partitions.push(partition); } return partitions; } var copyPart = function (params, S3) { var uploadPartCopyPromise = promisify(S3.uploadPartCopy).bind(S3); return uploadPartCopyPromise(params) .then((result) => { return Promise.resolve(result); }) .catch((err) => { return Promise.reject(err); }) } var prepareResultsForCopyCompletion = function (copyPartsResultsArray) { const resultArray = []; copyPartsResultsArray.forEach((copyPart, index) => { const newCopyPart = {}; newCopyPart.ETag = copyPart.CopyPartResult.ETag; newCopyPart.PartNumber = index + 1; resultArray.push(newCopyPart); }); return resultArray; } var completeMultipartCopy = function (params, S3) { var completeMultipartUploadPromise = promisify(S3.completeMultipartUpload).bind(S3); return completeMultipartUploadPromise(params) .then((result) => { return Promise.resolve(result); }) .catch((err) => { return Promise.reject(err); }); } var abortMultipartCopy = function(params, S3) { var abortMultipartUploadPromise = promisify(S3.abortMultipartUpload).bind(S3); return abortMultipartUploadPromise(params) .then(() => { return promisify(S3.listParts).bind(S3, params); }) .catch((err) => { return Promise.reject(err); }) .then((partsList) => { if (partsList.Parts && partsList.Parts.length > 0) { const err = new Error('Abort procedure passed but copy parts were not removed'); err.details = partsList; return Promise.reject(err); } else { const err = new Error('multipart copy aborted'); err.details = params; return Promise.reject(err); } }); } // FS var getFilesizeInBytes = function(filename) { if (typeof filename === 'undefined') { throw new Error('File path was expected'); } var stats = fs.statSync(filename); // console.log(filename); // console.log(stats); // TODO: double check size != 0 var fileSizeInBytes = stats.size; return fileSizeInBytes; }; // Params check var checkParams = function(params, mandatory) { if (typeof params === 'undefined') { throw new Error('Parameters are required'); } if (typeof mandatory === 'undefined') { throw new Error('Mandatory flags are required'); } // https://stackoverflow.com/a/41981796/467034 return mandatory.every(function(prop) { return typeof params[prop] !== 'undefined'; }); } // AWS Config var AWSConfig = { accessKeyId: null, secretAccessKey: null, region: null }; // initializing multipart upload var createMultipartCopy = function (params, S3) { var createMultipartUploadPromise = promisify(S3.createMultipartUpload).bind(S3); return createMultipartUploadPromise(params) .then((result) => { return Promise.resolve(result.UploadId); }) .catch((err) => { return Promise.reject(err); }); } // Bucket class var Bucket = function (params) { var flags = ['accessKeyId', 'secretAccessKey', 'region', 'bucketName']; var hasAllFlags = checkParams(params, flags); if (!hasAllFlags) { throw new Error('Unable to create bucket instance due parameters missing'); } // Setup S3 AWSConfig.accessKeyId = params.accessKeyId; AWSConfig.secretAccessKey = params.secretAccessKey; AWSConfig.region = params.region; // AWS S3 Docs // http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html this.S3 = new AWS.S3(AWSConfig); this.bucketName = params.bucketName; this.bucketACL = params.bucketACL || 'public-read'; // default paging delay in between calls this.pagingDelay = params.pagingDelay || 500; }; /* Get All buckets for this account Result: { Buckets: [ { Name: 'your-bucket-name', CreationDate: 2018-03-19T17:49:05.000Z } ], Owner: { DisplayName: 'cris', ID: '...' } } */ Bucket.prototype.getAllBuckets = function() { var S3 = this.S3; var listBuckets = promisify(S3.listBuckets).bind(S3); return listBuckets(); }; /* Usage: Result: { signedUrl: 'https://your-bucket-name.s3.amazonaws.com/your-dir/test.js?AWSAccessKeyId=...' } */ Bucket.prototype.getUploadUrl = function (customParams) { var flags = ['ContentType', 'Key']; var hasAllFlags = checkParams(customParams, flags); if (!hasAllFlags) { throw new Error('Unable to get upload url instance due parameters missing'); } var S3 = this.S3; var bucketName = this.bucketName || ''; var bucketACL = this.bucketACL || ''; var defaultParams = { Expires: 60, ACL: bucketACL, Bucket: bucketName }; var params = Object.assign(defaultParams, customParams); var getSignedUrlPromise = promisify(S3.getSignedUrl).bind(S3); return new Promise(function(resolve, reject) { getSignedUrlPromise('putObject', params) .then(function(signedUrl) { return resolve({signedUrl: signedUrl}); }) .catch(reject); }); }; /* Usage: Result: { response: { ETag: '"abc..."' }, url: 'https://your-bucket-name.s3.amazonaws.com/upload-test.txt' } */ Bucket.prototype.uploadFile = function(customParams) { var flags = ['filePath', 'Key']; var hasAllFlags = checkParams(customParams, flags); if (!hasAllFlags) { throw new Error('Unable to upload files due parameters missing'); } var S3 = this.S3; var bucketName = this.bucketName || ''; var bucketACL = this.bucketACL || ''; var filePath = customParams.filePath; var defaultParams = { ACL: bucketACL, Bucket: bucketName, ContentLength: getFilesizeInBytes(filePath), Body: fs.createReadStream(filePath) }; var params = Object.assign(defaultParams, customParams); delete params.filePath; // Params var Bucket = params.Bucket; var Key = params.Key; // console.log("AWS UPLOAD==>", params); var putObjectPromise = promisify(S3.upload).bind(S3); return new Promise(function (resolve, reject) { return putObjectPromise(params) .then(function(response) { var url = `https://${Bucket}.s3.amazonaws.com/${Key}`; resolve(Object.assign({ response: response, url: url })); }) .catch(reject); }); }; /* Usage: Result: { response: { ETag: '"abc..."', CopySourceVersionId: 'def...', CopyObjectResult: { ... } }, url: 'https://your-bucket-name.s3.amazonaws.com/upload-test-copied.txt' } } */ Bucket.prototype.copyFile = function(customParams, appendPrefix) { var autoAppendPrefix = appendPrefix || false; var flags = ['CopySource', 'Key']; var hasAllFlags = checkParams(customParams, flags); if (!hasAllFlags) { throw new Error('Unable to copy files due parameters missing'); } var S3 = this.S3; var bucketName = this.bucketName || ''; var bucketACL = this.bucketACL || ''; var defaultParams = { ACL: bucketACL, Bucket: bucketName, }; if (autoAppendPrefix) { // customParams.Key = `https://${bucketName}.s3.amazonaws.com/${customParams.Key}`; customParams.CopySource = `${bucketName}/${customParams.CopySource}`; } var params = Object.assign(defaultParams, customParams); // Params var Bucket = params.Bucket; var Key = params.Key; // console.log("AWS COPY==>", params); var copyObjectPromise = promisify(S3.copyObject).bind(S3); return new Promise(function (resolve, reject) { return copyObjectPromise(params) .then(function(response) { // console.log('cp => ', response); var url = `https://${Bucket}.s3.amazonaws.com/${Key}`; resolve(Object.assign({ response: response, url: url })); }) //.catch((err) => console.log(err)); .catch(reject); }); }; /* Usage: Result: { response: { ETag: '"abc..."', CopySourceVersionId: 'def...', CopyObjectResult: { ... } }, url: 'https://your-bucket-name.s3.amazonaws.com/upload-test-copied.txt' } } */ Bucket.prototype.copyFileMultipart = function(customParams, appendPrefix, fileSize) { var autoAppendPrefix = appendPrefix || false; var S3 = this.S3; var bucketName = this.bucketName || ''; var bucketACL = this.bucketACL || ''; var defaultParams = { ACL: bucketACL, Bucket: bucketName, }; var copySource = `${bucketName}/${customParams.CopySource}`; const bucketKey = customParams.Key; if (autoAppendPrefix) { customParams.Key = `https://${bucketName}.s3.amazonaws.com/${customParams.Key}`; delete customParams.CopySource; } var params = Object.assign(defaultParams, customParams); // Params var Bucket = params.Bucket; var Key = params.Key; // console.log("AWS COPY==>", params); const copyPartParams = Object.assign({}, params); delete copyPartParams.ACL; return new Promise(function (resolve, reject) { return createMultipartCopy(params, S3) .then(function(uploadId) { // console.log('cp => ', response); var url = `https://${Bucket}.s3.amazonaws.com/${Key}`; var partitionsRangeArray = calculatePartitionsRangeArray(fileSize); var copyPartFunctionsArray = []; copyPartParams.CopySource = copySource copyPartParams.UploadId = uploadId partitionsRangeArray.forEach( function(partitionRange, index) { copyPartParams.PartNumber = index + 1, copyPartParams.CopySourceRange = 'bytes=' + partitionRange, copyPartFunctionsArray.push( copyPart(copyPartParams, S3) ); }); return Promise.all(copyPartFunctionsArray) .then(function (copyResults) { var copyResultsForCopyCompletion = prepareResultsForCopyCompletion(copyResults); var completeParams = { Bucket: params.Bucket, Key: params.Key, MultipartUpload: { Parts: copyResultsForCopyCompletion }, UploadId: uploadId } return completeMultipartCopy(completeParams, S3).then(function (completeResponse) { resolve({ url: url, response: completeResponse }) }) }) .catch(function(err){ var abortParams = { Bucket: params.Bucket, Key: params.Key, UploadId: uploadId } return abortMultipartCopy(abortParams, S3); }); }) //.catch((err) => console.log(err)); .catch(function (e) { reject(e); }); }); }; Bucket.prototype.uploadMultipleFiles = function(customParams) { var self = this; var flags = ['files']; var hasAllFlags = checkParams(customParams, flags); if (!hasAllFlags) { throw new Error('Unable to upload multiple files due parameters missing'); } // check files not empty if (typeof customParams.files !== 'object' || customParams.files.length < 1) { throw new Error('Files array should not be empty'); } var uploadsQueue = []; // check files integrity customParams.files.forEach(function(file) { if (typeof file.Key !== 'string') { throw new Error('File name Key should be string'); } if (typeof file.filePath === 'undefined') { throw new Error('File path should be provided'); } uploadsQueue.push(function() { return self.uploadFile({ filePath: file.filePath, Key: file.Key }); }); }); return Promise.resolve(uploadsQueue).mapSeries(f => f()); }; Bucket.prototype.listPagedFileVersions = function(customParams) { /* var flags = ['Key']; var hasAllFlags = checkParams(customParams, flags); if (!hasAllFlags) { throw new Error('Unable to list file versions due parameters missing'); } */ // console.log('Executing...', customParams); var S3 = this.S3; var bucketName = this.bucketName || ''; // https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#listObjectVersions-property var defaultParams = { Bucket: bucketName }; // Max limit of objects requested if (typeof customParams.limit !== 'undefined') { if (typeof customParams.limit !== 'number') { throw new Error('Number was expected for limit parameter'); return; } defaultParams.MaxKeys = customParams.limit; delete customParams.limit; } // Key is only to maintain consistance with File Listing // the original s3 api uses prefix // https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#listObjectVersions-property if (typeof customParams.Key !== 'undefined') { if (typeof customParams.Key !== 'string' || customParams.Key === '') { throw new Error('Key parameter was expected to be String'); return; } defaultParams.Prefix = customParams.Key; delete customParams.Key; } var params = Object.assign(defaultParams, customParams); // console.log('Executing clean...', params); var listObjectVersionsPromise = promisify(S3.listObjectVersions).bind(S3); return new Promise(function (resolve, reject) { return listObjectVersionsPromise(params) .then(function(fileVersions) { resolve(fileVersions); }) .catch(reject); }); }; Bucket.prototype.listFileVersions = function(customParams) { var self = this; var versions = []; var markers = []; var pageDelay = self.pagingDelay; // Max limit of objects requested if (typeof customParams.limit !== 'undefined') { if (typeof customParams.limit !== 'number') { throw new Error('Number was expected for limit parameter'); return; } customParams.MaxKeys = customParams.limit; delete customParams.limit; // console.log(customParams); } // Max pagedelay of objects requested if (typeof customParams.delay !== 'undefined') { if (typeof customParams.delay !== 'number') { throw new Error('Number was expected for delay parameter'); return; } pageDelay = customParams.delay; delete customParams.delay; } // Key is only to maintain consistance with File Listing // the original s3 api uses prefix // https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#listObjectVersions-property if (typeof customParams.Key !== 'undefined') { if (typeof customParams.Key !== 'string' || customParams.Key === '') { throw new Error('Key parameter was expected to be String'); return; } customParams.Prefix = customParams.Key; delete customParams.Key; } return self._fetchVersionsAndMarkers(customParams, versions, markers, pageDelay); }; Bucket.prototype._fetchVersionsAndMarkers = function (customParams, versions, markers, pageDelay) { var self = this; var delay = pageDelay || self.pagingDelay; // console.log(customParams); return self.listPagedFileVersions(customParams).then(function(res){ // console.log(res); versions = versions.concat(res.Versions); markers = markers.concat(res.DeleteMarkers); if (!res.IsTruncated) { return { Versions: versions, DeleteMarkers: markers }; } // console.log(delay); return Promise.delay(delay).then(function(){ // console.log('--->', res); // console.log('custom params--->', customParams); customParams.VersionIdMarker = res.NextVersionIdMarker; // A version-id marker cannot be specified without a key marker. customParams.KeyMarker = res.NextKeyMarker; // return self.listPagedFiles(customParams); return self._fetchVersionsAndMarkers(customParams, versions, markers, delay); }); }) }; Bucket.prototype.deleteAllVersions = function(customParams, deleteVersions, deleteMarkers) { var self = this; var deleteMarkers = deleteMarkers || false; var deleteVersions = deleteVersions || true; var flags = ['Key']; var hasAllFlags = checkParams(customParams, flags); if (!hasAllFlags) { throw new Error('Unable to delete all file versions due parameters missing'); } var S3 = self.S3; var bucketName = self.bucketName || ''; var fileKey = customParams.Key; // https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#listObjectVersions-property var params = { Key: fileKey }; return self.listFileVersions(params).then(function(fileVersions){ // Create and array of files versions to remove var files = []; if (deleteVersions) { files = fileVersions.Versions.map(function(version) { var file = { Key: null, VersionId: null } if (typeof version.Key !== 'string') { throw new Error('File name Key should be string'); } if (typeof version.VersionId === 'undefined') { throw new Error('File VersionId should be provided'); } file.Key = version.Key; file.VersionId = version.VersionId; return file; }); } if (deleteMarkers) { var markers = fileVersions.DeleteMarkers.map(function(version) { var file = { Key: null, VersionId: null } if (typeof version.Key !== 'string') { throw new Error('File name Key should be string'); } if (typeof version.VersionId === 'undefined') { throw new Error('File VersionId should be provided'); } file.Key = version.Key; file.VersionId = version.VersionId; return file; }); // Apend files with VersionId to the operation files = files.concat(markers); } if (files.length === 0) { // No files to delete return { Deleted: [] }; } // Delete All the Versions (and Markers) found for the same file Key return self.deleteFilesVersioned({ files: files }); }); }; Bucket.prototype.deleteAllMarkers = function(customParams) { return this.deleteAllVersions(customParams, false, true); }; Bucket.prototype.deleteAllVersionsAndMarkers = function(customParams) { return this.deleteAllVersions(customParams, true, true); }; /* Usage: Result: { IsTruncated: false, Contents: [ { Key: 'upload-test.txt', LastModified: 2018-04-15T22:48:27.000Z, ETag: '"abc..."', Size: 26, StorageClass: 'STANDARD' } ], Name: 'your-bucket-name', Prefix: '', MaxKeys: 1000, CommonPrefixes: [], KeyCount: 1 } */ Bucket.prototype.listPagedFiles = function (customParams) { var customBucketName = false; if (typeof customParams !== 'undefined' && typeof customParams.bucketName === 'string' && customParams.bucketName !== '') { customBucketName = customParams.bucketName; } var S3 = this.S3; var bucketName = customBucketName || this.bucketName; var defaultParams = { Bucket: bucketName }; // Max limit of objects requested if (typeof customParams.limit !== 'undefined') { if (typeof customParams.limit !== 'number') { throw new Error('Number was expected for limit parameter'); return; } defaultParams.MaxKeys = customParams.limit; delete customParams.limit; } /* // this was for listObjects v1 now we are using V2 if (typeof customParams.startMarker !== 'undefined') { if (typeof customParams.startMarker !== 'string' || customParams.startMarker === '') { throw new Error('Marker parameter was expected to be String'); return; } defaultParams.Marker = customParams.startMarker; } */ var params = Object.assign(defaultParams, customParams); var listObjectsPromise = promisify(S3.listObjectsV2).bind(S3); return new Promise(function (resolve, reject) { listObjectsPromise(params) .then(function(files) { resolve(files); }) .catch(reject); }); }; Bucket.prototype.listFiles = function (customParams) { var self = this; var files = []; var pageDelay = self.pagingDelay; // Max limit of objects requested if (typeof customParams.limit !== 'undefined') { if (typeof customParams.limit !== 'number') { throw new Error('Number was expected for limit parameter'); return; } customParams.MaxKeys = customParams.limit; delete customParams.limit; } // Max pagedelay of objects requested if (typeof customParams.delay !== 'undefined') { if (typeof customParams.delay !== 'number') { throw new Error('Number was expected for delay parameter'); return; } pageDelay = customParams.delay; delete customParams.delay; } return self._fetchFiles(customParams, files, pageDelay); }; Bucket.prototype._fetchFiles = function (customParams, files, pageDelay) { var self = this; var delay = pageDelay || self.pagingDelay; return self.listPagedFiles(customParams).then(function(res){ files = files.concat(res.Contents); if (!res.IsTruncated) { return files; } // console.log(delay); return Promise.delay(delay).then(function(){ // console.log(res); customParams.ContinuationToken = res.NextContinuationToken; // return self.listPagedFiles(customParams); return self._fetchFiles(customParams, files, delay); }); }) }; /* Usage: Result: { Deleted: [ { Key: 'upload-test.txt' } ], Errors: [] } */ Bucket.prototype.deleteFiles = function (customParams) { var flags = ['files']; var hasAllFlags = checkParams(customParams, flags); if (!hasAllFlags) { throw new Error('Unable to upload files due parameters missing'); } if (typeof customParams.files !== 'object' || typeof customParams.files.length < 1) { throw new Error('Files array should not be empty'); } var S3 = this.S3; var bucketName = this.bucketName; var files = customParams.files.map(function(file) { if (typeof file !== 'string') { throw new Error('File name Key should be string'); } return { Key: file }; }); var params = { Bucket: bucketName, Delete: { Objects: files } }; var deleteObjectsPromise = promisify(S3.deleteObjects).bind(S3); return new Promise(function (resolve, reject) { deleteObjectsPromise(params) .then(function (response) { resolve(response); }) .catch(reject); }); }; Bucket.prototype.deleteFilesVersioned = function (customParams) { var flags = ['files']; var hasAllFlags = checkParams(customParams, flags); if (!hasAllFlags) { throw new Error('Unable to upload files due parameters missing'); } if (typeof customParams.files !== 'object' || customParams.files.length < 1) { throw new Error('Files array should not be empty'); } var S3 = this.S3; var bucketName = this.bucketName; var files = customParams.files.map(function(file) { if (typeof file.Key !== 'string') { throw new Error('File name Key should be string'); } if (typeof file.VersionId === 'undefined') { throw new Error('File VersionId should be provided'); } return file; }); var params = { Bucket: bucketName, Delete: { Objects: files } // Quiet: false }; var deleteObjectsPromise = promisify(S3.deleteObjects).bind(S3); return new Promise(function (resolve, reject) { deleteObjectsPromise(params) .then(function (response) { resolve(response); }) .catch(reject); }); }; Bucket.prototype.updateCredentials = function(credentials) { if (typeof credentials === 'undefined') { throw new Error('Credentials parameter is mandatory'); } this.S3.config.update({ credentials: new AWS.Credentials(credentials) }); }; Bucket.prototype.updateRegion = function(region) { if (typeof region === 'undefined') { throw new Error('Region parameter is mandatory'); } this.S3.config.update({region: region}) }; Bucket.prototype.updateBucketName = function(name) { if (typeof name === 'undefined') { throw new Error('Name parameter is mandatory'); } this.bucketName = name; }; module.exports = Bucket;