@cumulus/aws-client
Version:
Utilities for working with AWS
863 lines • 34 kB
JavaScript
;
/**
* @module S3
*/
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.moveObject = exports.copyObject = exports.multipartCopyObject = exports.createS3Buckets = exports.createBucket = exports.getFileBucketAndKey = exports.validateS3ObjectChecksum = exports.calculateObjectHash = exports.deleteS3Buckets = exports.recursivelyDeleteS3Bucket = exports.listS3ObjectsV2Batch = exports.listS3ObjectsV2 = exports.listS3Objects = exports.uploadS3FileStream = exports.uploadS3Files = exports.deleteS3Files = exports.fileExists = exports.putJsonS3Object = exports.getJsonS3Object = exports.getTextObject = exports.getObjectStreamContents = exports.getObjectStreamBuffers = exports.getS3Object = exports.waitForObject = exports.getObject = exports.s3PutObjectTagging = exports.s3GetObjectTagging = exports.getObjectSize = exports.downloadS3File = exports.getObjectReadStream = exports.streamS3Upload = exports.promiseS3Upload = exports.s3CopyObject = exports.putFile = exports.s3PutObject = exports.waitForObjectToExist = exports.s3ObjectExists = exports.headObject = exports.deleteS3Objects = exports.deleteS3Object = exports.s3TagSetToQueryString = exports.buildS3Uri = exports.parseS3Uri = exports.s3Join = void 0;
const fs_1 = __importDefault(require("fs"));
const isBoolean_1 = __importDefault(require("lodash/isBoolean"));
const path_1 = __importDefault(require("path"));
const p_map_1 = __importDefault(require("p-map"));
const p_retry_1 = __importDefault(require("p-retry"));
const p_wait_for_1 = __importDefault(require("p-wait-for"));
const p_timeout_1 = __importDefault(require("p-timeout"));
const pump_1 = __importDefault(require("pump"));
const querystring_1 = __importDefault(require("querystring"));
const stream_1 = require("stream");
const util_1 = require("util");
const lib_storage_1 = require("@aws-sdk/lib-storage");
const checksum_1 = require("@cumulus/checksum");
const errors_1 = require("@cumulus/errors");
const logger_1 = __importDefault(require("@cumulus/logger"));
const S3MultipartUploads = __importStar(require("./lib/S3MultipartUploads"));
const services_1 = require("./services");
const test_utils_1 = require("./test-utils");
const utils_1 = require("./utils");
const log = new logger_1.default({ sender: 'aws-client/s3' });
const buildDeprecationMessage = (name, version, alternative) => {
let message = `${name} is deprecated after version ${version} and will be removed in a future release.`;
if (alternative)
message += ` Use ${alternative} instead.`;
return log.buildMessage('warn', message);
};
const S3_RATE_LIMIT = (0, test_utils_1.inTestMode)() ? 1 : 20;
/**
* Join strings into an S3 key without a leading slash
*
* @param {...string|Array<string>} args - the strings to join
* @returns {string} the full S3 key
*/
const s3Join = (...args) => {
let tokens;
if (typeof args[0] === 'string')
tokens = args;
else
tokens = args[0];
const removeLeadingSlash = (token) => token.replace(/^\//, '');
const removeTrailingSlash = (token) => token.replace(/\/$/, '');
const isNotEmptyString = (token) => token.length > 0;
const key = tokens
.map(removeLeadingSlash)
.map(removeTrailingSlash)
.filter(isNotEmptyString)
.join('/');
if (tokens[tokens.length - 1].endsWith('/'))
return `${key}/`;
return key;
};
exports.s3Join = s3Join;
/**
* parse an s3 uri to get the bucket and key
*
* @param {string} uri - must be a uri with the `s3://` protocol
* @returns {Object} Returns an object with `Bucket` and `Key` properties
**/
const parseS3Uri = (uri) => {
const match = uri.match('^s3://([^/]+)/(.*)$');
if (match === null) {
throw new TypeError(`Unable to parse S3 URI: ${uri}`);
}
return {
Bucket: match[1],
Key: match[2],
};
};
exports.parseS3Uri = parseS3Uri;
/**
* Given a bucket and key, return an S3 URI
*
* @param {string} bucket - an S3 bucket name
* @param {string} key - an S3 key
* @returns {string} an S3 URI
*/
const buildS3Uri = (bucket, key) => `s3://${bucket}/${key.replace(/^\/+/, '')}`;
exports.buildS3Uri = buildS3Uri;
/**
* Convert S3 TagSet Object to query string
* e.g. [{ Key: 'tag', Value: 'value }] to 'tag=value'
*
* @param {Array<Object>} tagset - S3 TagSet array
* @returns {string} tags query string
*/
const s3TagSetToQueryString = (tagset) => tagset?.map(({ Key, Value }) => `${Key}=${Value}`).join('&');
exports.s3TagSetToQueryString = s3TagSetToQueryString;
/**
* Delete an object from S3
*
* @param {string} bucket - bucket where the object exists
* @param {string} key - key of the object to be deleted
* @returns {Promise} promise of the object being deleted
*/
const deleteS3Object = (bucket, key) => (0, services_1.s3)().deleteObject({ Bucket: bucket, Key: key });
exports.deleteS3Object = deleteS3Object;
const deleteS3Objects = (params) => {
const { bucket, client, keys } = params;
const objects = {
Bucket: bucket,
Delete: {
Objects: keys.map((key) => ({ Key: key })),
},
};
return client.deleteObjects(objects);
};
exports.deleteS3Objects = deleteS3Objects;
/**
* Get an object header from S3
*
* @param Bucket - name of bucket
* @param Key - key for object (filepath + filename)
* @param retryOptions - options to control retry behavior when an
* object does not exist. See https://github.com/tim-kos/node-retry#retryoperationoptions
* By default, retries will not be performed
* @returns returns response from `S3.headObject` as a promise
**/
const headObject = (Bucket, Key, retryOptions = { retries: 0 }) => (0, p_retry_1.default)(async () => {
try {
return await (0, services_1.s3)().headObject({ Bucket, Key });
}
catch (error) {
if (error.name === 'NotFound')
throw error;
throw new p_retry_1.default.AbortError(error);
}
}, { maxTimeout: 10000, ...retryOptions });
exports.headObject = headObject;
/**
* Test if an object exists in S3
*
* @param params - same params as https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#headObject-property
* @returns {Promise<boolean>} a Promise that will resolve to a boolean indicating
* if the object exists
*/
const s3ObjectExists = (params) => (0, exports.headObject)(params.Bucket, params.Key)
.then(() => true)
.catch((error) => {
if (error.name === 'NotFound')
return false;
throw error;
});
exports.s3ObjectExists = s3ObjectExists;
/**
* Asynchronously waits for an S3 object to exist at a specified location.
*
* This function uses `p-wait-for` to repeatedly check for the object's existence
* until it's found or a timeout is reached. It provides configurable `interval`
* between checks and a total `timeout` duration.
*
* @param params - The parameters for waiting for the object.
* @param params.bucket - The name of the S3 bucket where the object is expected.
* @param params.key - The key (path) of the S3 object within the bucket.
* @param params.interval - The time in milliseconds to wait between checks.
* Defaults to 1000ms (1 second).
* @param params.timeout - The maximum time in milliseconds to wait before
* giving up and throwing a `TimeoutError`. Defaults to 30000ms (30 seconds).
* @returns A Promise that resolves when the S3 object is found.
* @throws {TimeoutError} If the object does not exist within the specified `timeout` period.
* @throws {Error} If an unexpected error occurs during the S3 existence check.
*/
const waitForObjectToExist = async (params) => {
const { bucket, key, interval = 1000, timeout = 30 * 1000, } = params;
try {
await (0, p_wait_for_1.default)(() => (0, exports.s3ObjectExists)({ Bucket: bucket, Key: key }), { interval, timeout });
}
catch (error) {
if (error instanceof p_timeout_1.default) {
log.error(`Timed out after ${timeout}ms waiting for existence of s3://${bucket}/${key}`);
}
else {
log.error(`Unexpected error while waiting for existence of s3://${bucket}/${key}: ${error}`);
}
throw error;
}
};
exports.waitForObjectToExist = waitForObjectToExist;
/**
* Put an object on S3
*
* @param params - same params as https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property
**/
const s3PutObject = (params) => (0, services_1.s3)().putObject(params);
exports.s3PutObject = s3PutObject;
/**
* Upload a file to S3
*
*/
const putFile = (bucket, key, filename) => (0, exports.s3PutObject)({
Bucket: bucket,
Key: key,
Body: fs_1.default.createReadStream(filename),
});
exports.putFile = putFile;
/**
* Copy an object from one location on S3 to another
**/
const s3CopyObject = (params) => (0, services_1.s3)().copyObject({
TaggingDirective: 'COPY',
...params,
});
exports.s3CopyObject = s3CopyObject;
/**
* Upload data to S3
*
* see https://github.com/aws/aws-sdk-js-v3/tree/main/lib/lib-storage
*/
const promiseS3Upload = async (params) => {
const parallelUploads = new lib_storage_1.Upload({
...params,
client: (0, services_1.s3)(),
});
parallelUploads.on('httpUploadProgress', (progress) => {
log.info(progress);
});
const result = await parallelUploads.done();
return result;
};
exports.promiseS3Upload = promiseS3Upload;
/**
* Upload data to S3 using a stream
*/
const streamS3Upload = (uploadStream, uploadParams) => {
const parallelUploads3 = new lib_storage_1.Upload({
...uploadParams,
params: {
...uploadParams.params,
Body: uploadStream,
},
client: (0, services_1.s3)(),
});
parallelUploads3.on('httpUploadProgress', (progress) => {
log.info(progress);
});
return parallelUploads3.done();
};
exports.streamS3Upload = streamS3Upload;
/**
* Get a readable stream for an S3 object
*/
const getObjectReadStream = async (params) => {
// eslint-disable-next-line no-shadow
const { s3: s3Client, bucket, key } = params;
const response = await s3Client.getObject({ Bucket: bucket, Key: key });
if (!response.Body) {
throw new Error(`Could not get object for bucket ${bucket} and key ${key}`);
}
if (!(response.Body instanceof stream_1.Readable)) {
throw new TypeError('Unknown object stream type.');
}
return response.Body;
};
exports.getObjectReadStream = getObjectReadStream;
/**
* Downloads the given s3Obj to the given filename in a streaming manner
*/
const downloadS3File = async (s3Obj, filepath) => {
if (!s3Obj.Bucket || !s3Obj.Key) {
throw new Error('Bucket and Key are required');
}
const fileWriteStream = fs_1.default.createWriteStream(filepath);
const objectStream = await (0, exports.getObjectReadStream)({
bucket: s3Obj.Bucket,
key: s3Obj.Key,
s3: (0, services_1.s3)(),
});
return new Promise((resolve, reject) => (0, pump_1.default)(objectStream, fileWriteStream, (err) => {
if (err)
reject(err);
else
resolve(filepath);
}));
};
exports.downloadS3File = downloadS3File;
/**
* Get the size of an S3 object
*/
const getObjectSize = async (params) => {
// eslint-disable-next-line no-shadow
const { s3: s3Client, bucket, key } = params;
const headObjectResponse = await s3Client.headObject({
Bucket: bucket,
Key: key,
});
return headObjectResponse.ContentLength;
};
exports.getObjectSize = getObjectSize;
/**
* Get object Tagging from S3
**/
const s3GetObjectTagging = (bucket, key) => (0, services_1.s3)().getObjectTagging({ Bucket: bucket, Key: key });
exports.s3GetObjectTagging = s3GetObjectTagging;
const getObjectTags = async (bucket, key) => {
const taggingResponse = await (0, exports.s3GetObjectTagging)(bucket, key);
return taggingResponse?.TagSet?.reduce((accumulator, { Key, Value }) => {
if (Key && Value) {
return { ...accumulator, [Key]: Value };
}
return accumulator;
}, {});
};
const getObjectTaggingString = async (bucket, key) => {
const tags = await getObjectTags(bucket, key);
return querystring_1.default.stringify(tags);
};
/**
* Puts object Tagging in S3
* https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObjectTagging-property
**/
const s3PutObjectTagging = (Bucket, Key, ObjectTagging) => (0, services_1.s3)().putObjectTagging({
Bucket,
Key,
Tagging: ObjectTagging,
});
exports.s3PutObjectTagging = s3PutObjectTagging;
/**
* Gets an object from S3.
*/
const getObject = (s3Client, params) => s3Client.getObject(params);
exports.getObject = getObject;
/**
* Get an object from S3, waiting for it to exist and, if specified, have the
* correct ETag.
*/
const waitForObject = (s3Client, params, retryOptions = {}) => (0, p_retry_1.default)(async () => {
try {
return await (0, exports.getObject)(s3Client, params);
}
catch (error) {
// Retry if the object does not exist
if (error.name === 'NoSuchKey')
throw error;
// Retry if the etag did not match
if (params.IfMatch && error.name === 'PreconditionFailed')
throw error;
// For any other error, fail without retrying
throw new p_retry_1.default.AbortError(error);
}
}, retryOptions);
exports.waitForObject = waitForObject;
/**
* Gets an object from S3.
* @deprecated
*/
exports.getS3Object = (0, util_1.deprecate)((Bucket, Key, retryOptions = { retries: 0 }) => (0, exports.waitForObject)((0, services_1.s3)(), { Bucket, Key }, {
maxTimeout: 10000,
onFailedAttempt: (err) => log.debug(`getS3Object('${Bucket}', '${Key}') failed with ${err.retriesLeft} retries left: ${err.message}`),
...retryOptions,
}), buildDeprecationMessage('@cumulus/aws-client/S3.getS3Object', '2.0.1', '@cumulus/aws-client/S3.getObject or @cumulus/aws-client/S3.waitForObject'));
const getObjectStreamBuffers = (objectReadStream) => new Promise((resolve, reject) => {
try {
const responseDataChunks = [];
objectReadStream.once('error', (error) => reject(error));
objectReadStream.on('data', (chunk) => responseDataChunks.push(chunk));
// Once the stream has no more data, join the chunks into a string and
// return the string
objectReadStream.once('end', () => resolve(responseDataChunks));
}
catch (error) {
reject(error);
}
});
exports.getObjectStreamBuffers = getObjectStreamBuffers;
/**
* Transform streaming response from S3 object to text content
*/
const getObjectStreamContents = async (objectReadStream) => {
const buffers = await (0, exports.getObjectStreamBuffers)(objectReadStream);
return buffers.join('');
};
exports.getObjectStreamContents = getObjectStreamContents;
/**
* Fetch the contents of an S3 object
*/
const getTextObject = (bucket, key) => (0, exports.getObjectReadStream)({ s3: (0, services_1.s3)(), bucket, key })
.then((objectReadStream) => (0, exports.getObjectStreamContents)(objectReadStream));
exports.getTextObject = getTextObject;
/**
* Fetch JSON stored in an S3 object
*/
const getJsonS3Object = (bucket, key) => (0, exports.getTextObject)(bucket, key)
.then((text) => {
if (text === undefined)
return undefined;
return JSON.parse(text);
});
exports.getJsonS3Object = getJsonS3Object;
const putJsonS3Object = (bucket, key, data) => (0, exports.s3PutObject)({
Bucket: bucket,
Key: key,
Body: JSON.stringify(data),
});
exports.putJsonS3Object = putJsonS3Object;
/**
* Check if a file exists in an S3 object
**/
const fileExists = async (bucket, key) => {
try {
return await (0, services_1.s3)().headObject({ Key: key, Bucket: bucket });
}
catch (error) {
// if file is not return false
if (error.stack.match(/(NotFound)/) || error.stack.match(/(NoSuchBucket)/)) {
return false;
}
throw error;
}
};
exports.fileExists = fileExists;
/**
* Delete files from S3
*
* @param s3Objs - An array of objects containing keys 'Bucket' and 'Key'
*/
const deleteS3Files = async (s3Objs) => await (0, p_map_1.default)(s3Objs, (s3Obj) => (0, services_1.s3)().deleteObject(s3Obj), { concurrency: S3_RATE_LIMIT });
exports.deleteS3Files = deleteS3Files;
const uploadS3Files = async (files, defaultBucket, keyPath, s3opts = {}) => {
let i = 0;
const n = files.length;
if (n > 1) {
log.info(`Starting upload of ${n} keys`);
}
const promiseUpload = async (file) => {
let bucket;
let filename;
let key;
if (typeof file === 'string') {
bucket = defaultBucket;
filename = file;
if (typeof keyPath === 'string') {
key = (0, exports.s3Join)(keyPath, path_1.default.basename(file));
}
else {
key = keyPath(file);
}
}
else {
bucket = file.bucket || defaultBucket;
filename = file.filename;
key = file.key;
}
await (0, exports.promiseS3Upload)({
...s3opts,
params: {
Bucket: bucket,
Key: key,
Body: fs_1.default.createReadStream(filename),
},
});
i += 1;
log.info(`Progress: [${i} of ${n}] ${filename} -> s3://${bucket}/${key}`);
return { key, bucket };
};
return await (0, p_map_1.default)(files, promiseUpload, { concurrency: S3_RATE_LIMIT });
};
exports.uploadS3Files = uploadS3Files;
/**
* Upload the file associated with the given stream to an S3 bucket
*
* @param {ReadableStream} fileStream - The stream for the file's contents
* @param {string} bucket - The S3 bucket to which the file is to be uploaded
* @param {string} key - The key to the file in the bucket
* @param {Object} s3opts - Options to pass to the AWS sdk call (defaults to `{}`)
* @returns {Promise} A promise
*/
const uploadS3FileStream = (fileStream, bucket, key, s3opts = {}) => (0, exports.promiseS3Upload)({
params: {
...s3opts,
Bucket: bucket,
Key: key,
Body: fileStream,
},
});
exports.uploadS3FileStream = uploadS3FileStream;
/**
* List the objects in an S3 bucket
*/
const listS3Objects = async (bucket, prefix, skipFolders = true) => {
log.info(`Listing objects in s3://${bucket}`);
const params = {
Bucket: bucket,
};
if (prefix)
params.Prefix = prefix;
const data = await (0, services_1.s3)().listObjects(params);
if (!data.Contents) {
return [];
}
let contents = data.Contents.filter((obj) => obj.Key !== undefined);
if (skipFolders) {
// Filter out any references to folders
contents = contents.filter((obj) => obj.Key && !obj.Key.endsWith('/'));
}
return contents;
};
exports.listS3Objects = listS3Objects;
/**
* Fetch complete list of S3 objects
*
* listObjectsV2 is limited to 1,000 results per call. This function continues
* listing objects until there are no more to be fetched.
*
* The passed params must be compatible with the listObjectsV2 call.
*
* https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#listObjectsV2-property
*
* @param {Object} params - params for the s3.listObjectsV2 call
* @returns {Promise<Array>} resolves to an array of objects corresponding to
* the Contents property of the listObjectsV2 response
*
* @static
*/
const listS3ObjectsV2 = async (params) => {
// Fetch the first list of objects from S3
let listObjectsResponse = await (0, services_1.s3)().listObjectsV2(params);
let discoveredObjects = listObjectsResponse.Contents ?? [];
// Keep listing more objects from S3 until we have all of them
while (listObjectsResponse.IsTruncated) {
// eslint-disable-next-line no-await-in-loop
listObjectsResponse = (await (0, services_1.s3)().listObjectsV2(
// Update the params with a Continuation Token
{
...params,
ContinuationToken: listObjectsResponse.NextContinuationToken,
}));
discoveredObjects = discoveredObjects.concat(listObjectsResponse.Contents ?? []);
}
return discoveredObjects.filter((obj) => obj.Key);
};
exports.listS3ObjectsV2 = listS3ObjectsV2;
/**
* Fetch lazy list of S3 objects
*
* listObjectsV2 is limited to 1,000 results per call. This function continues
* listing objects until there are no more to be fetched.
*
* The passed params must be compatible with the listObjectsV2 call.
*
* https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#listObjectsV2-property
*
* @param params - params for the s3.listObjectsV2 call
* @yields a series of objects corresponding to
* the Contents property of the listObjectsV2 response
* batched to allow processing of one chunk at a time
*
* @static
*/
async function* listS3ObjectsV2Batch(params) {
let listObjectsResponse = await (0, services_1.s3)().listObjectsV2(params);
let discoveredObjects = listObjectsResponse.Contents ?? [];
yield discoveredObjects.filter((obj) => 'Key' in obj);
// Keep listing more objects from S3 until we have all of them
while (listObjectsResponse.IsTruncated) {
// eslint-disable-next-line no-await-in-loop
listObjectsResponse = (await (0, services_1.s3)().listObjectsV2(
// Update the params with a Continuation Token
{
...params,
ContinuationToken: listObjectsResponse.NextContinuationToken,
}));
discoveredObjects = listObjectsResponse.Contents ?? [];
yield discoveredObjects.filter((obj) => 'Key' in obj);
}
}
exports.listS3ObjectsV2Batch = listS3ObjectsV2Batch;
/**
* Delete a bucket and all of its objects from S3
*
* @param bucket - name of the bucket
* @returns the promised result of `S3.deleteBucket`
**/
exports.recursivelyDeleteS3Bucket = (0, utils_1.improveStackTrace)(async (bucket) => {
for await (const objectBatch of listS3ObjectsV2Batch({ Bucket: bucket })) {
if (objectBatch) {
const deleteRequests = objectBatch.filter((obj) => obj.Key).map((obj) => ({ Bucket: bucket, Key: obj.Key }));
await (0, exports.deleteS3Files)(deleteRequests);
}
}
return await (0, services_1.s3)().deleteBucket({ Bucket: bucket });
});
/**
* Delete a list of buckets and all of their objects from S3
*
* @param {Array} buckets - list of bucket names
* @returns {Promise} the promised result of `S3.deleteBucket`
**/
const deleteS3Buckets = async (buckets) => await Promise.all(buckets.map(exports.recursivelyDeleteS3Bucket));
exports.deleteS3Buckets = deleteS3Buckets;
/**
* Calculate the cryptographic hash of an S3 object
*
* @param {Object} params
* @param {S3} params.s3 - an S3 instance
* @param {string} params.algorithm - `cksum`, or an algorithm listed in
* `openssl list -digest-algorithms`
* @param {string} params.bucket
* @param {string} params.key
*/
const calculateObjectHash = async (params) => {
// eslint-disable-next-line no-shadow
const { algorithm, bucket, key, s3: s3Client } = params;
const stream = await (0, exports.getObjectReadStream)({
s3: s3Client,
bucket,
key,
});
return await (0, checksum_1.generateChecksumFromStream)(algorithm, stream);
};
exports.calculateObjectHash = calculateObjectHash;
/**
* Validate S3 object checksum against expected sum
*
* @param {Object} params - params
* @param {string} params.algorithm - checksum algorithm
* @param {string} params.bucket - S3 bucket
* @param {string} params.key - S3 key
* @param {number|string} params.expectedSum - expected checksum
* @param {Object} [params.options] - crypto.createHash options
*
* @throws {InvalidChecksum} - Throws error if validation fails
* @returns {Promise<boolean>} returns true for success
*/
const validateS3ObjectChecksum = async (params) => {
const { algorithm, bucket, key, expectedSum, options } = params;
const fileStream = await (0, exports.getObjectReadStream)({ s3: (0, services_1.s3)(), bucket, key });
if (await (0, checksum_1.validateChecksumFromStream)(algorithm, fileStream, expectedSum, options)) {
return true;
}
const msg = `Invalid checksum for S3 object s3://${bucket}/${key} with type ${algorithm} and expected sum ${expectedSum}`;
throw new errors_1.InvalidChecksum(msg);
};
exports.validateS3ObjectChecksum = validateS3ObjectChecksum;
/**
* Extract the S3 bucket and key from the URL path parameters
*
* @param {string} pathParams - path parameters from the URL
* bucket/key in the form of
* @returns {Array<string>} `[Bucket, Key]`
*/
const getFileBucketAndKey = (pathParams) => {
const [Bucket, ...fields] = pathParams.split('/');
const Key = fields.join('/');
if (Bucket.length === 0 || Key.length === 0) {
throw new errors_1.UnparsableFileLocationError(`File location "${pathParams}" could not be parsed`);
}
return [Bucket, Key];
};
exports.getFileBucketAndKey = getFileBucketAndKey;
/**
* Create an S3 bucket
*
* @param {string} Bucket - the name of the S3 bucket to create
* @returns {Promise}
*/
const createBucket = (Bucket) => (0, services_1.s3)().createBucket({ Bucket });
exports.createBucket = createBucket;
/**
* Create multiple S3 buckets
*
* @param {Array<string>} buckets - the names of the S3 buckets to create
* @returns {Promise}
*/
const createS3Buckets = async (buckets) => await Promise.all(buckets.map(exports.createBucket));
exports.createS3Buckets = createS3Buckets;
const createMultipartUpload = async (params) => {
const uploadParams = {
Bucket: params.destinationBucket,
Key: params.destinationKey,
ACL: params.ACL,
ContentType: params.contentType,
};
if (params.copyTags) {
uploadParams.Tagging = await getObjectTaggingString(params.sourceBucket, params.sourceKey);
}
// Create a multi-part upload (copy) and get its UploadId
const { UploadId } = await S3MultipartUploads.createMultipartUpload(uploadParams);
if (UploadId === undefined) {
throw new Error('Unable to create multipart upload');
}
return UploadId;
};
// This performs an S3 `uploadPartCopy` call. That response includes an `ETag`
// value specific to the part that was uploaded. When `completeMultipartUpload`
// is called later, it needs that `ETag` value, as well as the `PartNumber` for
// each part. Since the `PartNumber` is not included in the `uploadPartCopy`
// response, we are adding it here to make our lives easier when we eventually
// call `completeMultipartUpload`.
const uploadPartCopy = async (params) => {
const response = await S3MultipartUploads.uploadPartCopy({
UploadId: params.uploadId,
Bucket: params.destinationBucket,
Key: params.destinationKey,
PartNumber: params.partNumber,
CopySource: `/${params.sourceBucket}/${params.sourceKey}`,
CopySourceRange: `bytes=${params.start}-${params.end}`,
});
if (response.CopyPartResult === undefined) {
throw new Error('Did not get ETag from uploadPartCopy');
}
return {
ETag: response.CopyPartResult.ETag,
PartNumber: params.partNumber,
};
};
/**
* Copy an S3 object to another location in S3 using a multipart copy
*
* @param {Object} params
* @param {string} params.sourceBucket
* @param {string} params.sourceKey
* @param {string} params.destinationBucket
* @param {string} params.destinationKey
* @param {S3.HeadObjectOutput} [params.sourceObject]
* Output from https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#headObject-property
* @param {string} [params.ACL] - an [S3 Canned ACL](https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl)
* @param {boolean} [params.copyTags=false]
* @param {number} [params.chunkSize] - chunk size of the S3 multipart uploads
* @returns {Promise.<{ etag: string }>} object containing the ETag of the
* destination object
*
* note: this method may error if used with zero byte files. see CUMULUS-2557 and https://github.com/nasa/cumulus/pull/2117.
*/
const multipartCopyObject = async (params) => {
const { sourceBucket, sourceKey, destinationBucket, destinationKey, ACL, copyTags = false, chunkSize, } = params;
const sourceObject = params.sourceObject ?? await (0, exports.headObject)(sourceBucket, sourceKey);
// Create a multi-part upload (copy) and get its UploadId
const uploadId = await createMultipartUpload({
sourceBucket,
sourceKey,
destinationBucket,
destinationKey,
ACL: ACL,
copyTags,
contentType: sourceObject.ContentType,
});
try {
// Build the separate parts of the multi-part upload (copy)
const objectSize = sourceObject.ContentLength;
if (objectSize === undefined) {
throw new Error(`Unable to determine size of s3://${sourceBucket}/${sourceKey}`);
}
const chunks = S3MultipartUploads.createMultipartChunks(objectSize, chunkSize);
// Submit all of the upload (copy) parts to S3
const uploadPartCopyResponses = await Promise.all(chunks.map(({ start, end }, index) => uploadPartCopy({
uploadId,
partNumber: index + 1,
start,
end,
sourceBucket,
sourceKey,
destinationBucket,
destinationKey,
})));
// Let S3 know that the multi-part upload (copy) is completed
const { ETag: etag } = await S3MultipartUploads.completeMultipartUpload({
UploadId: uploadId,
Bucket: destinationBucket,
Key: destinationKey,
MultipartUpload: {
Parts: uploadPartCopyResponses,
},
});
return { etag };
}
catch (error) {
// If anything went wrong, make sure that the multi-part upload (copy)
// is aborted.
await S3MultipartUploads.abortMultipartUpload({
Bucket: destinationBucket,
Key: destinationKey,
UploadId: uploadId,
});
throw error;
}
};
exports.multipartCopyObject = multipartCopyObject;
/**
* Copy an S3 object to another location in S3
*
* @param params.ACL - an [S3 Canned ACL](https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl)
* @param params.copyTags=false
* @param params.chunkSize - chunk size of the S3 multipart uploads
*/
const copyObject = async ({ sourceBucket, sourceKey, destinationBucket, destinationKey, ACL, copyTags, chunkSize, }) => {
const sourceObject = await (0, exports.headObject)(sourceBucket, sourceKey);
if (sourceObject.ContentLength === 0) {
// 0 byte files cannot be copied with multipart upload,
// so use a regular S3 PUT
const s3uri = (0, exports.buildS3Uri)(destinationBucket, destinationKey);
const { CopyObjectResult } = await (0, exports.s3CopyObject)({
CopySource: path_1.default.join(sourceBucket, sourceKey),
Bucket: destinationBucket,
Key: destinationKey,
});
// This error should never actually be reached in practice. It's a
// necessary workaround for bad typings in the AWS SDK.
// https://github.com/aws/aws-sdk-js/issues/1719
if (!CopyObjectResult?.ETag) {
throw new Error(`ETag could not be determined for copy of ${(0, exports.buildS3Uri)(sourceBucket, sourceKey)} to ${s3uri}`);
}
}
else {
await (0, exports.multipartCopyObject)({
sourceBucket: sourceBucket,
sourceKey: sourceKey,
destinationBucket: destinationBucket,
destinationKey: destinationKey,
sourceObject: sourceObject,
ACL: ACL,
copyTags: (0, isBoolean_1.default)(copyTags) ? copyTags : true,
chunkSize: chunkSize,
});
}
};
exports.copyObject = copyObject;
/**
* Move an S3 object to another location in S3
*
* @param {string} [params.ACL] - an [S3 Canned ACL](https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl)
* @param {boolean} [params.copyTags=false]
* @param {number} [params.chunkSize] - chunk size of the S3 multipart uploads
* @returns {Promise<undefined>}
*/
const moveObject = async (params) => {
const { sourceBucket, sourceKey, } = params;
await (0, exports.copyObject)(params);
const deleteS3ObjRes = await (0, exports.deleteS3Object)(sourceBucket, sourceKey);
return deleteS3ObjRes;
};
exports.moveObject = moveObject;
//# sourceMappingURL=S3.js.map