@cumulus/aws-client/S3.d.ts

Utilities for working with AWS

/**
 * @module S3
 */
/// <reference types="node" />
/// <reference types="node" />
import pRetry from 'p-retry';
import { Readable, TransformOptions } from 'stream';
import { CopyObjectCommandInput, DeleteObjectRequest, DeleteBucketCommandOutput, GetObjectCommandInput, GetObjectOutput, HeadObjectOutput, ListObjectsV2Request, ObjectCannedACL, PutObjectCommandInput, PutObjectRequest, S3, Tagging, ListObjectsCommandOutput, DeleteObjectCommandOutput, PutObjectCommandOutput, CopyObjectCommandOutput, CompleteMultipartUploadCommandOutput, GetObjectTaggingCommandOutput } from '@aws-sdk/client-s3';
import { Options as UploadOptions } from '@aws-sdk/lib-storage';
import { s3 } from './services';
export declare type GetObjectMethod = (params: GetObjectCommandInput) => Promise<GetObjectOutput>;
/**
 * Join strings into an S3 key without a leading slash
 *
 * @param {...string|Array<string>} args - the strings to join
 * @returns {string} the full S3 key
 */
export declare const s3Join: (args_0: string | string[], ...args_1: string[]) => string;
/**
* parse an s3 uri to get the bucket and key
*
* @param {string} uri - must be a uri with the `s3://` protocol
* @returns {Object} Returns an object with `Bucket` and `Key` properties
**/
export declare const parseS3Uri: (uri: string) => {
    Bucket: string;
    Key: string;
};
/**
 * Given a bucket and key, return an S3 URI
 *
 * @param {string} bucket - an S3 bucket name
 * @param {string} key - an S3 key
 * @returns {string} an S3 URI
 */
export declare const buildS3Uri: (bucket: string, key: string) => string;
/**
* Convert S3 TagSet Object to query string
* e.g. [{ Key: 'tag', Value: 'value }] to 'tag=value'
*
* @param {Array<Object>} tagset - S3 TagSet array
* @returns {string} tags query string
*/
export declare const s3TagSetToQueryString: (tagset: Tagging['TagSet']) => string | undefined;
/**
 * Delete an object from S3
 *
 * @param {string} bucket - bucket where the object exists
 * @param {string} key - key of the object to be deleted
 * @returns {Promise} promise of the object being deleted
 */
export declare const deleteS3Object: (bucket: string, key: string) => Promise<DeleteObjectCommandOutput>;
export declare const deleteS3Objects: (params: {
    client: S3;
    bucket: string;
    keys: string[];
}) => Promise<import("@aws-sdk/client-s3").DeleteObjectsCommandOutput>;
/**
* Get an object header from S3
*
* @param Bucket - name of bucket
* @param Key - key for object (filepath + filename)
* @param retryOptions - options to control retry behavior when an
*   object does not exist. See https://github.com/tim-kos/node-retry#retryoperationoptions
*   By default, retries will not be performed
* @returns  returns response from `S3.headObject` as a promise
**/
export declare const headObject: (Bucket: string, Key: string, retryOptions?: pRetry.Options) => Promise<HeadObjectOutput>;
/**
 * Test if an object exists in S3
 *
 * @param params - same params as https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#headObject-property
 * @returns {Promise<boolean>} a Promise that will resolve to a boolean indicating
 *                             if the object exists
 */
export declare const s3ObjectExists: (params: {
    Bucket: string;
    Key: string;
}) => Promise<boolean>;
/**
 * Asynchronously waits for an S3 object to exist at a specified location.
 *
 * This function uses `p-wait-for` to repeatedly check for the object's existence
 * until it's found or a timeout is reached. It provides configurable `interval`
 * between checks and a total `timeout` duration.
 *
 * @param params - The parameters for waiting for the object.
 * @param params.bucket - The name of the S3 bucket where the object is expected.
 * @param params.key - The key (path) of the S3 object within the bucket.
 * @param params.interval - The time in milliseconds to wait between checks.
 * Defaults to 1000ms (1 second).
 * @param params.timeout - The maximum time in milliseconds to wait before
 * giving up and throwing a `TimeoutError`. Defaults to 30000ms (30 seconds).
 * @returns A Promise that resolves when the S3 object is found.
 * @throws {TimeoutError} If the object does not exist within the specified `timeout` period.
 * @throws {Error} If an unexpected error occurs during the S3 existence check.
 */
export declare const waitForObjectToExist: (params: {
    bucket: string;
    key: string;
    interval?: number;
    timeout?: number;
}) => Promise<void>;
/**
* Put an object on S3
*
* @param params - same params as https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property
**/
export declare const s3PutObject: (params: PutObjectCommandInput) => Promise<PutObjectCommandOutput>;
/**
 * Upload a file to S3
 *
 */
export declare const putFile: (bucket: string, key: string, filename: string) => Promise<PutObjectCommandOutput>;
/**
* Copy an object from one location on S3 to another
**/
export declare const s3CopyObject: (params: CopyObjectCommandInput) => Promise<CopyObjectCommandOutput>;
/**
 * Upload data to S3
 *
 * see https://github.com/aws/aws-sdk-js-v3/tree/main/lib/lib-storage
 */
export declare const promiseS3Upload: (params: Omit<UploadOptions, 'client'>) => Promise<{
    [key: string]: any;
    ETag?: string | undefined;
}>;
/**
 * Upload data to S3 using a stream
 */
export declare const streamS3Upload: (uploadStream: Readable, uploadParams: UploadOptions) => Promise<CompleteMultipartUploadCommandOutput>;
/**
 * Get a readable stream for an S3 object
 */
export declare const getObjectReadStream: (params: {
    s3: {
        getObject: GetObjectMethod;
    };
    bucket: string;
    key: string;
}) => Promise<Readable>;
/**
 * Downloads the given s3Obj to the given filename in a streaming manner
 */
export declare const downloadS3File: (s3Obj: GetObjectCommandInput, filepath: string) => Promise<string>;
/**
 * Get the size of an S3 object
 */
export declare const getObjectSize: (params: {
    s3: S3;
    bucket: string;
    key: string;
}) => Promise<number | undefined>;
/**
* Get object Tagging from S3
**/
export declare const s3GetObjectTagging: (bucket: string, key: string) => Promise<GetObjectTaggingCommandOutput>;
/**
* Puts object Tagging in S3
* https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObjectTagging-property
**/
export declare const s3PutObjectTagging: (Bucket: string, Key: string, ObjectTagging: Tagging) => Promise<PutObjectCommandOutput>;
/**
 * Gets an object from S3.
 */
export declare const getObject: (s3Client: S3, params: GetObjectCommandInput) => Promise<GetObjectOutput>;
/**
 * Get an object from S3, waiting for it to exist and, if specified, have the
 * correct ETag.
 */
export declare const waitForObject: (s3Client: S3, params: GetObjectCommandInput, retryOptions?: pRetry.Options) => Promise<GetObjectOutput>;
/**
 * Gets an object from S3.
 * @deprecated
 */
export declare const getS3Object: (Bucket: string, Key: string, retryOptions?: pRetry.Options) => Promise<GetObjectOutput>;
export declare const getObjectStreamBuffers: (objectReadStream: Readable) => Promise<Buffer[]>;
/**
 * Transform streaming response from S3 object to text content
 */
export declare const getObjectStreamContents: (objectReadStream: Readable) => Promise<string>;
/**
 * Fetch the contents of an S3 object
 */
export declare const getTextObject: (bucket: string, key: string) => Promise<string>;
/**
 * Fetch JSON stored in an S3 object
 */
export declare const getJsonS3Object: (bucket: string, key: string) => Promise<any>;
export declare const putJsonS3Object: (bucket: string, key: string, data: any) => Promise<PutObjectCommandOutput>;
/**
* Check if a file exists in an S3 object
**/
export declare const fileExists: (bucket: string, key: string) => Promise<HeadObjectOutput | false>;
/**
 * Delete files from S3
 *
 * @param s3Objs - An array of objects containing keys 'Bucket' and 'Key'
 */
export declare const deleteS3Files: (s3Objs: DeleteObjectRequest[]) => Promise<DeleteObjectCommandOutput[]>;
declare type FileInfo = {
    filename: string;
    key: string;
    bucket: string;
};
export declare const uploadS3Files: (files: Array<string | FileInfo>, defaultBucket: string, keyPath: string | ((x: string) => string), s3opts?: Partial<PutObjectRequest>) => Promise<{
    key: string;
    bucket: string;
}[]>;
/**
 * Upload the file associated with the given stream to an S3 bucket
 *
 * @param {ReadableStream} fileStream - The stream for the file's contents
 * @param {string} bucket - The S3 bucket to which the file is to be uploaded
 * @param {string} key - The key to the file in the bucket
 * @param {Object} s3opts - Options to pass to the AWS sdk call (defaults to `{}`)
 * @returns {Promise} A promise
 */
export declare const uploadS3FileStream: (fileStream: Readable, bucket: string, key: string, s3opts?: Partial<PutObjectRequest>) => Promise<{
    [key: string]: any;
    ETag?: string | undefined;
}>;
/**
 * List the objects in an S3 bucket
 */
export declare const listS3Objects: (bucket: string, prefix?: string, skipFolders?: boolean) => Promise<ListObjectsCommandOutput['Contents']>;
/**
 * Fetch complete list of S3 objects
 *
 * listObjectsV2 is limited to 1,000 results per call.  This function continues
 * listing objects until there are no more to be fetched.
 *
 * The passed params must be compatible with the listObjectsV2 call.
 *
 * https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#listObjectsV2-property
 *
 * @param {Object} params - params for the s3.listObjectsV2 call
 * @returns {Promise<Array>} resolves to an array of objects corresponding to
 *   the Contents property of the listObjectsV2 response
 *
 * @static
 */
export declare const listS3ObjectsV2: (params: ListObjectsV2Request) => Promise<ListObjectsCommandOutput['Contents']>;
/**
 * Fetch lazy list of S3 objects
 *
 * listObjectsV2 is limited to 1,000 results per call.  This function continues
 * listing objects until there are no more to be fetched.
 *
 * The passed params must be compatible with the listObjectsV2 call.
 *
 * https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#listObjectsV2-property
 *
 * @param params - params for the s3.listObjectsV2 call
 * @yields a series of objects corresponding to
 *   the Contents property of the listObjectsV2 response
 *   batched to allow processing of one chunk at a time
 *
 * @static
 */
export declare function listS3ObjectsV2Batch(params: ListObjectsV2Request): AsyncIterable<ListObjectsCommandOutput['Contents']>;
/**
* Delete a bucket and all of its objects from S3
*
* @param bucket - name of the bucket
* @returns the promised result of `S3.deleteBucket`
**/
export declare const recursivelyDeleteS3Bucket: (bucket: string) => Promise<DeleteBucketCommandOutput>;
/**
* Delete a list of buckets and all of their objects from S3
*
* @param {Array} buckets - list of bucket names
* @returns {Promise} the promised result of `S3.deleteBucket`
**/
export declare const deleteS3Buckets: (buckets: Array<string>) => Promise<any>;
/**
 * Calculate the cryptographic hash of an S3 object
 *
 * @param {Object} params
 * @param {S3} params.s3 - an S3 instance
 * @param {string} params.algorithm - `cksum`, or an algorithm listed in
 *   `openssl list -digest-algorithms`
 * @param {string} params.bucket
 * @param {string} params.key
 */
export declare const calculateObjectHash: (params: {
    s3: {
        getObject: GetObjectMethod;
    };
    algorithm: string;
    bucket: string;
    key: string;
}) => Promise<string | number>;
/**
 * Validate S3 object checksum against expected sum
 *
 * @param {Object} params - params
 * @param {string} params.algorithm - checksum algorithm
 * @param {string} params.bucket - S3 bucket
 * @param {string} params.key - S3 key
 * @param {number|string} params.expectedSum - expected checksum
 * @param {Object} [params.options] - crypto.createHash options
 *
 * @throws {InvalidChecksum} - Throws error if validation fails
 * @returns {Promise<boolean>} returns true for success
 */
export declare const validateS3ObjectChecksum: (params: {
    algorithm: string;
    bucket: string;
    key: string;
    expectedSum: string;
    options: TransformOptions;
}) => Promise<boolean>;
/**
 * Extract the S3 bucket and key from the URL path parameters
 *
 * @param {string} pathParams - path parameters from the URL
 * bucket/key in the form of
 * @returns {Array<string>} `[Bucket, Key]`
 */
export declare const getFileBucketAndKey: (pathParams: string) => [string, string];
/**
 * Create an S3 bucket
 *
 * @param {string} Bucket - the name of the S3 bucket to create
 * @returns {Promise}
 */
export declare const createBucket: (Bucket: string) => Promise<import("@aws-sdk/client-s3").CreateBucketCommandOutput>;
/**
 * Create multiple S3 buckets
 *
 * @param {Array<string>} buckets - the names of the S3 buckets to create
 * @returns {Promise}
 */
export declare const createS3Buckets: (buckets: Array<string>) => Promise<any>;
/**
 * Copy an S3 object to another location in S3 using a multipart copy
 *
 * @param {Object} params
 * @param {string} params.sourceBucket
 * @param {string} params.sourceKey
 * @param {string} params.destinationBucket
 * @param {string} params.destinationKey
 * @param {S3.HeadObjectOutput} [params.sourceObject]
 *   Output from https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#headObject-property
 * @param {string} [params.ACL] - an [S3 Canned ACL](https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl)
 * @param {boolean} [params.copyTags=false]
 * @param {number} [params.chunkSize] - chunk size of the S3 multipart uploads
 * @returns {Promise.<{ etag: string }>} object containing the ETag of the
 *    destination object
 *
 * note: this method may error if used with zero byte files. see CUMULUS-2557 and https://github.com/nasa/cumulus/pull/2117.
 */
export declare const multipartCopyObject: (params: {
    sourceBucket: string;
    sourceKey: string;
    destinationBucket: string;
    destinationKey: string;
    sourceObject?: HeadObjectOutput;
    ACL?: ObjectCannedACL | string;
    copyTags?: boolean;
    chunkSize?: number;
}) => Promise<{
    etag: string;
}>;
/**
 * Copy an S3 object to another location in S3
 *
 * @param params.ACL - an [S3 Canned ACL](https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl)
 * @param params.copyTags=false
 * @param params.chunkSize - chunk size of the S3 multipart uploads
 */
export declare const copyObject: ({ sourceBucket, sourceKey, destinationBucket, destinationKey, ACL, copyTags, chunkSize, }: {
    sourceBucket: string;
    sourceKey: string;
    destinationBucket: string;
    destinationKey: string;
    ACL?: string | undefined;
    copyTags?: boolean | undefined;
    chunkSize?: number | undefined;
}) => Promise<void>;
/**
 * Move an S3 object to another location in S3
 *
 * @param {string} [params.ACL] - an [S3 Canned ACL](https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl)
 * @param {boolean} [params.copyTags=false]
 * @param {number} [params.chunkSize] - chunk size of the S3 multipart uploads
 * @returns {Promise<undefined>}
 */
export declare const moveObject: (params: {
    sourceBucket: string;
    sourceKey: string;
    destinationBucket: string;
    destinationKey: string;
    ACL?: string;
    copyTags?: boolean;
    chunkSize?: number;
}) => Promise<DeleteObjectCommandOutput>;
export {};
//# sourceMappingURL=S3.d.ts.map