@cumulus/ingest/granule.d.ts

Ingest utilities

import { FilePgModel, Knex } from '@cumulus/db';
import { ApiFile, DuplicateHandling } from '@cumulus/types';
export interface EventWithDuplicateHandling {
    config: {
        collection: {
            duplicateHandling?: DuplicateHandling;
        };
        duplicateHandling?: DuplicateHandling;
    };
    cumulus_config?: {
        cumulus_context?: {
            forceDuplicateOverwrite?: boolean;
        };
    };
}
export interface File {
    bucket?: string;
    key?: string;
    fileName?: string;
    name?: string;
    filename?: string;
    size?: string;
}
export interface MovedGranuleFile {
    bucket: string;
    key: string;
    name?: string;
}
export interface MoveFileParams {
    source?: {
        Bucket: string;
        Key: string;
    };
    target?: {
        Bucket: string;
        Key: string;
    };
    file: File;
}
export interface VersionedObject {
    Bucket: string;
    Key?: string;
    size?: number;
}
/**
  * rename s3 file with timestamp
  *
  * @param {string} bucket - bucket of the file
  * @param {string} key - s3 key of the file
  * @returns {Promise} promise that resolves when file is renamed
  */
export declare function renameS3FileWithTimestamp(bucket: string, key: string): Promise<void>;
/**
  * get all renamed s3 files for a given bucket and key
  *
  * @param {string} bucket - bucket of the file
  * @param {string} key - s3 key of the file
  * @returns {Array<Object>} returns renamed files
  */
export declare function listVersionedObjects(bucket: string, key: string): Promise<VersionedObject[]>;
/**
* Move granule file from one s3 bucket & keypath to another,
* creating a versioned copy of any file already existing at the target location
* and returning an array of the moved file and all versioned filenames.
*
* @param {Object} source - source
* @param {string} source.Bucket - source
* @param {string} source.Key - source
* @param {Object} target - target
* @param {string} target.Bucket - target
* @param {string} target.Key - target
* @param {Object} sourceChecksumObject - source checksum information
* @param {string} sourceChecksumObject.checksumType - checksum type, e.g. 'md5'
* @param {Object} sourceChecksumObject.checksum - checksum value
* @param {string} ACL - an S3 [Canned ACL](https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl)
* @returns {Promise<Array>} returns a promise that resolves to a list of s3 version file objects.
*
* @private
**/
export declare function moveGranuleFileWithVersioning(source: {
    Bucket: string;
    Key: string;
}, target: {
    Bucket: string;
    Key: string;
}, sourceChecksumObject?: {
    checksumType?: string;
    checksum?: string;
}, ACL?: string): Promise<VersionedObject[]>;
/**
 * handle duplicate file in S3 syncs and moves
 *
 * @param {Object} params - params object
 * @param {Object} params.source - source object: { Bucket, Key }
 * @param {Object} params.target - target object: { Bucket, Key }
 * @param {string} params.ACL - an S3 [Canned ACL](https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl)
 * @param {string} params.duplicateHandling - duplicateHandling config string
 * One of [`error`, `skip`, `replace`, `version`].
 * @param {Function} [params.checksumFunction] - optional function to verify source & target:
 * Called as `await checksumFunction(bucket, key);`, expected to return array where:
 * array[0] - string - checksum type
 * array[1] - string - checksum value
 * For example of partial application of expected values see `ingestFile` in this module.
 * @param {Function} [params.syncFileFunction] - optional function to sync file from non-s3 source.
 * Syncs to temporary source location for `version` case and to target location for `replace` case.
 * Called as `await syncFileFunction(bucket, key);`, expected to create file on S3.
 * For example of function prepared with partial application see `ingestFile` in this module.
 * @param {Function} [params.moveGranuleFileWithVersioningFunction] - optional -
 * override for moveGranuleFileWithVersioning.  Defaults to local module method
 * @param {Object} [params.s3Object] - optional - replacement for S3 import object,
 * intended for use in testing
 * @throws {DuplicateFile} DuplicateFile error in `error` case.
 * @returns {Array<Object>} List of file version S3 Objects in `version` case, otherwise empty.
 */
export declare function handleDuplicateFile(params: {
    source: {
        Bucket: string;
        Key: string;
    };
    target: {
        Bucket: string;
        Key: string;
    };
    duplicateHandling: DuplicateHandling;
    checksumFunction?: (bucket: string, key: string) => Promise<[string, string]>;
    syncFileFunction?: (params: {
        destinationBucket: string;
        destinationKey: string;
        bucket?: string;
        fileRemotePath: string;
    }) => Promise<void>;
    ACL?: string;
    sourceBucket?: string;
    fileRemotePath?: string;
    s3Object?: {
        moveObject: Function;
    };
    moveGranuleFileWithVersioningFunction?: Function;
}): Promise<VersionedObject[]>;
/**
 * Get the name of the file from the following properties (in order of preference):
 * 1. fileName (e.g. 'granuleFileNamec.md')
 * 2. name (e.g. 'granuleFileName.md')
 * 3. key (e.g. 'stackname/filepath/granuleFileName.md')
 * @param {File} file - file object with the above properties
 * @returns {string | undefined} - The file name as a string or undefined
 */
export declare const getNameOfFile: (file: File) => string | undefined;
/**
 * For each source file, see if there is a destination and generate the source
 * and target for the file moves.
 * @param {Array<Object>} sourceFiles - granule file objects
 * @param {Array<Object>} destinations - array of objects defining the destination of granule files
 * @returns {Array<Object>} - array containing the parameters for moving the file:
 *  {
 *    source: { Bucket, Key },
 *    target: { Bucket, Key },
 *    file: file object
 *  }
 */
export declare function generateMoveFileParams(sourceFiles: File[], destinations: {
    bucket: string;
    filepath?: string;
    regex: string | RegExp;
}[]): MoveFileParams[];
/**
* Moves a granule file and updates the datastore accordingly
* @summary Moves a granule file record according to MoveFileParams and updates database accordingly
* @param {MoveFileParams} moveFileParam - Parameter object describing the move operation
* @param {FilePgModel} filesPgModel - FilePgModel instance
* @param {Knex.Transaction | Knex} trx - Knex transaction or (optionally) Knex object
* @param {number | undefined } postgresCumulusGranuleId - postgres internal granule id
* @returns {Promise<Object>} - Returns object of type Omit<ApiFile, 'granuleId>>
*/
export declare function moveGranuleFile(moveFileParam: MoveFileParams, filesPgModel: FilePgModel, trx: Knex.Transaction | Knex, postgresCumulusGranuleId: number | undefined): Promise<Omit<ApiFile, 'granuleId'>>;
/**
 * Returns the input filename stripping off any versioned timestamp.
 *
 * @param {string} filename
 * @returns {string} - filename with timestamp removed
 */
export declare function unversionFilename(filename: string): string;
/**
 * Generates a unique granule ID by appending a truncated MD5 hash of values from
 * a producer provided granule object
 *
 * @param id - An ID associated with the object to be hashed.  Likely the ID
 * assigned by the granule producer
 * @param collectionId - The api collection ID (name___version) associated with the granule
 * @param hashLength - The length of the hash to append to the granuleId.
 * @param includeTimestampHashKey - Boolean value for whether hash string should contain timestamp
 * @returns - A unique granule ID in the format: granuleId_hash.
 */
export declare function generateUniqueGranuleId(id: string, collectionId: string, hashLength: number, includeTimestampHashKey?: boolean): string;
/**
 * Returns a directive on how to act when duplicate files are encountered.
 *
 * @param {Object} event - lambda function event.
 * @param {Object} event.config - the config object
 * @param {Object} event.config.collection - collection object.

 * @returns {DuplicateHandling} - duplicate handling directive.
 */
export declare function duplicateHandlingType(event: EventWithDuplicateHandling): DuplicateHandling;
//# sourceMappingURL=granule.d.ts.map