@aws-solutions-constructs/core/lib/glue-job-helper.d.ts

Core CDK Construct for patterns library

/**
 *  Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 *
 *  Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
 *  with the License. A copy of the License is located at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES
 *  OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions
 *  and limitations under the License.
 */
import { Construct } from 'constructs';
import * as glue from 'aws-cdk-lib/aws-glue';
import { IRole, Role } from 'aws-cdk-lib/aws-iam';
import { Bucket, BucketProps } from 'aws-cdk-lib/aws-s3';
import * as s3assets from "aws-cdk-lib/aws-s3-assets";
/**
 * Enumeration of data store types that could include S3, DynamoDB, DocumentDB, RDS or Redshift. Current
 * construct implementation only supports S3, but potential to add other output types in the future
 */
export declare enum SinkStoreType {
    S3 = "S3"
}
/**
 * Interface to define potential outputs to allow the construct define additional output destinations for ETL
 * transformation
 */
export interface SinkDataStoreProps {
    /**
     * Sink data store type
     */
    readonly datastoreType: SinkStoreType;
    /**
     * The output S3 location where the data should be written. The provided S3 bucket will be used to pass
     * the output location to the etl script as an argument to the AWS Glue job.
     *
     * If no location is provided, it will check if @outputBucketProps are provided. If not it will create a new
     * bucket if the @datastoreType is S3.
     *
     * The argument key is `output_path`. The value of the argument can be retrieve in the python script
     * as follows:
     *  getResolvedOptions(sys.argv, ["JOB_NAME", "output_path", <other arguments that are passed> ])
     *  output_path = args["output_path"]
     */
    readonly existingS3OutputBucket?: Bucket;
    /**
     * If @existingS3OutputBUcket is provided, this parameter is ignored. If this parameter is not provided,
     * the construct will create a new bucket if the @datastoreType is S3.
     */
    readonly outputBucketProps?: BucketProps;
}
export interface BuildGlueJobProps {
    /**
     * Glue ETL job properties.
     */
    readonly glueJobProps?: glue.CfnJobProps | any;
    /**
     * Existing instance of the S3 bucket object, if this is set then the script location is ignored.
     */
    readonly existingCfnJob?: glue.CfnJob;
    /**
     * AWS Glue table
     */
    readonly table: glue.CfnTable;
    /**
     * AWS Glue database
     */
    readonly database: glue.CfnDatabase;
    /**
     * Output storage options
     */
    readonly outputDataStore?: SinkDataStoreProps;
    /**
     * Asset instance for the ETL code that performs Glue Job transformation
     *
     * @default - None
     */
    readonly etlCodeAsset?: s3assets.Asset;
}
export interface BuildGlueJobResponse {
    readonly job: glue.CfnJob;
    readonly role: IRole;
    readonly bucket?: Bucket;
    readonly loggingBucket?: Bucket;
}
/**
 * @internal This is an internal core function and should not be called directly by Solutions Constructs clients.
 */
export declare function buildGlueJob(scope: Construct, props: BuildGlueJobProps): BuildGlueJobResponse;
export interface DeployGlueJobResponse {
    readonly job: glue.CfnJob;
    readonly role: IRole;
    readonly bucket?: Bucket;
    readonly loggingBucket?: Bucket;
}
/**
 * @internal This is an internal core function and should not be called directly by Solutions Constructs clients.
 */
export declare function deployGlueJob(scope: Construct, glueJobProps: glue.CfnJobProps, database: glue.CfnDatabase, table: glue.CfnTable, outputDataStore: SinkDataStoreProps, etlCodeAsset?: s3assets.Asset): DeployGlueJobResponse;
/**
 * @internal This is an internal core function and should not be called directly by Solutions Constructs clients.
 *
 * This is a helper method to create the Role required for the Glue Job. If a role is already created then this
 * method is not required to be called.
 *
 * @param scope - The AWS Construct under which the role is to be created
 */
export declare function createGlueJobRole(scope: Construct): Role;
/**
 * @internal This is an internal core function and should not be called directly by Solutions Constructs clients.
 *
 * This method creates an AWS Glue table. The method is called when an existing Glue table is not provided
 */
export declare function createGlueTable(scope: Construct, database: glue.CfnDatabase, tableProps?: glue.CfnTableProps, fieldSchema?: glue.CfnTable.ColumnProperty[], sourceType?: string, parameters?: any): glue.CfnTable;
/**
 * @internal This is an internal core function and should not be called directly by Solutions Constructs clients.
 *
 * This method creates an AWS Glue database. The method is only called with an existing Glue database type is not provided.
 * The method uses the user provided props to override the default props for the Glue database
 *
 * @param scope
 * @param databaseProps
 */
export declare function createGlueDatabase(scope: Construct, databaseProps?: glue.CfnDatabaseProps): glue.CfnDatabase;