@aws-cdk/aws-glue-alpha/lib/jobs/spark-job.d.ts

The CDK Construct Library for AWS::Glue

import * as iam from 'aws-cdk-lib/aws-iam';
import * as s3 from 'aws-cdk-lib/aws-s3';
import * as constructs from 'constructs';
import { Code } from '../code';
import { Job, JobProps } from './job';
/**
 * Code props for different {@link Code} assets used by different types of Spark jobs.
 */
export interface SparkExtraCodeProps {
    /**
     * Extra Python Files S3 URL (optional)
     * S3 URL where additional python dependencies are located
     *
     * @default - no extra files
     */
    readonly extraPythonFiles?: Code[];
    /**
     * Additional files, such as configuration files that AWS Glue copies to the working directory of your script before executing it.
     *
     * @default - no extra files specified.
     *
     * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html
     */
    readonly extraFiles?: Code[];
    /**
     * Extra Jars S3 URL (optional)
     * S3 URL where additional jar dependencies are located
     * @default - no extra jar files
     */
    readonly extraJars?: Code[];
    /**
     * Setting this value to true prioritizes the customer's extra JAR files in the classpath.
     *
     * @default false - priority is not given to user-provided jars
     *
     * @see `--user-jars-first` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html
     */
    readonly extraJarsFirst?: boolean;
}
/**
 * Properties for enabling Spark UI monitoring feature for Spark-based Glue jobs.
 *
 * @see https://docs.aws.amazon.com/glue/latest/dg/monitor-spark-ui-jobs.html
 * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html
 */
export interface SparkUIProps {
    /**
     * The bucket where the Glue job stores the logs.
     *
     * @default a new bucket will be created.
     */
    readonly bucket?: s3.IBucket;
    /**
     * The path inside the bucket (objects prefix) where the Glue job stores the logs.
     * Use format `'/foo/bar'`
     *
     * @default - the logs will be written at the root of the bucket
     */
    readonly prefix?: string;
}
/**
 * The Spark UI logging location.
 *
 * @see https://docs.aws.amazon.com/glue/latest/dg/monitor-spark-ui-jobs.html
 * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html
 */
export interface SparkUILoggingLocation {
    /**
     * The bucket where the Glue job stores the logs.
     */
    readonly bucket: s3.IBucket;
    /**
     * The path inside the bucket (objects prefix) where the Glue job stores the logs.
     *
     * @default '/' - the logs will be written at the root of the bucket
     */
    readonly prefix?: string;
}
/**
 * Common properties for different types of Spark jobs.
 */
export interface SparkJobProps extends JobProps {
    /**
     * Enables the Spark UI debugging and monitoring with the specified props.
     *
     * @default - Spark UI debugging and monitoring is disabled.
     *
     * @see https://docs.aws.amazon.com/glue/latest/dg/monitor-spark-ui-jobs.html
     * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html
     */
    readonly sparkUI?: SparkUIProps;
    /**
     * Enable profiling metrics for the Glue job.
     *
     * When enabled, adds '--enable-metrics' to job arguments.
     *
     * @default true
     */
    readonly enableMetrics?: boolean;
    /**
     * Enable observability metrics for the Glue job.
     *
     * When enabled, adds '--enable-observability-metrics': 'true' to job arguments.
     *
     * @default true
     */
    readonly enableObservabilityMetrics?: boolean;
}
/**
 * Base class for different types of Spark Jobs.
 */
export declare abstract class SparkJob extends Job {
    readonly role: iam.IRole;
    readonly grantPrincipal: iam.IPrincipal;
    /**
     * The Spark UI logs location if Spark UI monitoring and debugging is enabled.
     *
     * @see https://docs.aws.amazon.com/glue/latest/dg/monitor-spark-ui-jobs.html
     * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html
     */
    readonly sparkUILoggingLocation?: SparkUILoggingLocation;
    constructor(scope: constructs.Construct, id: string, props: SparkJobProps);
    protected nonExecutableCommonArguments(props: SparkJobProps): {
        [key: string]: string;
    };
    /**
     * Set the arguments for extra {@link Code}-related properties
     */
    protected setupExtraCodeArguments(args: {
        [key: string]: string;
    }, props: SparkExtraCodeProps): void;
    private setupSparkUILoggingLocation;
}