@pulumi/aws

import * as pulumi from "@pulumi/pulumi"; import * as inputs from "../types/input"; import * as outputs from "../types/output"; /** * Provides a Glue Job resource. * * > Glue functionality, such as monitoring and logging of jobs, is typically managed with the `defaultArguments` argument. See the [Special Parameters Used by AWS Glue](https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html) topic in the Glue developer guide for additional information. * * ## Example Usage * * ### Python Glue Job * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as aws from "@pulumi/aws"; * * // IAM role for Glue jobs * const glueJobRole = new aws.iam.Role("glue_job_role", { * name: "glue-job-role", * assumeRolePolicy: JSON.stringify({ * Version: "2012-10-17", * Statement: [{ * Action: "sts:AssumeRole", * Effect: "Allow", * Principal: { * Service: "glue.amazonaws.com", * }, * }], * }), * }); * const etlJob = new aws.glue.Job("etl_job", { * name: "example-etl-job", * description: "An example Glue ETL job", * roleArn: glueJobRole.arn, * glueVersion: "5.0", * maxRetries: 0, * timeout: 2880, * numberOfWorkers: 2, * workerType: "G.1X", * connections: [example.name], * executionClass: "STANDARD", * command: { * scriptLocation: `s3://${glueScripts.bucket}/jobs/etl_job.py`, * name: "glueetl", * pythonVersion: "3", * }, * notificationProperty: { * notifyDelayAfter: 3, * }, * defaultArguments: { * "--job-language": "python", * "--continuous-log-logGroup": "/aws-glue/jobs", * "--enable-continuous-cloudwatch-log": "true", * "--enable-continuous-log-filter": "true", * "--enable-metrics": "", * "--enable-auto-scaling": "true", * }, * executionProperty: { * maxConcurrentRuns: 1, * }, * tags: { * ManagedBy: "AWS", * }, * }); * const glueEtlScript = new aws.s3.BucketObjectv2("glue_etl_script", { * bucket: glueScripts.id, * key: "jobs/etl_job.py", * source: new pulumi.asset.FileAsset("jobs/etl_job.py"), * }); * ``` * * ### Pythonshell Job * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as aws from "@pulumi/aws"; * * // IAM role for Glue jobs * const glueJobRole = new aws.iam.Role("glue_job_role", { * name: "glue-job-role", * assumeRolePolicy: JSON.stringify({ * Version: "2012-10-17", * Statement: [{ * Action: "sts:AssumeRole", * Effect: "Allow", * Principal: { * Service: "glue.amazonaws.com", * }, * }], * }), * }); * const pythonShellJob = new aws.glue.Job("python_shell_job", { * name: "example-python-shell-job", * description: "An example Python shell job", * roleArn: glueJobRole.arn, * maxCapacity: 0.0625, * maxRetries: 0, * timeout: 2880, * connections: [example.name], * command: { * scriptLocation: `s3://${glueScripts.bucket}/jobs/shell_job.py`, * name: "pythonshell", * pythonVersion: "3.9", * }, * defaultArguments: { * "--job-language": "python", * "--continuous-log-logGroup": "/aws-glue/jobs", * "--enable-continuous-cloudwatch-log": "true", * "library-set": "analytics", * }, * executionProperty: { * maxConcurrentRuns: 1, * }, * tags: { * ManagedBy: "AWS", * }, * }); * const pythonShellScript = new aws.s3.BucketObjectv2("python_shell_script", { * bucket: glueScripts.id, * key: "jobs/shell_job.py", * source: new pulumi.asset.FileAsset("jobs/shell_job.py"), * }); * ``` * * ### Ray Job * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as aws from "@pulumi/aws"; * * const example = new aws.glue.Job("example", { * name: "example", * roleArn: exampleAwsIamRole.arn, * glueVersion: "4.0", * workerType: "Z.2X", * command: { * name: "glueray", * pythonVersion: "3.9", * runtime: "Ray2.4", * scriptLocation: `s3://${exampleAwsS3Bucket.bucket}/example.py`, * }, * }); * ``` * * ### Scala Job * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as aws from "@pulumi/aws"; * * const example = new aws.glue.Job("example", { * name: "example", * roleArn: exampleAwsIamRole.arn, * command: { * scriptLocation: `s3://${exampleAwsS3Bucket.bucket}/example.scala`, * }, * defaultArguments: { * "--job-language": "scala", * }, * }); * ``` * * ### Streaming Job * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as aws from "@pulumi/aws"; * * const example = new aws.glue.Job("example", { * name: "example streaming job", * roleArn: exampleAwsIamRole.arn, * command: { * name: "gluestreaming", * scriptLocation: `s3://${exampleAwsS3Bucket.bucket}/example.script`, * }, * }); * ``` * * ### Enabling CloudWatch Logs and Metrics * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as aws from "@pulumi/aws"; * * const example = new aws.cloudwatch.LogGroup("example", { * name: "example", * retentionInDays: 14, * }); * const exampleJob = new aws.glue.Job("example", {defaultArguments: { * "--continuous-log-logGroup": example.name, * "--enable-continuous-cloudwatch-log": "true", * "--enable-continuous-log-filter": "true", * "--enable-metrics": "", * }}); * ``` * * ## Import * * Using `pulumi import`, import Glue Jobs using `name`. For example: * * ```sh * $ pulumi import aws:glue/job:Job MyJob MyJob * ``` */ export declare class Job extends pulumi.CustomResource { /** * Get an existing Job resource's state with the given name, ID, and optional extra * properties used to qualify the lookup. * * @param name The _unique_ name of the resulting resource. * @param id The _unique_ provider ID of the resource to lookup. * @param state Any extra arguments used during the lookup. * @param opts Optional settings to control the behavior of the CustomResource. */ static get(name: string, id: pulumi.Input<pulumi.ID>, state?: JobState, opts?: pulumi.CustomResourceOptions): Job; /** * Returns true if the given object is an instance of Job. This is designed to work even * when multiple copies of the Pulumi SDK have been loaded into the same process. */ static isInstance(obj: any): obj is Job; /** * Amazon Resource Name (ARN) of Glue Job */ readonly arn: pulumi.Output<string>; /** * The command of the job. Defined below. */ readonly command: pulumi.Output<outputs.glue.JobCommand>; /** * The list of connections used for this job. */ readonly connections: pulumi.Output<string[] | undefined>; /** * The map of default arguments for this job. You can specify arguments here that your own job-execution script consumes, as well as arguments that AWS Glue itself consumes. For information about how to specify and consume your own Job arguments, see the [Calling AWS Glue APIs in Python](http://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-python-calling.html) topic in the developer guide. For information about the key-value pairs that AWS Glue consumes to set up your job, see the [Special Parameters Used by AWS Glue](http://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-python-glue-arguments.html) topic in the developer guide. */ readonly defaultArguments: pulumi.Output<{ [key: string]: string; } | undefined>; /** * Description of the job. */ readonly description: pulumi.Output<string | undefined>; /** * Indicates whether the job is run with a standard or flexible execution class. The standard execution class is ideal for time-sensitive workloads that require fast job startup and dedicated resources. Valid value: `FLEX`, `STANDARD`. */ readonly executionClass: pulumi.Output<string | undefined>; /** * Execution property of the job. Defined below. */ readonly executionProperty: pulumi.Output<outputs.glue.JobExecutionProperty>; /** * The version of glue to use, for example "1.0". Ray jobs should set this to 4.0 or greater. For information about available versions, see the [AWS Glue Release Notes](https://docs.aws.amazon.com/glue/latest/dg/release-notes.html). */ readonly glueVersion: pulumi.Output<string>; /** * Describes how a job was created. Valid values are `SCRIPT`, `NOTEBOOK` and `VISUAL`. */ readonly jobMode: pulumi.Output<string>; /** * Specifies whether job run queuing is enabled for the job runs for this job. A value of true means job run queuing is enabled for the job runs. If false or not populated, the job runs will not be considered for queueing. */ readonly jobRunQueuingEnabled: pulumi.Output<boolean | undefined>; /** * Specifies the day of the week and hour for the maintenance window for streaming jobs. */ readonly maintenanceWindow: pulumi.Output<string | undefined>; /** * The maximum number of AWS Glue data processing units (DPUs) that can be allocated when this job runs. `Required` when `pythonshell` is set, accept either `0.0625` or `1.0`. Use `numberOfWorkers` and `workerType` arguments instead with `glueVersion` `2.0` and above. */ readonly maxCapacity: pulumi.Output<number>; /** * The maximum number of times to retry this job if it fails. */ readonly maxRetries: pulumi.Output<number | undefined>; /** * The name you assign to this job. It must be unique in your account. */ readonly name: pulumi.Output<string>; /** * Non-overridable arguments for this job, specified as name-value pairs. */ readonly nonOverridableArguments: pulumi.Output<{ [key: string]: string; } | undefined>; /** * Notification property of the job. Defined below. */ readonly notificationProperty: pulumi.Output<outputs.glue.JobNotificationProperty>; /** * The number of workers of a defined workerType that are allocated when a job runs. */ readonly numberOfWorkers: pulumi.Output<number>; /** * Region where this resource will be [managed](https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints). Defaults to the Region set in the provider configuration. */ readonly region: pulumi.Output<string>; /** * The ARN of the IAM role associated with this job. */ readonly roleArn: pulumi.Output<string>; /** * The name of the Security Configuration to be associated with the job. */ readonly securityConfiguration: pulumi.Output<string | undefined>; /** * The details for a source control configuration for a job, allowing synchronization of job artifacts to or from a remote repository. Defined below. */ readonly sourceControlDetails: pulumi.Output<outputs.glue.JobSourceControlDetails | undefined>; /** * Key-value map of resource tags. If configured with a provider `defaultTags` configuration block present, tags with matching keys will overwrite those defined at the provider-level. */ readonly tags: pulumi.Output<{ [key: string]: string; } | undefined>; /** * A map of tags assigned to the resource, including those inherited from the provider `defaultTags` configuration block. */ readonly tagsAll: pulumi.Output<{ [key: string]: string; }>; /** * The job timeout in minutes. The default is 2880 minutes (48 hours) for `glueetl` and `pythonshell` jobs, and null (unlimited) for `gluestreaming` jobs. */ readonly timeout: pulumi.Output<number>; /** * The type of predefined worker that is allocated when a job runs. Valid values: `Standard`, `G.1X`, `G.2X`, `G.025X`, `G.4X`, `G.8X`, `G.12X`, `G.16X`, `R.1X`, `R.2X`, `R.4X`, `R.8X`, `Z.2X` (Ray jobs). See the [AWS documentation](https://docs.aws.amazon.com/glue/latest/dg/worker-types.html) for details. */ readonly workerType: pulumi.Output<string>; /** * Create a Job resource with the given unique name, arguments, and options. * * @param name The _unique_ name of the resource. * @param args The arguments to use to populate this resource's properties. * @param opts A bag of options that control this resource's behavior. */ constructor(name: string, args: JobArgs, opts?: pulumi.CustomResourceOptions); } /** * Input properties used for looking up and filtering Job resources. */ export interface JobState { /** * Amazon Resource Name (ARN) of Glue Job */ arn?: pulumi.Input<string>; /** * The command of the job. Defined below. */ command?: pulumi.Input<inputs.glue.JobCommand>; /** * The list of connections used for this job. */ connections?: pulumi.Input<pulumi.Input<string>[]>; /** * The map of default arguments for this job. You can specify arguments here that your own job-execution script consumes, as well as arguments that AWS Glue itself consumes. For information about how to specify and consume your own Job arguments, see the [Calling AWS Glue APIs in Python](http://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-python-calling.html) topic in the developer guide. For information about the key-value pairs that AWS Glue consumes to set up your job, see the [Special Parameters Used by AWS Glue](http://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-python-glue-arguments.html) topic in the developer guide. */ defaultArguments?: pulumi.Input<{ [key: string]: pulumi.Input<string>; }>; /** * Description of the job. */ description?: pulumi.Input<string>; /** * Indicates whether the job is run with a standard or flexible execution class. The standard execution class is ideal for time-sensitive workloads that require fast job startup and dedicated resources. Valid value: `FLEX`, `STANDARD`. */ executionClass?: pulumi.Input<string>; /** * Execution property of the job. Defined below. */ executionProperty?: pulumi.Input<inputs.glue.JobExecutionProperty>; /** * The version of glue to use, for example "1.0". Ray jobs should set this to 4.0 or greater. For information about available versions, see the [AWS Glue Release Notes](https://docs.aws.amazon.com/glue/latest/dg/release-notes.html). */ glueVersion?: pulumi.Input<string>; /** * Describes how a job was created. Valid values are `SCRIPT`, `NOTEBOOK` and `VISUAL`. */ jobMode?: pulumi.Input<string>; /** * Specifies whether job run queuing is enabled for the job runs for this job. A value of true means job run queuing is enabled for the job runs. If false or not populated, the job runs will not be considered for queueing. */ jobRunQueuingEnabled?: pulumi.Input<boolean>; /** * Specifies the day of the week and hour for the maintenance window for streaming jobs. */ maintenanceWindow?: pulumi.Input<string>; /** * The maximum number of AWS Glue data processing units (DPUs) that can be allocated when this job runs. `Required` when `pythonshell` is set, accept either `0.0625` or `1.0`. Use `numberOfWorkers` and `workerType` arguments instead with `glueVersion` `2.0` and above. */ maxCapacity?: pulumi.Input<number>; /** * The maximum number of times to retry this job if it fails. */ maxRetries?: pulumi.Input<number>; /** * The name you assign to this job. It must be unique in your account. */ name?: pulumi.Input<string>; /** * Non-overridable arguments for this job, specified as name-value pairs. */ nonOverridableArguments?: pulumi.Input<{ [key: string]: pulumi.Input<string>; }>; /** * Notification property of the job. Defined below. */ notificationProperty?: pulumi.Input<inputs.glue.JobNotificationProperty>; /** * The number of workers of a defined workerType that are allocated when a job runs. */ numberOfWorkers?: pulumi.Input<number>; /** * Region where this resource will be [managed](https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints). Defaults to the Region set in the provider configuration. */ region?: pulumi.Input<string>; /** * The ARN of the IAM role associated with this job. */ roleArn?: pulumi.Input<string>; /** * The name of the Security Configuration to be associated with the job. */ securityConfiguration?: pulumi.Input<string>; /** * The details for a source control configuration for a job, allowing synchronization of job artifacts to or from a remote repository. Defined below. */ sourceControlDetails?: pulumi.Input<inputs.glue.JobSourceControlDetails>; /** * Key-value map of resource tags. If configured with a provider `defaultTags` configuration block present, tags with matching keys will overwrite those defined at the provider-level. */ tags?: pulumi.Input<{ [key: string]: pulumi.Input<string>; }>; /** * A map of tags assigned to the resource, including those inherited from the provider `defaultTags` configuration block. */ tagsAll?: pulumi.Input<{ [key: string]: pulumi.Input<string>; }>; /** * The job timeout in minutes. The default is 2880 minutes (48 hours) for `glueetl` and `pythonshell` jobs, and null (unlimited) for `gluestreaming` jobs. */ timeout?: pulumi.Input<number>; /** * The type of predefined worker that is allocated when a job runs. Valid values: `Standard`, `G.1X`, `G.2X`, `G.025X`, `G.4X`, `G.8X`, `G.12X`, `G.16X`, `R.1X`, `R.2X`, `R.4X`, `R.8X`, `Z.2X` (Ray jobs). See the [AWS documentation](https://docs.aws.amazon.com/glue/latest/dg/worker-types.html) for details. */ workerType?: pulumi.Input<string>; } /** * The set of arguments for constructing a Job resource. */ export interface JobArgs { /** * The command of the job. Defined below. */ command: pulumi.Input<inputs.glue.JobCommand>; /** * The list of connections used for this job. */ connections?: pulumi.Input<pulumi.Input<string>[]>; /** * The map of default arguments for this job. You can specify arguments here that your own job-execution script consumes, as well as arguments that AWS Glue itself consumes. For information about how to specify and consume your own Job arguments, see the [Calling AWS Glue APIs in Python](http://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-python-calling.html) topic in the developer guide. For information about the key-value pairs that AWS Glue consumes to set up your job, see the [Special Parameters Used by AWS Glue](http://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-python-glue-arguments.html) topic in the developer guide. */ defaultArguments?: pulumi.Input<{ [key: string]: pulumi.Input<string>; }>; /** * Description of the job. */ description?: pulumi.Input<string>; /** * Indicates whether the job is run with a standard or flexible execution class. The standard execution class is ideal for time-sensitive workloads that require fast job startup and dedicated resources. Valid value: `FLEX`, `STANDARD`. */ executionClass?: pulumi.Input<string>; /** * Execution property of the job. Defined below. */ executionProperty?: pulumi.Input<inputs.glue.JobExecutionProperty>; /** * The version of glue to use, for example "1.0". Ray jobs should set this to 4.0 or greater. For information about available versions, see the [AWS Glue Release Notes](https://docs.aws.amazon.com/glue/latest/dg/release-notes.html). */ glueVersion?: pulumi.Input<string>; /** * Describes how a job was created. Valid values are `SCRIPT`, `NOTEBOOK` and `VISUAL`. */ jobMode?: pulumi.Input<string>; /** * Specifies whether job run queuing is enabled for the job runs for this job. A value of true means job run queuing is enabled for the job runs. If false or not populated, the job runs will not be considered for queueing. */ jobRunQueuingEnabled?: pulumi.Input<boolean>; /** * Specifies the day of the week and hour for the maintenance window for streaming jobs. */ maintenanceWindow?: pulumi.Input<string>; /** * The maximum number of AWS Glue data processing units (DPUs) that can be allocated when this job runs. `Required` when `pythonshell` is set, accept either `0.0625` or `1.0`. Use `numberOfWorkers` and `workerType` arguments instead with `glueVersion` `2.0` and above. */ maxCapacity?: pulumi.Input<number>; /** * The maximum number of times to retry this job if it fails. */ maxRetries?: pulumi.Input<number>; /** * The name you assign to this job. It must be unique in your account. */ name?: pulumi.Input<string>; /** * Non-overridable arguments for this job, specified as name-value pairs. */ nonOverridableArguments?: pulumi.Input<{ [key: string]: pulumi.Input<string>; }>; /** * Notification property of the job. Defined below. */ notificationProperty?: pulumi.Input<inputs.glue.JobNotificationProperty>; /** * The number of workers of a defined workerType that are allocated when a job runs. */ numberOfWorkers?: pulumi.Input<number>; /** * Region where this resource will be [managed](https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints). Defaults to the Region set in the provider configuration. */ region?: pulumi.Input<string>; /** * The ARN of the IAM role associated with this job. */ roleArn: pulumi.Input<string>; /** * The name of the Security Configuration to be associated with the job. */ securityConfiguration?: pulumi.Input<string>; /** * The details for a source control configuration for a job, allowing synchronization of job artifacts to or from a remote repository. Defined below. */ sourceControlDetails?: pulumi.Input<inputs.glue.JobSourceControlDetails>; /** * Key-value map of resource tags. If configured with a provider `defaultTags` configuration block present, tags with matching keys will overwrite those defined at the provider-level. */ tags?: pulumi.Input<{ [key: string]: pulumi.Input<string>; }>; /** * The job timeout in minutes. The default is 2880 minutes (48 hours) for `glueetl` and `pythonshell` jobs, and null (unlimited) for `gluestreaming` jobs. */ timeout?: pulumi.Input<number>; /** * The type of predefined worker that is allocated when a job runs. Valid values: `Standard`, `G.1X`, `G.2X`, `G.025X`, `G.4X`, `G.8X`, `G.12X`, `G.16X`, `R.1X`, `R.2X`, `R.4X`, `R.8X`, `Z.2X` (Ray jobs). See the [AWS documentation](https://docs.aws.amazon.com/glue/latest/dg/worker-types.html) for details. */ workerType?: pulumi.Input<string>; }