@pulumi/gcp

import * as pulumi from "@pulumi/pulumi"; import * as inputs from "../types/input"; import * as outputs from "../types/output"; /** * Dataproc Serverless Batches lets you run Spark workloads without requiring you to * provision and manage your own Dataproc cluster. * * To get more information about Batch, see: * * * [API documentation](https://cloud.google.com/dataproc-serverless/docs/reference/rest/v1/projects.locations.batches) * * How-to Guides * * [Dataproc Serverless Batches Intro](https://cloud.google.com/dataproc-serverless/docs/overview) * * ## Example Usage * * ### Dataproc Batch Spark * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const exampleBatchSpark = new gcp.dataproc.Batch("example_batch_spark", { * batchId: "tf-test-batch_88722", * location: "us-central1", * labels: { * batch_test: "terraform", * }, * runtimeConfig: { * properties: { * "spark.dynamicAllocation.enabled": "false", * "spark.executor.instances": "2", * }, * }, * environmentConfig: { * executionConfig: { * subnetworkUri: "default", * ttl: "3600s", * networkTags: ["tag1"], * }, * }, * sparkBatch: { * mainClass: "org.apache.spark.examples.SparkPi", * args: ["10"], * jarFileUris: ["file:///usr/lib/spark/examples/jars/spark-examples.jar"], * }, * }); * ``` * ### Dataproc Batch Spark Full * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const project = gcp.organizations.getProject({}); * const gcsAccount = gcp.storage.getProjectServiceAccount({}); * const bucket = new gcp.storage.Bucket("bucket", { * uniformBucketLevelAccess: true, * name: "dataproc-bucket", * location: "US", * forceDestroy: true, * }); * const cryptoKeyMember1 = new gcp.kms.CryptoKeyIAMMember("crypto_key_member_1", { * cryptoKeyId: "example-key", * role: "roles/cloudkms.cryptoKeyEncrypterDecrypter", * member: project.then(project => `serviceAccount:service-${project.number}@dataproc-accounts.iam.gserviceaccount.com`), * }); * const ms = new gcp.dataproc.MetastoreService("ms", { * serviceId: "dataproc-batch", * location: "us-central1", * port: 9080, * tier: "DEVELOPER", * maintenanceWindow: { * hourOfDay: 2, * dayOfWeek: "SUNDAY", * }, * hiveMetastoreConfig: { * version: "3.1.2", * }, * }); * const basic = new gcp.dataproc.Cluster("basic", { * name: "dataproc-batch", * region: "us-central1", * clusterConfig: { * softwareConfig: { * overrideProperties: { * "dataproc:dataproc.allow.zero.workers": "true", * "spark:spark.history.fs.logDirectory": pulumi.interpolate`gs://${bucket.name}/*/spark-job-history`, * }, * }, * endpointConfig: { * enableHttpPortAccess: true, * }, * masterConfig: { * numInstances: 1, * machineType: "e2-standard-2", * diskConfig: { * bootDiskSizeGb: 35, * }, * }, * metastoreConfig: { * dataprocMetastoreService: ms.name, * }, * }, * }); * const exampleBatchSpark = new gcp.dataproc.Batch("example_batch_spark", { * batchId: "dataproc-batch", * location: "us-central1", * labels: { * batch_test: "terraform", * }, * runtimeConfig: { * properties: { * "spark.dynamicAllocation.enabled": "false", * "spark.executor.instances": "2", * }, * version: "2.2", * }, * environmentConfig: { * executionConfig: { * ttl: "3600s", * networkTags: ["tag1"], * kmsKey: "example-key", * networkUri: "default", * serviceAccount: project.then(project => `${project.number}-compute@developer.gserviceaccount.com`), * stagingBucket: bucket.name, * }, * peripheralsConfig: { * metastoreService: ms.name, * sparkHistoryServerConfig: { * dataprocCluster: basic.id, * }, * }, * }, * sparkBatch: { * mainClass: "org.apache.spark.examples.SparkPi", * args: ["10"], * jarFileUris: ["file:///usr/lib/spark/examples/jars/spark-examples.jar"], * }, * }, { * dependsOn: [cryptoKeyMember1], * }); * ``` * ### Dataproc Batch Sparksql * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const exampleBatchSparsql = new gcp.dataproc.Batch("example_batch_sparsql", { * batchId: "tf-test-batch_39249", * location: "us-central1", * runtimeConfig: { * properties: { * "spark.dynamicAllocation.enabled": "false", * "spark.executor.instances": "2", * }, * }, * environmentConfig: { * executionConfig: { * subnetworkUri: "default", * }, * }, * sparkSqlBatch: { * queryFileUri: "gs://dataproc-examples/spark-sql/natality/cigarette_correlations.sql", * jarFileUris: ["file:///usr/lib/spark/examples/jars/spark-examples.jar"], * queryVariables: { * name: "value", * }, * }, * }); * ``` * ### Dataproc Batch Pyspark * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const exampleBatchPyspark = new gcp.dataproc.Batch("example_batch_pyspark", { * batchId: "tf-test-batch_74391", * location: "us-central1", * runtimeConfig: { * properties: { * "spark.dynamicAllocation.enabled": "false", * "spark.executor.instances": "2", * }, * }, * environmentConfig: { * executionConfig: { * subnetworkUri: "default", * }, * }, * pysparkBatch: { * mainPythonFileUri: "https://storage.googleapis.com/terraform-batches/test_util.py", * args: ["10"], * jarFileUris: ["file:///usr/lib/spark/examples/jars/spark-examples.jar"], * pythonFileUris: ["gs://dataproc-examples/pyspark/hello-world/hello-world.py"], * archiveUris: [ * "https://storage.googleapis.com/terraform-batches/animals.txt.tar.gz#unpacked", * "https://storage.googleapis.com/terraform-batches/animals.txt.jar", * "https://storage.googleapis.com/terraform-batches/animals.txt", * ], * fileUris: ["https://storage.googleapis.com/terraform-batches/people.txt"], * }, * }); * ``` * ### Dataproc Batch Sparkr * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const exampleBatchSparkr = new gcp.dataproc.Batch("example_batch_sparkr", { * batchId: "tf-test-batch_16511", * location: "us-central1", * labels: { * batch_test: "terraform", * }, * runtimeConfig: { * properties: { * "spark.dynamicAllocation.enabled": "false", * "spark.executor.instances": "2", * }, * }, * environmentConfig: { * executionConfig: { * subnetworkUri: "default", * ttl: "3600s", * networkTags: ["tag1"], * }, * }, * sparkRBatch: { * mainRFileUri: "https://storage.googleapis.com/terraform-batches/spark-r-flights.r", * args: ["https://storage.googleapis.com/terraform-batches/flights.csv"], * }, * }); * ``` * ### Dataproc Batch Autotuning * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const exampleBatchAutotuning = new gcp.dataproc.Batch("example_batch_autotuning", { * batchId: "tf-test-batch_8493", * location: "us-central1", * labels: { * batch_test: "terraform", * }, * runtimeConfig: { * version: "2.2", * properties: { * "spark.dynamicAllocation.enabled": "false", * "spark.executor.instances": "2", * }, * cohort: "tf-dataproc-batch-example", * autotuningConfig: { * scenarios: [ * "SCALING", * "MEMORY", * ], * }, * }, * environmentConfig: { * executionConfig: { * subnetworkUri: "default", * ttl: "3600s", * }, * }, * sparkBatch: { * mainClass: "org.apache.spark.examples.SparkPi", * args: ["10"], * jarFileUris: ["file:///usr/lib/spark/examples/jars/spark-examples.jar"], * }, * }); * ``` * * ## Import * * Batch can be imported using any of these accepted formats: * * * `projects/{{project}}/locations/{{location}}/batches/{{batch_id}}` * * * `{{project}}/{{location}}/{{batch_id}}` * * * `{{location}}/{{batch_id}}` * * When using the `pulumi import` command, Batch can be imported using one of the formats above. For example: * * ```sh * $ pulumi import gcp:dataproc/batch:Batch default projects/{{project}}/locations/{{location}}/batches/{{batch_id}} * ``` * * ```sh * $ pulumi import gcp:dataproc/batch:Batch default {{project}}/{{location}}/{{batch_id}} * ``` * * ```sh * $ pulumi import gcp:dataproc/batch:Batch default {{location}}/{{batch_id}} * ``` */ export declare class Batch extends pulumi.CustomResource { /** * Get an existing Batch resource's state with the given name, ID, and optional extra * properties used to qualify the lookup. * * @param name The _unique_ name of the resulting resource. * @param id The _unique_ provider ID of the resource to lookup. * @param state Any extra arguments used during the lookup. * @param opts Optional settings to control the behavior of the CustomResource. */ static get(name: string, id: pulumi.Input<pulumi.ID>, state?: BatchState, opts?: pulumi.CustomResourceOptions): Batch; /** * Returns true if the given object is an instance of Batch. This is designed to work even * when multiple copies of the Pulumi SDK have been loaded into the same process. */ static isInstance(obj: any): obj is Batch; /** * The ID to use for the batch, which will become the final component of the batch's resource name. * This value must be 4-63 characters. Valid characters are /[a-z][0-9]-/. */ readonly batchId: pulumi.Output<string | undefined>; /** * The time when the batch was created. */ readonly createTime: pulumi.Output<string>; /** * The email address of the user who created the batch. */ readonly creator: pulumi.Output<string>; /** * All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services. */ readonly effectiveLabels: pulumi.Output<{ [key: string]: string; }>; /** * Environment configuration for the batch execution. * Structure is documented below. */ readonly environmentConfig: pulumi.Output<outputs.dataproc.BatchEnvironmentConfig | undefined>; /** * The labels to associate with this batch. * * **Note**: This field is non-authoritative, and will only manage the labels present in your configuration. * Please refer to the field `effectiveLabels` for all of the labels present on the resource. */ readonly labels: pulumi.Output<{ [key: string]: string; } | undefined>; /** * The location in which the batch will be created in. */ readonly location: pulumi.Output<string | undefined>; /** * The resource name of the batch. */ readonly name: pulumi.Output<string>; /** * The resource name of the operation associated with this batch. */ readonly operation: pulumi.Output<string>; /** * The ID of the project in which the resource belongs. * If it is not provided, the provider project is used. */ readonly project: pulumi.Output<string>; /** * The combination of labels configured directly on the resource * and default labels configured on the provider. */ readonly pulumiLabels: pulumi.Output<{ [key: string]: string; }>; /** * PySpark batch config. * Structure is documented below. */ readonly pysparkBatch: pulumi.Output<outputs.dataproc.BatchPysparkBatch | undefined>; /** * Runtime configuration for the batch execution. * Structure is documented below. */ readonly runtimeConfig: pulumi.Output<outputs.dataproc.BatchRuntimeConfig | undefined>; /** * Runtime information about batch execution. * Structure is documented below. */ readonly runtimeInfos: pulumi.Output<outputs.dataproc.BatchRuntimeInfo[]>; /** * Spark batch config. * Structure is documented below. */ readonly sparkBatch: pulumi.Output<outputs.dataproc.BatchSparkBatch | undefined>; /** * SparkR batch config. * Structure is documented below. */ readonly sparkRBatch: pulumi.Output<outputs.dataproc.BatchSparkRBatch | undefined>; /** * Spark SQL batch config. * Structure is documented below. */ readonly sparkSqlBatch: pulumi.Output<outputs.dataproc.BatchSparkSqlBatch | undefined>; /** * (Output) * The state of the batch at this point in history. For possible values, see the [API documentation](https://cloud.google.com/dataproc-serverless/docs/reference/rest/v1/projects.locations.batches#State). */ readonly state: pulumi.Output<string>; /** * Historical state information for the batch. * Structure is documented below. */ readonly stateHistories: pulumi.Output<outputs.dataproc.BatchStateHistory[]>; /** * (Output) * Details about the state at this point in history. */ readonly stateMessage: pulumi.Output<string>; /** * Batch state details, such as a failure description if the state is FAILED. */ readonly stateTime: pulumi.Output<string>; /** * A batch UUID (Unique Universal Identifier). The service generates this value when it creates the batch. */ readonly uuid: pulumi.Output<string>; /** * Create a Batch resource with the given unique name, arguments, and options. * * @param name The _unique_ name of the resource. * @param args The arguments to use to populate this resource's properties. * @param opts A bag of options that control this resource's behavior. */ constructor(name: string, args?: BatchArgs, opts?: pulumi.CustomResourceOptions); } /** * Input properties used for looking up and filtering Batch resources. */ export interface BatchState { /** * The ID to use for the batch, which will become the final component of the batch's resource name. * This value must be 4-63 characters. Valid characters are /[a-z][0-9]-/. */ batchId?: pulumi.Input<string>; /** * The time when the batch was created. */ createTime?: pulumi.Input<string>; /** * The email address of the user who created the batch. */ creator?: pulumi.Input<string>; /** * All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services. */ effectiveLabels?: pulumi.Input<{ [key: string]: pulumi.Input<string>; }>; /** * Environment configuration for the batch execution. * Structure is documented below. */ environmentConfig?: pulumi.Input<inputs.dataproc.BatchEnvironmentConfig>; /** * The labels to associate with this batch. * * **Note**: This field is non-authoritative, and will only manage the labels present in your configuration. * Please refer to the field `effectiveLabels` for all of the labels present on the resource. */ labels?: pulumi.Input<{ [key: string]: pulumi.Input<string>; }>; /** * The location in which the batch will be created in. */ location?: pulumi.Input<string>; /** * The resource name of the batch. */ name?: pulumi.Input<string>; /** * The resource name of the operation associated with this batch. */ operation?: pulumi.Input<string>; /** * The ID of the project in which the resource belongs. * If it is not provided, the provider project is used. */ project?: pulumi.Input<string>; /** * The combination of labels configured directly on the resource * and default labels configured on the provider. */ pulumiLabels?: pulumi.Input<{ [key: string]: pulumi.Input<string>; }>; /** * PySpark batch config. * Structure is documented below. */ pysparkBatch?: pulumi.Input<inputs.dataproc.BatchPysparkBatch>; /** * Runtime configuration for the batch execution. * Structure is documented below. */ runtimeConfig?: pulumi.Input<inputs.dataproc.BatchRuntimeConfig>; /** * Runtime information about batch execution. * Structure is documented below. */ runtimeInfos?: pulumi.Input<pulumi.Input<inputs.dataproc.BatchRuntimeInfo>[]>; /** * Spark batch config. * Structure is documented below. */ sparkBatch?: pulumi.Input<inputs.dataproc.BatchSparkBatch>; /** * SparkR batch config. * Structure is documented below. */ sparkRBatch?: pulumi.Input<inputs.dataproc.BatchSparkRBatch>; /** * Spark SQL batch config. * Structure is documented below. */ sparkSqlBatch?: pulumi.Input<inputs.dataproc.BatchSparkSqlBatch>; /** * (Output) * The state of the batch at this point in history. For possible values, see the [API documentation](https://cloud.google.com/dataproc-serverless/docs/reference/rest/v1/projects.locations.batches#State). */ state?: pulumi.Input<string>; /** * Historical state information for the batch. * Structure is documented below. */ stateHistories?: pulumi.Input<pulumi.Input<inputs.dataproc.BatchStateHistory>[]>; /** * (Output) * Details about the state at this point in history. */ stateMessage?: pulumi.Input<string>; /** * Batch state details, such as a failure description if the state is FAILED. */ stateTime?: pulumi.Input<string>; /** * A batch UUID (Unique Universal Identifier). The service generates this value when it creates the batch. */ uuid?: pulumi.Input<string>; } /** * The set of arguments for constructing a Batch resource. */ export interface BatchArgs { /** * The ID to use for the batch, which will become the final component of the batch's resource name. * This value must be 4-63 characters. Valid characters are /[a-z][0-9]-/. */ batchId?: pulumi.Input<string>; /** * Environment configuration for the batch execution. * Structure is documented below. */ environmentConfig?: pulumi.Input<inputs.dataproc.BatchEnvironmentConfig>; /** * The labels to associate with this batch. * * **Note**: This field is non-authoritative, and will only manage the labels present in your configuration. * Please refer to the field `effectiveLabels` for all of the labels present on the resource. */ labels?: pulumi.Input<{ [key: string]: pulumi.Input<string>; }>; /** * The location in which the batch will be created in. */ location?: pulumi.Input<string>; /** * The ID of the project in which the resource belongs. * If it is not provided, the provider project is used. */ project?: pulumi.Input<string>; /** * PySpark batch config. * Structure is documented below. */ pysparkBatch?: pulumi.Input<inputs.dataproc.BatchPysparkBatch>; /** * Runtime configuration for the batch execution. * Structure is documented below. */ runtimeConfig?: pulumi.Input<inputs.dataproc.BatchRuntimeConfig>; /** * Spark batch config. * Structure is documented below. */ sparkBatch?: pulumi.Input<inputs.dataproc.BatchSparkBatch>; /** * SparkR batch config. * Structure is documented below. */ sparkRBatch?: pulumi.Input<inputs.dataproc.BatchSparkRBatch>; /** * Spark SQL batch config. * Structure is documented below. */ sparkSqlBatch?: pulumi.Input<inputs.dataproc.BatchSparkSqlBatch>; }