@pulumi/gcp
Version:
A Pulumi package for creating and managing Google Cloud Platform resources.
390 lines • 14.9 kB
JavaScript
"use strict";
// *** WARNING: this file was generated by the Pulumi Terraform Bridge (tfgen) Tool. ***
// *** Do not edit by hand unless you're certain you know what you are doing! ***
Object.defineProperty(exports, "__esModule", { value: true });
exports.Batch = void 0;
const pulumi = require("@pulumi/pulumi");
const utilities = require("../utilities");
/**
* Dataproc Serverless Batches lets you run Spark workloads without requiring you to
* provision and manage your own Dataproc cluster.
*
* To get more information about Batch, see:
*
* * [API documentation](https://cloud.google.com/dataproc-serverless/docs/reference/rest/v1/projects.locations.batches)
* * How-to Guides
* * [Dataproc Serverless Batches Intro](https://cloud.google.com/dataproc-serverless/docs/overview)
*
* ## Example Usage
*
* ### Dataproc Batch Spark
*
* ```typescript
* import * as pulumi from "@pulumi/pulumi";
* import * as gcp from "@pulumi/gcp";
*
* const exampleBatchSpark = new gcp.dataproc.Batch("example_batch_spark", {
* batchId: "tf-test-batch_88722",
* location: "us-central1",
* labels: {
* batch_test: "terraform",
* },
* runtimeConfig: {
* properties: {
* "spark.dynamicAllocation.enabled": "false",
* "spark.executor.instances": "2",
* },
* },
* environmentConfig: {
* executionConfig: {
* subnetworkUri: "default",
* ttl: "3600s",
* networkTags: ["tag1"],
* },
* },
* sparkBatch: {
* mainClass: "org.apache.spark.examples.SparkPi",
* args: ["10"],
* jarFileUris: ["file:///usr/lib/spark/examples/jars/spark-examples.jar"],
* },
* });
* ```
* ### Dataproc Batch Spark Full
*
* ```typescript
* import * as pulumi from "@pulumi/pulumi";
* import * as gcp from "@pulumi/gcp";
*
* const project = gcp.organizations.getProject({});
* const gcsAccount = gcp.storage.getProjectServiceAccount({});
* const bucket = new gcp.storage.Bucket("bucket", {
* uniformBucketLevelAccess: true,
* name: "dataproc-bucket",
* location: "US",
* forceDestroy: true,
* });
* const cryptoKeyMember1 = new gcp.kms.CryptoKeyIAMMember("crypto_key_member_1", {
* cryptoKeyId: "example-key",
* role: "roles/cloudkms.cryptoKeyEncrypterDecrypter",
* member: project.then(project => `serviceAccount:service-${project.number}@dataproc-accounts.iam.gserviceaccount.com`),
* });
* const ms = new gcp.dataproc.MetastoreService("ms", {
* serviceId: "dataproc-batch",
* location: "us-central1",
* port: 9080,
* tier: "DEVELOPER",
* maintenanceWindow: {
* hourOfDay: 2,
* dayOfWeek: "SUNDAY",
* },
* hiveMetastoreConfig: {
* version: "3.1.2",
* },
* });
* const basic = new gcp.dataproc.Cluster("basic", {
* name: "dataproc-batch",
* region: "us-central1",
* clusterConfig: {
* softwareConfig: {
* overrideProperties: {
* "dataproc:dataproc.allow.zero.workers": "true",
* "spark:spark.history.fs.logDirectory": pulumi.interpolate`gs://${bucket.name}/*/spark-job-history`,
* },
* },
* endpointConfig: {
* enableHttpPortAccess: true,
* },
* masterConfig: {
* numInstances: 1,
* machineType: "e2-standard-2",
* diskConfig: {
* bootDiskSizeGb: 35,
* },
* },
* metastoreConfig: {
* dataprocMetastoreService: ms.name,
* },
* },
* });
* const exampleBatchSpark = new gcp.dataproc.Batch("example_batch_spark", {
* batchId: "dataproc-batch",
* location: "us-central1",
* labels: {
* batch_test: "terraform",
* },
* runtimeConfig: {
* properties: {
* "spark.dynamicAllocation.enabled": "false",
* "spark.executor.instances": "2",
* },
* version: "2.2",
* },
* environmentConfig: {
* executionConfig: {
* ttl: "3600s",
* networkTags: ["tag1"],
* kmsKey: "example-key",
* networkUri: "default",
* serviceAccount: project.then(project => `${project.number}-compute@developer.gserviceaccount.com`),
* stagingBucket: bucket.name,
* },
* peripheralsConfig: {
* metastoreService: ms.name,
* sparkHistoryServerConfig: {
* dataprocCluster: basic.id,
* },
* },
* },
* sparkBatch: {
* mainClass: "org.apache.spark.examples.SparkPi",
* args: ["10"],
* jarFileUris: ["file:///usr/lib/spark/examples/jars/spark-examples.jar"],
* },
* }, {
* dependsOn: [cryptoKeyMember1],
* });
* ```
* ### Dataproc Batch Sparksql
*
* ```typescript
* import * as pulumi from "@pulumi/pulumi";
* import * as gcp from "@pulumi/gcp";
*
* const exampleBatchSparsql = new gcp.dataproc.Batch("example_batch_sparsql", {
* batchId: "tf-test-batch_39249",
* location: "us-central1",
* runtimeConfig: {
* properties: {
* "spark.dynamicAllocation.enabled": "false",
* "spark.executor.instances": "2",
* },
* },
* environmentConfig: {
* executionConfig: {
* subnetworkUri: "default",
* },
* },
* sparkSqlBatch: {
* queryFileUri: "gs://dataproc-examples/spark-sql/natality/cigarette_correlations.sql",
* jarFileUris: ["file:///usr/lib/spark/examples/jars/spark-examples.jar"],
* queryVariables: {
* name: "value",
* },
* },
* });
* ```
* ### Dataproc Batch Pyspark
*
* ```typescript
* import * as pulumi from "@pulumi/pulumi";
* import * as gcp from "@pulumi/gcp";
*
* const exampleBatchPyspark = new gcp.dataproc.Batch("example_batch_pyspark", {
* batchId: "tf-test-batch_74391",
* location: "us-central1",
* runtimeConfig: {
* properties: {
* "spark.dynamicAllocation.enabled": "false",
* "spark.executor.instances": "2",
* },
* },
* environmentConfig: {
* executionConfig: {
* subnetworkUri: "default",
* },
* },
* pysparkBatch: {
* mainPythonFileUri: "https://storage.googleapis.com/terraform-batches/test_util.py",
* args: ["10"],
* jarFileUris: ["file:///usr/lib/spark/examples/jars/spark-examples.jar"],
* pythonFileUris: ["gs://dataproc-examples/pyspark/hello-world/hello-world.py"],
* archiveUris: [
* "https://storage.googleapis.com/terraform-batches/animals.txt.tar.gz#unpacked",
* "https://storage.googleapis.com/terraform-batches/animals.txt.jar",
* "https://storage.googleapis.com/terraform-batches/animals.txt",
* ],
* fileUris: ["https://storage.googleapis.com/terraform-batches/people.txt"],
* },
* });
* ```
* ### Dataproc Batch Sparkr
*
* ```typescript
* import * as pulumi from "@pulumi/pulumi";
* import * as gcp from "@pulumi/gcp";
*
* const exampleBatchSparkr = new gcp.dataproc.Batch("example_batch_sparkr", {
* batchId: "tf-test-batch_16511",
* location: "us-central1",
* labels: {
* batch_test: "terraform",
* },
* runtimeConfig: {
* properties: {
* "spark.dynamicAllocation.enabled": "false",
* "spark.executor.instances": "2",
* },
* },
* environmentConfig: {
* executionConfig: {
* subnetworkUri: "default",
* ttl: "3600s",
* networkTags: ["tag1"],
* },
* },
* sparkRBatch: {
* mainRFileUri: "https://storage.googleapis.com/terraform-batches/spark-r-flights.r",
* args: ["https://storage.googleapis.com/terraform-batches/flights.csv"],
* },
* });
* ```
* ### Dataproc Batch Autotuning
*
* ```typescript
* import * as pulumi from "@pulumi/pulumi";
* import * as gcp from "@pulumi/gcp";
*
* const exampleBatchAutotuning = new gcp.dataproc.Batch("example_batch_autotuning", {
* batchId: "tf-test-batch_8493",
* location: "us-central1",
* labels: {
* batch_test: "terraform",
* },
* runtimeConfig: {
* version: "2.2",
* properties: {
* "spark.dynamicAllocation.enabled": "false",
* "spark.executor.instances": "2",
* },
* cohort: "tf-dataproc-batch-example",
* autotuningConfig: {
* scenarios: [
* "SCALING",
* "MEMORY",
* ],
* },
* },
* environmentConfig: {
* executionConfig: {
* subnetworkUri: "default",
* ttl: "3600s",
* },
* },
* sparkBatch: {
* mainClass: "org.apache.spark.examples.SparkPi",
* args: ["10"],
* jarFileUris: ["file:///usr/lib/spark/examples/jars/spark-examples.jar"],
* },
* });
* ```
*
* ## Import
*
* Batch can be imported using any of these accepted formats:
*
* * `projects/{{project}}/locations/{{location}}/batches/{{batch_id}}`
*
* * `{{project}}/{{location}}/{{batch_id}}`
*
* * `{{location}}/{{batch_id}}`
*
* When using the `pulumi import` command, Batch can be imported using one of the formats above. For example:
*
* ```sh
* $ pulumi import gcp:dataproc/batch:Batch default projects/{{project}}/locations/{{location}}/batches/{{batch_id}}
* ```
*
* ```sh
* $ pulumi import gcp:dataproc/batch:Batch default {{project}}/{{location}}/{{batch_id}}
* ```
*
* ```sh
* $ pulumi import gcp:dataproc/batch:Batch default {{location}}/{{batch_id}}
* ```
*/
class Batch extends pulumi.CustomResource {
/**
* Get an existing Batch resource's state with the given name, ID, and optional extra
* properties used to qualify the lookup.
*
* @param name The _unique_ name of the resulting resource.
* @param id The _unique_ provider ID of the resource to lookup.
* @param state Any extra arguments used during the lookup.
* @param opts Optional settings to control the behavior of the CustomResource.
*/
static get(name, id, state, opts) {
return new Batch(name, state, Object.assign(Object.assign({}, opts), { id: id }));
}
/**
* Returns true if the given object is an instance of Batch. This is designed to work even
* when multiple copies of the Pulumi SDK have been loaded into the same process.
*/
static isInstance(obj) {
if (obj === undefined || obj === null) {
return false;
}
return obj['__pulumiType'] === Batch.__pulumiType;
}
constructor(name, argsOrState, opts) {
let resourceInputs = {};
opts = opts || {};
if (opts.id) {
const state = argsOrState;
resourceInputs["batchId"] = state ? state.batchId : undefined;
resourceInputs["createTime"] = state ? state.createTime : undefined;
resourceInputs["creator"] = state ? state.creator : undefined;
resourceInputs["effectiveLabels"] = state ? state.effectiveLabels : undefined;
resourceInputs["environmentConfig"] = state ? state.environmentConfig : undefined;
resourceInputs["labels"] = state ? state.labels : undefined;
resourceInputs["location"] = state ? state.location : undefined;
resourceInputs["name"] = state ? state.name : undefined;
resourceInputs["operation"] = state ? state.operation : undefined;
resourceInputs["project"] = state ? state.project : undefined;
resourceInputs["pulumiLabels"] = state ? state.pulumiLabels : undefined;
resourceInputs["pysparkBatch"] = state ? state.pysparkBatch : undefined;
resourceInputs["runtimeConfig"] = state ? state.runtimeConfig : undefined;
resourceInputs["runtimeInfos"] = state ? state.runtimeInfos : undefined;
resourceInputs["sparkBatch"] = state ? state.sparkBatch : undefined;
resourceInputs["sparkRBatch"] = state ? state.sparkRBatch : undefined;
resourceInputs["sparkSqlBatch"] = state ? state.sparkSqlBatch : undefined;
resourceInputs["state"] = state ? state.state : undefined;
resourceInputs["stateHistories"] = state ? state.stateHistories : undefined;
resourceInputs["stateMessage"] = state ? state.stateMessage : undefined;
resourceInputs["stateTime"] = state ? state.stateTime : undefined;
resourceInputs["uuid"] = state ? state.uuid : undefined;
}
else {
const args = argsOrState;
resourceInputs["batchId"] = args ? args.batchId : undefined;
resourceInputs["environmentConfig"] = args ? args.environmentConfig : undefined;
resourceInputs["labels"] = args ? args.labels : undefined;
resourceInputs["location"] = args ? args.location : undefined;
resourceInputs["project"] = args ? args.project : undefined;
resourceInputs["pysparkBatch"] = args ? args.pysparkBatch : undefined;
resourceInputs["runtimeConfig"] = args ? args.runtimeConfig : undefined;
resourceInputs["sparkBatch"] = args ? args.sparkBatch : undefined;
resourceInputs["sparkRBatch"] = args ? args.sparkRBatch : undefined;
resourceInputs["sparkSqlBatch"] = args ? args.sparkSqlBatch : undefined;
resourceInputs["createTime"] = undefined /*out*/;
resourceInputs["creator"] = undefined /*out*/;
resourceInputs["effectiveLabels"] = undefined /*out*/;
resourceInputs["name"] = undefined /*out*/;
resourceInputs["operation"] = undefined /*out*/;
resourceInputs["pulumiLabels"] = undefined /*out*/;
resourceInputs["runtimeInfos"] = undefined /*out*/;
resourceInputs["state"] = undefined /*out*/;
resourceInputs["stateHistories"] = undefined /*out*/;
resourceInputs["stateMessage"] = undefined /*out*/;
resourceInputs["stateTime"] = undefined /*out*/;
resourceInputs["uuid"] = undefined /*out*/;
}
opts = pulumi.mergeOptions(utilities.resourceOptsDefaults(), opts);
const secretOpts = { additionalSecretOutputs: ["effectiveLabels", "pulumiLabels"] };
opts = pulumi.mergeOptions(opts, secretOpts);
super(Batch.__pulumiType, name, resourceInputs, opts);
}
}
exports.Batch = Batch;
/** @internal */
Batch.__pulumiType = 'gcp:dataproc/batch:Batch';
//# sourceMappingURL=batch.js.map