@pulumi/gcp
Version:
A Pulumi package for creating and managing Google Cloud Platform resources.
411 lines • 15.5 kB
JavaScript
;
// *** WARNING: this file was generated by pulumi-language-nodejs. ***
// *** Do not edit by hand unless you're certain you know what you are doing! ***
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.Batch = void 0;
const pulumi = __importStar(require("@pulumi/pulumi"));
const utilities = __importStar(require("../utilities"));
/**
* Dataproc Serverless Batches lets you run Spark workloads without requiring you to
* provision and manage your own Dataproc cluster.
*
* To get more information about Batch, see:
*
* * [API documentation](https://cloud.google.com/dataproc-serverless/docs/reference/rest/v1/projects.locations.batches)
* * How-to Guides
* * [Dataproc Serverless Batches Intro](https://cloud.google.com/dataproc-serverless/docs/overview)
*
* ## Example Usage
*
* ### Dataproc Batch Spark
*
* ```typescript
* import * as pulumi from "@pulumi/pulumi";
* import * as gcp from "@pulumi/gcp";
*
* const exampleBatchSpark = new gcp.dataproc.Batch("example_batch_spark", {
* batchId: "tf-test-batch_60461",
* location: "us-central1",
* labels: {
* batch_test: "terraform",
* },
* runtimeConfig: {
* properties: {
* "spark.dynamicAllocation.enabled": "false",
* "spark.executor.instances": "2",
* },
* },
* environmentConfig: {
* executionConfig: {
* subnetworkUri: "default",
* ttl: "3600s",
* networkTags: ["tag1"],
* },
* },
* sparkBatch: {
* mainClass: "org.apache.spark.examples.SparkPi",
* args: ["10"],
* jarFileUris: ["file:///usr/lib/spark/examples/jars/spark-examples.jar"],
* },
* });
* ```
* ### Dataproc Batch Spark Full
*
* ```typescript
* import * as pulumi from "@pulumi/pulumi";
* import * as gcp from "@pulumi/gcp";
*
* const project = gcp.organizations.getProject({});
* const gcsAccount = gcp.storage.getProjectServiceAccount({});
* const bucket = new gcp.storage.Bucket("bucket", {
* uniformBucketLevelAccess: true,
* name: "dataproc-bucket",
* location: "US",
* forceDestroy: true,
* });
* const cryptoKeyMember1 = new gcp.kms.CryptoKeyIAMMember("crypto_key_member_1", {
* cryptoKeyId: "example-key",
* role: "roles/cloudkms.cryptoKeyEncrypterDecrypter",
* member: project.then(project => `serviceAccount:service-${project.number}@dataproc-accounts.iam.gserviceaccount.com`),
* });
* const ms = new gcp.dataproc.MetastoreService("ms", {
* serviceId: "dataproc-batch",
* location: "us-central1",
* port: 9080,
* tier: "DEVELOPER",
* maintenanceWindow: {
* hourOfDay: 2,
* dayOfWeek: "SUNDAY",
* },
* hiveMetastoreConfig: {
* version: "3.1.2",
* },
* });
* const basic = new gcp.dataproc.Cluster("basic", {
* name: "dataproc-batch",
* region: "us-central1",
* clusterConfig: {
* softwareConfig: {
* overrideProperties: {
* "dataproc:dataproc.allow.zero.workers": "true",
* "spark:spark.history.fs.logDirectory": pulumi.interpolate`gs://${bucket.name}/*/spark-job-history`,
* },
* },
* endpointConfig: {
* enableHttpPortAccess: true,
* },
* masterConfig: {
* numInstances: 1,
* machineType: "e2-standard-2",
* diskConfig: {
* bootDiskSizeGb: 35,
* },
* },
* metastoreConfig: {
* dataprocMetastoreService: ms.name,
* },
* },
* });
* const exampleBatchSpark = new gcp.dataproc.Batch("example_batch_spark", {
* batchId: "dataproc-batch",
* location: "us-central1",
* labels: {
* batch_test: "terraform",
* },
* runtimeConfig: {
* properties: {
* "spark.dynamicAllocation.enabled": "false",
* "spark.executor.instances": "2",
* },
* version: "2.2",
* },
* environmentConfig: {
* executionConfig: {
* ttl: "3600s",
* networkTags: ["tag1"],
* kmsKey: "example-key",
* networkUri: "default",
* serviceAccount: project.then(project => `${project.number}-compute@developer.gserviceaccount.com`),
* stagingBucket: bucket.name,
* authenticationConfig: {
* userWorkloadAuthenticationType: "SERVICE_ACCOUNT",
* },
* },
* peripheralsConfig: {
* metastoreService: ms.name,
* sparkHistoryServerConfig: {
* dataprocCluster: basic.id,
* },
* },
* },
* sparkBatch: {
* mainClass: "org.apache.spark.examples.SparkPi",
* args: ["10"],
* jarFileUris: ["file:///usr/lib/spark/examples/jars/spark-examples.jar"],
* },
* }, {
* dependsOn: [cryptoKeyMember1],
* });
* ```
* ### Dataproc Batch Sparksql
*
* ```typescript
* import * as pulumi from "@pulumi/pulumi";
* import * as gcp from "@pulumi/gcp";
*
* const exampleBatchSparsql = new gcp.dataproc.Batch("example_batch_sparsql", {
* batchId: "tf-test-batch_45397",
* location: "us-central1",
* runtimeConfig: {
* properties: {
* "spark.dynamicAllocation.enabled": "false",
* "spark.executor.instances": "2",
* },
* },
* environmentConfig: {
* executionConfig: {
* subnetworkUri: "default",
* },
* },
* sparkSqlBatch: {
* queryFileUri: "gs://dataproc-examples/spark-sql/natality/cigarette_correlations.sql",
* jarFileUris: ["file:///usr/lib/spark/examples/jars/spark-examples.jar"],
* queryVariables: {
* name: "value",
* },
* },
* });
* ```
* ### Dataproc Batch Pyspark
*
* ```typescript
* import * as pulumi from "@pulumi/pulumi";
* import * as gcp from "@pulumi/gcp";
*
* const exampleBatchPyspark = new gcp.dataproc.Batch("example_batch_pyspark", {
* batchId: "tf-test-batch_16451",
* location: "us-central1",
* runtimeConfig: {
* properties: {
* "spark.dynamicAllocation.enabled": "false",
* "spark.executor.instances": "2",
* },
* },
* environmentConfig: {
* executionConfig: {
* subnetworkUri: "default",
* },
* },
* pysparkBatch: {
* mainPythonFileUri: "https://storage.googleapis.com/terraform-batches/test_util.py",
* args: ["10"],
* jarFileUris: ["file:///usr/lib/spark/examples/jars/spark-examples.jar"],
* pythonFileUris: ["gs://dataproc-examples/pyspark/hello-world/hello-world.py"],
* archiveUris: [
* "https://storage.googleapis.com/terraform-batches/animals.txt.tar.gz#unpacked",
* "https://storage.googleapis.com/terraform-batches/animals.txt.jar",
* "https://storage.googleapis.com/terraform-batches/animals.txt",
* ],
* fileUris: ["https://storage.googleapis.com/terraform-batches/people.txt"],
* },
* });
* ```
* ### Dataproc Batch Sparkr
*
* ```typescript
* import * as pulumi from "@pulumi/pulumi";
* import * as gcp from "@pulumi/gcp";
*
* const exampleBatchSparkr = new gcp.dataproc.Batch("example_batch_sparkr", {
* batchId: "tf-test-batch_3686",
* location: "us-central1",
* labels: {
* batch_test: "terraform",
* },
* runtimeConfig: {
* properties: {
* "spark.dynamicAllocation.enabled": "false",
* "spark.executor.instances": "2",
* },
* },
* environmentConfig: {
* executionConfig: {
* subnetworkUri: "default",
* ttl: "3600s",
* networkTags: ["tag1"],
* },
* },
* sparkRBatch: {
* mainRFileUri: "https://storage.googleapis.com/terraform-batches/spark-r-flights.r",
* args: ["https://storage.googleapis.com/terraform-batches/flights.csv"],
* },
* });
* ```
* ### Dataproc Batch Autotuning
*
* ```typescript
* import * as pulumi from "@pulumi/pulumi";
* import * as gcp from "@pulumi/gcp";
*
* const exampleBatchAutotuning = new gcp.dataproc.Batch("example_batch_autotuning", {
* batchId: "tf-test-batch_54136",
* location: "us-central1",
* labels: {
* batch_test: "terraform",
* },
* runtimeConfig: {
* version: "2.2",
* properties: {
* "spark.dynamicAllocation.enabled": "false",
* "spark.executor.instances": "2",
* },
* cohort: "tf-dataproc-batch-example",
* autotuningConfig: {
* scenarios: [
* "AUTO",
* "SCALING",
* "MEMORY",
* ],
* },
* },
* environmentConfig: {
* executionConfig: {
* subnetworkUri: "default",
* ttl: "3600s",
* },
* },
* sparkBatch: {
* mainClass: "org.apache.spark.examples.SparkPi",
* args: ["10"],
* jarFileUris: ["file:///usr/lib/spark/examples/jars/spark-examples.jar"],
* },
* });
* ```
*
* ## Import
*
* Batch can be imported using any of these accepted formats:
*
* * `projects/{{project}}/locations/{{location}}/batches/{{batch_id}}`
* * `{{project}}/{{location}}/{{batch_id}}`
* * `{{location}}/{{batch_id}}`
*
* When using the `pulumi import` command, Batch can be imported using one of the formats above. For example:
*
* ```sh
* $ pulumi import gcp:dataproc/batch:Batch default projects/{{project}}/locations/{{location}}/batches/{{batch_id}}
* $ pulumi import gcp:dataproc/batch:Batch default {{project}}/{{location}}/{{batch_id}}
* $ pulumi import gcp:dataproc/batch:Batch default {{location}}/{{batch_id}}
* ```
*/
class Batch extends pulumi.CustomResource {
/**
* Get an existing Batch resource's state with the given name, ID, and optional extra
* properties used to qualify the lookup.
*
* @param name The _unique_ name of the resulting resource.
* @param id The _unique_ provider ID of the resource to lookup.
* @param state Any extra arguments used during the lookup.
* @param opts Optional settings to control the behavior of the CustomResource.
*/
static get(name, id, state, opts) {
return new Batch(name, state, { ...opts, id: id });
}
/** @internal */
static __pulumiType = 'gcp:dataproc/batch:Batch';
/**
* Returns true if the given object is an instance of Batch. This is designed to work even
* when multiple copies of the Pulumi SDK have been loaded into the same process.
*/
static isInstance(obj) {
if (obj === undefined || obj === null) {
return false;
}
return obj['__pulumiType'] === Batch.__pulumiType;
}
constructor(name, argsOrState, opts) {
let resourceInputs = {};
opts = opts || {};
if (opts.id) {
const state = argsOrState;
resourceInputs["batchId"] = state?.batchId;
resourceInputs["createTime"] = state?.createTime;
resourceInputs["creator"] = state?.creator;
resourceInputs["deletionPolicy"] = state?.deletionPolicy;
resourceInputs["effectiveLabels"] = state?.effectiveLabels;
resourceInputs["environmentConfig"] = state?.environmentConfig;
resourceInputs["labels"] = state?.labels;
resourceInputs["location"] = state?.location;
resourceInputs["name"] = state?.name;
resourceInputs["operation"] = state?.operation;
resourceInputs["project"] = state?.project;
resourceInputs["pulumiLabels"] = state?.pulumiLabels;
resourceInputs["pysparkBatch"] = state?.pysparkBatch;
resourceInputs["runtimeConfig"] = state?.runtimeConfig;
resourceInputs["runtimeInfos"] = state?.runtimeInfos;
resourceInputs["sparkBatch"] = state?.sparkBatch;
resourceInputs["sparkRBatch"] = state?.sparkRBatch;
resourceInputs["sparkSqlBatch"] = state?.sparkSqlBatch;
resourceInputs["state"] = state?.state;
resourceInputs["stateHistories"] = state?.stateHistories;
resourceInputs["stateMessage"] = state?.stateMessage;
resourceInputs["stateTime"] = state?.stateTime;
resourceInputs["uuid"] = state?.uuid;
}
else {
const args = argsOrState;
resourceInputs["batchId"] = args?.batchId;
resourceInputs["deletionPolicy"] = args?.deletionPolicy;
resourceInputs["environmentConfig"] = args?.environmentConfig;
resourceInputs["labels"] = args?.labels;
resourceInputs["location"] = args?.location;
resourceInputs["project"] = args?.project;
resourceInputs["pysparkBatch"] = args?.pysparkBatch;
resourceInputs["runtimeConfig"] = args?.runtimeConfig;
resourceInputs["sparkBatch"] = args?.sparkBatch;
resourceInputs["sparkRBatch"] = args?.sparkRBatch;
resourceInputs["sparkSqlBatch"] = args?.sparkSqlBatch;
resourceInputs["createTime"] = undefined /*out*/;
resourceInputs["creator"] = undefined /*out*/;
resourceInputs["effectiveLabels"] = undefined /*out*/;
resourceInputs["name"] = undefined /*out*/;
resourceInputs["operation"] = undefined /*out*/;
resourceInputs["pulumiLabels"] = undefined /*out*/;
resourceInputs["runtimeInfos"] = undefined /*out*/;
resourceInputs["state"] = undefined /*out*/;
resourceInputs["stateHistories"] = undefined /*out*/;
resourceInputs["stateMessage"] = undefined /*out*/;
resourceInputs["stateTime"] = undefined /*out*/;
resourceInputs["uuid"] = undefined /*out*/;
}
opts = pulumi.mergeOptions(utilities.resourceOptsDefaults(), opts);
const secretOpts = { additionalSecretOutputs: ["effectiveLabels", "pulumiLabels"] };
opts = pulumi.mergeOptions(opts, secretOpts);
super(Batch.__pulumiType, name, resourceInputs, opts);
}
}
exports.Batch = Batch;
//# sourceMappingURL=batch.js.map