@pulumi/gcp
Version:
A Pulumi package for creating and managing Google Cloud Platform resources.
506 lines (505 loc) • 21.7 kB
TypeScript
import * as pulumi from "@pulumi/pulumi";
/**
* Creates a [Flex Template](https://cloud.google.com/dataflow/docs/guides/templates/using-flex-templates)
* job on Dataflow, which is an implementation of Apache Beam running on Google
* Compute Engine. For more information see the official documentation for [Beam](https://beam.apache.org)
* and [Dataflow](https://cloud.google.com/dataflow/).
*
* ## Example Usage
*
* ```typescript
* import * as pulumi from "@pulumi/pulumi";
* import * as gcp from "@pulumi/gcp";
*
* const bigDataJob = new gcp.dataflow.FlexTemplateJob("big_data_job", {
* name: "dataflow-flextemplates-job",
* containerSpecGcsPath: "gs://my-bucket/templates/template.json",
* parameters: {
* inputSubscription: "messages",
* },
* });
* ```
*
* ## Note on "destroy" / "apply"
*
* There are many types of Dataflow jobs. Some Dataflow jobs run constantly,
* getting new data from (e.g.) a GCS bucket, and outputting data continuously.
* Some jobs process a set amount of data then terminate. All jobs can fail while
* running due to programming errors or other issues. In this way, Dataflow jobs
* are different from most other provider / Google resources.
*
* The Dataflow resource is considered 'existing' while it is in a nonterminal
* state. If it reaches a terminal state (e.g. 'FAILED', 'COMPLETE',
* 'CANCELLED'), it will be recreated on the next 'apply'. This is as expected for
* jobs which run continuously, but may surprise users who use this resource for
* other kinds of Dataflow jobs.
*
* A Dataflow job which is 'destroyed' may be "cancelled" or "drained". If
* "cancelled", the job terminates - any data written remains where it is, but no
* new data will be processed. If "drained", no new data will enter the pipeline,
* but any data currently in the pipeline will finish being processed. The default
* is "cancelled", but if a user sets `onDelete` to `"drain"` in the
* configuration, you may experience a long wait for your `pulumi destroy` to
* complete.
*
* You can potentially short-circuit the wait by setting `skipWaitOnJobTermination`
* to `true`, but beware that unless you take active steps to ensure that the job
* `name` parameter changes between instances, the name will conflict and the launch
* of the new job will fail. One way to do this is with a
* randomId
* resource, for example:
*
* ```typescript
* import * as pulumi from "@pulumi/pulumi";
* import * as gcp from "@pulumi/gcp";
* import * as random from "@pulumi/random";
*
* const config = new pulumi.Config();
* const bigDataJobSubscriptionId = config.get("bigDataJobSubscriptionId") || "projects/myproject/subscriptions/messages";
* const bigDataJobNameSuffix = new random.RandomId("big_data_job_name_suffix", {
* byteLength: 4,
* keepers: {
* region: region,
* subscription_id: bigDataJobSubscriptionId,
* },
* });
* const bigDataJob = new gcp.dataflow.FlexTemplateJob("big_data_job", {
* name: pulumi.interpolate`dataflow-flextemplates-job-${bigDataJobNameSuffix.dec}`,
* region: region,
* containerSpecGcsPath: "gs://my-bucket/templates/template.json",
* skipWaitOnJobTermination: true,
* parameters: {
* inputSubscription: bigDataJobSubscriptionId,
* },
* });
* ```
*
* ## Import
*
* This resource does not support import.
*/
export declare class FlexTemplateJob extends pulumi.CustomResource {
/**
* Get an existing FlexTemplateJob resource's state with the given name, ID, and optional extra
* properties used to qualify the lookup.
*
* @param name The _unique_ name of the resulting resource.
* @param id The _unique_ provider ID of the resource to lookup.
* @param state Any extra arguments used during the lookup.
* @param opts Optional settings to control the behavior of the CustomResource.
*/
static get(name: string, id: pulumi.Input<pulumi.ID>, state?: FlexTemplateJobState, opts?: pulumi.CustomResourceOptions): FlexTemplateJob;
/**
* Returns true if the given object is an instance of FlexTemplateJob. This is designed to work even
* when multiple copies of the Pulumi SDK have been loaded into the same process.
*/
static isInstance(obj: any): obj is FlexTemplateJob;
/**
* List of experiments that should be used by the job. An example value is `["enableStackdriverAgentMetrics"]`.
*/
readonly additionalExperiments: pulumi.Output<string[]>;
/**
* List of pipeline options that should be used by the job. An example value is `["numberOfWorkerHarnessThreads=20"]`.
*/
readonly additionalPipelineOptions: pulumi.Output<string[] | undefined>;
/**
* The algorithm to use for autoscaling.
*/
readonly autoscalingAlgorithm: pulumi.Output<string>;
/**
* The GCS path to the Dataflow job Flex
* Template.
*
* - - -
*/
readonly containerSpecGcsPath: pulumi.Output<string>;
readonly effectiveLabels: pulumi.Output<{
[key: string]: string;
}>;
/**
* Immutable. Indicates if the job should use the streaming engine feature.
*/
readonly enableStreamingEngine: pulumi.Output<boolean | undefined>;
/**
* The configuration for VM IPs. Options are `"WORKER_IP_PUBLIC"` or `"WORKER_IP_PRIVATE"`.
*/
readonly ipConfiguration: pulumi.Output<string | undefined>;
/**
* The unique ID of this job.
*/
readonly jobId: pulumi.Output<string>;
/**
* The name for the Cloud KMS key for the job. Key format is: `projects/PROJECT_ID/locations/LOCATION/keyRings/KEY_RING/cryptoKeys/KEY`
*/
readonly kmsKeyName: pulumi.Output<string>;
/**
* User labels to be specified for the job. Keys and values
* should follow the restrictions specified in the [labeling restrictions](https://cloud.google.com/compute/docs/labeling-resources#restrictions)
* page. **Note**: This field is marked as deprecated as the API does not currently
* support adding labels.
* **NOTE**: Google-provided Dataflow templates often provide default labels
* that begin with `goog-dataflow-provided`. Unless explicitly set in config, these
* labels will be ignored to prevent diffs on re-apply.
*/
readonly labels: pulumi.Output<{
[key: string]: string;
} | undefined>;
/**
* The machine type to use for launching the job. The default is n1-standard-1.
*/
readonly launcherMachineType: pulumi.Output<string>;
/**
* The machine type to use for the job.
*/
readonly machineType: pulumi.Output<string>;
/**
* Immutable. The maximum number of Google Compute Engine instances to be made available to your pipeline during execution, from 1 to 1000.
*/
readonly maxWorkers: pulumi.Output<number>;
/**
* Immutable. A unique name for the resource, required by Dataflow.
*/
readonly name: pulumi.Output<string>;
/**
* The network to which VMs will be assigned. If it is not provided, "default" will be used.
*/
readonly network: pulumi.Output<string>;
/**
* Immutable. The initial number of Google Compute Engine instances for the job.
*/
readonly numWorkers: pulumi.Output<number>;
/**
* One of "drain" or "cancel". Specifies behavior of
* deletion during `pulumi destroy`. See above note.
*/
readonly onDelete: pulumi.Output<string | undefined>;
/**
* **Template specific** Key/Value pairs to be forwarded to the pipeline's options; keys are
* case-sensitive based on the language on which the pipeline is coded, mostly Java.
* **Note**: do not configure Dataflow options here in parameters.
*/
readonly parameters: pulumi.Output<{
[key: string]: string;
} | undefined>;
/**
* The project in which the resource belongs. If it is not
* provided, the provider project is used.
*/
readonly project: pulumi.Output<string>;
/**
* The combination of labels configured directly on the resource and default labels configured on the provider.
*/
readonly pulumiLabels: pulumi.Output<{
[key: string]: string;
}>;
/**
* Immutable. The region in which the created job should run.
*/
readonly region: pulumi.Output<string>;
/**
* Docker registry location of container image to use for the 'worker harness. Default is the container for the version of the SDK. Note this field is only valid for portable pipelines.
*/
readonly sdkContainerImage: pulumi.Output<string>;
/**
* Service account email to run the workers as. This should be just an email e.g. `myserviceaccount@myproject.iam.gserviceaccount.com`. Do not include any `serviceAccount:` or other prefix.
*/
readonly serviceAccountEmail: pulumi.Output<string>;
readonly skipWaitOnJobTermination: pulumi.Output<boolean | undefined>;
/**
* The Cloud Storage path to use for staging files. Must be a valid Cloud Storage URL, beginning with gs://.
*/
readonly stagingLocation: pulumi.Output<string>;
/**
* The current state of the resource, selected from the [JobState enum](https://cloud.google.com/dataflow/docs/reference/rest/v1b3/projects.jobs#Job.JobState)
*/
readonly state: pulumi.Output<string>;
/**
* The subnetwork to which VMs will be assigned. Should be of the form "regions/REGION/subnetworks/SUBNETWORK".
*/
readonly subnetwork: pulumi.Output<string>;
/**
* The Cloud Storage path to use for temporary files. Must be a valid Cloud Storage URL, beginning with gs://.
*/
readonly tempLocation: pulumi.Output<string>;
/**
* Only applicable when updating a pipeline. Map of transform name prefixes of the job to be replaced with the corresponding name prefixes of the new job.Only applicable when updating a pipeline. Map of transform name prefixes of the job to be replaced with the corresponding name prefixes of the new job.
*/
readonly transformNameMapping: pulumi.Output<{
[key: string]: string;
} | undefined>;
/**
* The type of this job, selected from the JobType enum.
*/
readonly type: pulumi.Output<string>;
/**
* Create a FlexTemplateJob resource with the given unique name, arguments, and options.
*
* @param name The _unique_ name of the resource.
* @param args The arguments to use to populate this resource's properties.
* @param opts A bag of options that control this resource's behavior.
*/
constructor(name: string, args: FlexTemplateJobArgs, opts?: pulumi.CustomResourceOptions);
}
/**
* Input properties used for looking up and filtering FlexTemplateJob resources.
*/
export interface FlexTemplateJobState {
/**
* List of experiments that should be used by the job. An example value is `["enableStackdriverAgentMetrics"]`.
*/
additionalExperiments?: pulumi.Input<pulumi.Input<string>[]>;
/**
* List of pipeline options that should be used by the job. An example value is `["numberOfWorkerHarnessThreads=20"]`.
*/
additionalPipelineOptions?: pulumi.Input<pulumi.Input<string>[]>;
/**
* The algorithm to use for autoscaling.
*/
autoscalingAlgorithm?: pulumi.Input<string>;
/**
* The GCS path to the Dataflow job Flex
* Template.
*
* - - -
*/
containerSpecGcsPath?: pulumi.Input<string>;
effectiveLabels?: pulumi.Input<{
[key: string]: pulumi.Input<string>;
}>;
/**
* Immutable. Indicates if the job should use the streaming engine feature.
*/
enableStreamingEngine?: pulumi.Input<boolean>;
/**
* The configuration for VM IPs. Options are `"WORKER_IP_PUBLIC"` or `"WORKER_IP_PRIVATE"`.
*/
ipConfiguration?: pulumi.Input<string>;
/**
* The unique ID of this job.
*/
jobId?: pulumi.Input<string>;
/**
* The name for the Cloud KMS key for the job. Key format is: `projects/PROJECT_ID/locations/LOCATION/keyRings/KEY_RING/cryptoKeys/KEY`
*/
kmsKeyName?: pulumi.Input<string>;
/**
* User labels to be specified for the job. Keys and values
* should follow the restrictions specified in the [labeling restrictions](https://cloud.google.com/compute/docs/labeling-resources#restrictions)
* page. **Note**: This field is marked as deprecated as the API does not currently
* support adding labels.
* **NOTE**: Google-provided Dataflow templates often provide default labels
* that begin with `goog-dataflow-provided`. Unless explicitly set in config, these
* labels will be ignored to prevent diffs on re-apply.
*/
labels?: pulumi.Input<{
[key: string]: pulumi.Input<string>;
}>;
/**
* The machine type to use for launching the job. The default is n1-standard-1.
*/
launcherMachineType?: pulumi.Input<string>;
/**
* The machine type to use for the job.
*/
machineType?: pulumi.Input<string>;
/**
* Immutable. The maximum number of Google Compute Engine instances to be made available to your pipeline during execution, from 1 to 1000.
*/
maxWorkers?: pulumi.Input<number>;
/**
* Immutable. A unique name for the resource, required by Dataflow.
*/
name?: pulumi.Input<string>;
/**
* The network to which VMs will be assigned. If it is not provided, "default" will be used.
*/
network?: pulumi.Input<string>;
/**
* Immutable. The initial number of Google Compute Engine instances for the job.
*/
numWorkers?: pulumi.Input<number>;
/**
* One of "drain" or "cancel". Specifies behavior of
* deletion during `pulumi destroy`. See above note.
*/
onDelete?: pulumi.Input<string>;
/**
* **Template specific** Key/Value pairs to be forwarded to the pipeline's options; keys are
* case-sensitive based on the language on which the pipeline is coded, mostly Java.
* **Note**: do not configure Dataflow options here in parameters.
*/
parameters?: pulumi.Input<{
[key: string]: pulumi.Input<string>;
}>;
/**
* The project in which the resource belongs. If it is not
* provided, the provider project is used.
*/
project?: pulumi.Input<string>;
/**
* The combination of labels configured directly on the resource and default labels configured on the provider.
*/
pulumiLabels?: pulumi.Input<{
[key: string]: pulumi.Input<string>;
}>;
/**
* Immutable. The region in which the created job should run.
*/
region?: pulumi.Input<string>;
/**
* Docker registry location of container image to use for the 'worker harness. Default is the container for the version of the SDK. Note this field is only valid for portable pipelines.
*/
sdkContainerImage?: pulumi.Input<string>;
/**
* Service account email to run the workers as. This should be just an email e.g. `myserviceaccount@myproject.iam.gserviceaccount.com`. Do not include any `serviceAccount:` or other prefix.
*/
serviceAccountEmail?: pulumi.Input<string>;
skipWaitOnJobTermination?: pulumi.Input<boolean>;
/**
* The Cloud Storage path to use for staging files. Must be a valid Cloud Storage URL, beginning with gs://.
*/
stagingLocation?: pulumi.Input<string>;
/**
* The current state of the resource, selected from the [JobState enum](https://cloud.google.com/dataflow/docs/reference/rest/v1b3/projects.jobs#Job.JobState)
*/
state?: pulumi.Input<string>;
/**
* The subnetwork to which VMs will be assigned. Should be of the form "regions/REGION/subnetworks/SUBNETWORK".
*/
subnetwork?: pulumi.Input<string>;
/**
* The Cloud Storage path to use for temporary files. Must be a valid Cloud Storage URL, beginning with gs://.
*/
tempLocation?: pulumi.Input<string>;
/**
* Only applicable when updating a pipeline. Map of transform name prefixes of the job to be replaced with the corresponding name prefixes of the new job.Only applicable when updating a pipeline. Map of transform name prefixes of the job to be replaced with the corresponding name prefixes of the new job.
*/
transformNameMapping?: pulumi.Input<{
[key: string]: pulumi.Input<string>;
}>;
/**
* The type of this job, selected from the JobType enum.
*/
type?: pulumi.Input<string>;
}
/**
* The set of arguments for constructing a FlexTemplateJob resource.
*/
export interface FlexTemplateJobArgs {
/**
* List of experiments that should be used by the job. An example value is `["enableStackdriverAgentMetrics"]`.
*/
additionalExperiments?: pulumi.Input<pulumi.Input<string>[]>;
/**
* List of pipeline options that should be used by the job. An example value is `["numberOfWorkerHarnessThreads=20"]`.
*/
additionalPipelineOptions?: pulumi.Input<pulumi.Input<string>[]>;
/**
* The algorithm to use for autoscaling.
*/
autoscalingAlgorithm?: pulumi.Input<string>;
/**
* The GCS path to the Dataflow job Flex
* Template.
*
* - - -
*/
containerSpecGcsPath: pulumi.Input<string>;
/**
* Immutable. Indicates if the job should use the streaming engine feature.
*/
enableStreamingEngine?: pulumi.Input<boolean>;
/**
* The configuration for VM IPs. Options are `"WORKER_IP_PUBLIC"` or `"WORKER_IP_PRIVATE"`.
*/
ipConfiguration?: pulumi.Input<string>;
/**
* The name for the Cloud KMS key for the job. Key format is: `projects/PROJECT_ID/locations/LOCATION/keyRings/KEY_RING/cryptoKeys/KEY`
*/
kmsKeyName?: pulumi.Input<string>;
/**
* User labels to be specified for the job. Keys and values
* should follow the restrictions specified in the [labeling restrictions](https://cloud.google.com/compute/docs/labeling-resources#restrictions)
* page. **Note**: This field is marked as deprecated as the API does not currently
* support adding labels.
* **NOTE**: Google-provided Dataflow templates often provide default labels
* that begin with `goog-dataflow-provided`. Unless explicitly set in config, these
* labels will be ignored to prevent diffs on re-apply.
*/
labels?: pulumi.Input<{
[key: string]: pulumi.Input<string>;
}>;
/**
* The machine type to use for launching the job. The default is n1-standard-1.
*/
launcherMachineType?: pulumi.Input<string>;
/**
* The machine type to use for the job.
*/
machineType?: pulumi.Input<string>;
/**
* Immutable. The maximum number of Google Compute Engine instances to be made available to your pipeline during execution, from 1 to 1000.
*/
maxWorkers?: pulumi.Input<number>;
/**
* Immutable. A unique name for the resource, required by Dataflow.
*/
name?: pulumi.Input<string>;
/**
* The network to which VMs will be assigned. If it is not provided, "default" will be used.
*/
network?: pulumi.Input<string>;
/**
* Immutable. The initial number of Google Compute Engine instances for the job.
*/
numWorkers?: pulumi.Input<number>;
/**
* One of "drain" or "cancel". Specifies behavior of
* deletion during `pulumi destroy`. See above note.
*/
onDelete?: pulumi.Input<string>;
/**
* **Template specific** Key/Value pairs to be forwarded to the pipeline's options; keys are
* case-sensitive based on the language on which the pipeline is coded, mostly Java.
* **Note**: do not configure Dataflow options here in parameters.
*/
parameters?: pulumi.Input<{
[key: string]: pulumi.Input<string>;
}>;
/**
* The project in which the resource belongs. If it is not
* provided, the provider project is used.
*/
project?: pulumi.Input<string>;
/**
* Immutable. The region in which the created job should run.
*/
region?: pulumi.Input<string>;
/**
* Docker registry location of container image to use for the 'worker harness. Default is the container for the version of the SDK. Note this field is only valid for portable pipelines.
*/
sdkContainerImage?: pulumi.Input<string>;
/**
* Service account email to run the workers as. This should be just an email e.g. `myserviceaccount@myproject.iam.gserviceaccount.com`. Do not include any `serviceAccount:` or other prefix.
*/
serviceAccountEmail?: pulumi.Input<string>;
skipWaitOnJobTermination?: pulumi.Input<boolean>;
/**
* The Cloud Storage path to use for staging files. Must be a valid Cloud Storage URL, beginning with gs://.
*/
stagingLocation?: pulumi.Input<string>;
/**
* The subnetwork to which VMs will be assigned. Should be of the form "regions/REGION/subnetworks/SUBNETWORK".
*/
subnetwork?: pulumi.Input<string>;
/**
* The Cloud Storage path to use for temporary files. Must be a valid Cloud Storage URL, beginning with gs://.
*/
tempLocation?: pulumi.Input<string>;
/**
* Only applicable when updating a pipeline. Map of transform name prefixes of the job to be replaced with the corresponding name prefixes of the new job.Only applicable when updating a pipeline. Map of transform name prefixes of the job to be replaced with the corresponding name prefixes of the new job.
*/
transformNameMapping?: pulumi.Input<{
[key: string]: pulumi.Input<string>;
}>;
}