@pulumi/databricks
Version:
A Pulumi package for creating and managing databricks cloud resources.
474 lines (473 loc) • 22.9 kB
TypeScript
import * as pulumi from "@pulumi/pulumi";
import * as inputs from "./types/input";
import * as outputs from "./types/output";
/**
* Use `databricks.Pipeline` to deploy [Lakeflow Declarative Pipelines](https://docs.databricks.com/aws/en/dlt).
*
* > This resource can only be used with a workspace-level provider!
*
* ## Example Usage
*
* ```typescript
* import * as pulumi from "@pulumi/pulumi";
* import * as databricks from "@pulumi/databricks";
*
* const ldpDemo = new databricks.Notebook("ldp_demo", {});
* const ldpDemoRepo = new databricks.Repo("ldp_demo", {});
* const _this = new databricks.Pipeline("this", {
* name: "Pipeline Name",
* catalog: "main",
* schema: "ldp_demo",
* configuration: {
* key1: "value1",
* key2: "value2",
* },
* clusters: [
* {
* label: "default",
* numWorkers: 2,
* customTags: {
* cluster_type: "default",
* },
* },
* {
* label: "maintenance",
* numWorkers: 1,
* customTags: {
* cluster_type: "maintenance",
* },
* },
* ],
* libraries: [
* {
* notebook: {
* path: ldpDemo.id,
* },
* },
* {
* file: {
* path: pulumi.interpolate`${ldpDemoRepo.path}/pipeline.sql`,
* },
* },
* {
* glob: {
* include: pulumi.interpolate`${ldpDemoRepo.path}/subfolder/**`,
* },
* },
* ],
* continuous: false,
* notifications: [{
* emailRecipients: [
* "user@domain.com",
* "user1@domain.com",
* ],
* alerts: [
* "on-update-failure",
* "on-update-fatal-failure",
* "on-update-success",
* "on-flow-failure",
* ],
* }],
* });
* ```
*
* ## Related Resources
*
* The following resources are often used in the same context:
*
* * End to end workspace management guide.
* * databricks.getPipelines to retrieve [Lakeflow Declarative Pipelines](https://docs.databricks.com/aws/en/dlt) data.
* * databricks.Cluster to create [Databricks Clusters](https://docs.databricks.com/clusters/index.html).
* * databricks.Job to manage [Databricks Jobs](https://docs.databricks.com/jobs.html) to run non-interactive code in a databricks_cluster.
* * databricks.Notebook to manage [Databricks Notebooks](https://docs.databricks.com/notebooks/index.html).
*
* ## Import
*
* The resource job can be imported using the id of the pipeline
*
* hcl
*
* import {
*
* to = databricks_pipeline.this
*
* id = "<pipeline-id>"
*
* }
*
* Alternatively, when using `terraform` version 1.4 or earlier, import using the `pulumi import` command:
*
* bash
*
* ```sh
* $ pulumi import databricks:index/pipeline:Pipeline this <pipeline-id>
* ```
*/
export declare class Pipeline extends pulumi.CustomResource {
/**
* Get an existing Pipeline resource's state with the given name, ID, and optional extra
* properties used to qualify the lookup.
*
* @param name The _unique_ name of the resulting resource.
* @param id The _unique_ provider ID of the resource to lookup.
* @param state Any extra arguments used during the lookup.
* @param opts Optional settings to control the behavior of the CustomResource.
*/
static get(name: string, id: pulumi.Input<pulumi.ID>, state?: PipelineState, opts?: pulumi.CustomResourceOptions): Pipeline;
/**
* Returns true if the given object is an instance of Pipeline. This is designed to work even
* when multiple copies of the Pulumi SDK have been loaded into the same process.
*/
static isInstance(obj: any): obj is Pipeline;
/**
* Optional boolean flag. If false, deployment will fail if name conflicts with that of another pipeline. default is `false`.
*/
readonly allowDuplicateNames: pulumi.Output<boolean | undefined>;
/**
* optional string specifying ID of the budget policy for this Lakeflow Declarative Pipeline.
*/
readonly budgetPolicyId: pulumi.Output<string | undefined>;
/**
* The name of catalog in Unity Catalog. *Change of this parameter forces recreation of the pipeline.* (Conflicts with `storage`).
*/
readonly catalog: pulumi.Output<string | undefined>;
readonly cause: pulumi.Output<string>;
/**
* optional name of the release channel for Spark version used by Lakeflow Declarative Pipeline. Supported values are: `CURRENT` (default) and `PREVIEW`.
*/
readonly channel: pulumi.Output<string | undefined>;
readonly clusterId: pulumi.Output<string>;
/**
* blocks - Clusters to run the pipeline. If none is specified, pipelines will automatically select a default cluster configuration for the pipeline. *Please note that Lakeflow Declarative Pipeline clusters are supporting only subset of attributes as described in [documentation](https://docs.databricks.com/api/workspace/pipelines/create#clusters).* Also, note that `autoscale` block is extended with the `mode` parameter that controls the autoscaling algorithm (possible values are `ENHANCED` for new, enhanced autoscaling algorithm, or `LEGACY` for old algorithm).
*/
readonly clusters: pulumi.Output<outputs.PipelineCluster[] | undefined>;
/**
* An optional list of values to apply to the entire pipeline. Elements must be formatted as key:value pairs.
*/
readonly configuration: pulumi.Output<{
[key: string]: string;
} | undefined>;
/**
* A flag indicating whether to run the pipeline continuously. The default value is `false`.
*/
readonly continuous: pulumi.Output<boolean | undefined>;
readonly creatorUserName: pulumi.Output<string>;
/**
* Deployment type of this pipeline. Supports following attributes:
*/
readonly deployment: pulumi.Output<outputs.PipelineDeployment | undefined>;
/**
* A flag indicating whether to run the pipeline in development mode. The default value is `false`.
*/
readonly development: pulumi.Output<boolean | undefined>;
/**
* optional name of the [product edition](https://docs.databricks.com/aws/en/dlt/configure-pipeline#choose-a-product-edition). Supported values are: `CORE`, `PRO`, `ADVANCED` (default). Not required when `serverless` is set to `true`.
*/
readonly edition: pulumi.Output<string | undefined>;
readonly environment: pulumi.Output<outputs.PipelineEnvironment | undefined>;
/**
* an optional block specifying a table where LDP Event Log will be stored. Consists of the following fields:
*/
readonly eventLog: pulumi.Output<outputs.PipelineEventLog | undefined>;
readonly expectedLastModified: pulumi.Output<number | undefined>;
/**
* Filters on which Pipeline packages to include in the deployed graph. This block consists of following attributes:
*/
readonly filters: pulumi.Output<outputs.PipelineFilters | undefined>;
/**
* The definition of a gateway pipeline to support CDC. Consists of following attributes:
*/
readonly gatewayDefinition: pulumi.Output<outputs.PipelineGatewayDefinition | undefined>;
readonly health: pulumi.Output<string>;
readonly ingestionDefinition: pulumi.Output<outputs.PipelineIngestionDefinition | undefined>;
readonly lastModified: pulumi.Output<number>;
readonly latestUpdates: pulumi.Output<outputs.PipelineLatestUpdate[]>;
/**
* blocks - Specifies pipeline code.
*/
readonly libraries: pulumi.Output<outputs.PipelineLibrary[] | undefined>;
/**
* A user-friendly name for this pipeline. The name can be used to identify pipeline jobs in the UI.
*/
readonly name: pulumi.Output<string>;
readonly notifications: pulumi.Output<outputs.PipelineNotification[] | undefined>;
/**
* A flag indicating whether to use Photon engine. The default value is `false`.
*/
readonly photon: pulumi.Output<boolean | undefined>;
readonly restartWindow: pulumi.Output<outputs.PipelineRestartWindow | undefined>;
/**
* An optional string specifying the root path for this pipeline. This is used as the root directory when editing the pipeline in the Databricks user interface and it is added to `sys.path` when executing Python sources during pipeline execution.
*/
readonly rootPath: pulumi.Output<string | undefined>;
readonly runAs: pulumi.Output<outputs.PipelineRunAs | undefined>;
readonly runAsUserName: pulumi.Output<string>;
/**
* The default schema (database) where tables are read from or published to. The presence of this attribute implies that the pipeline is in direct publishing mode.
*/
readonly schema: pulumi.Output<string | undefined>;
/**
* An optional flag indicating if serverless compute should be used for this Lakeflow Declarative Pipeline. Requires `catalog` to be set, as it could be used only with Unity Catalog.
*/
readonly serverless: pulumi.Output<boolean | undefined>;
readonly state: pulumi.Output<string>;
/**
* A location on cloud storage where output data and metadata required for pipeline execution are stored. By default, tables are stored in a subdirectory of this location. *Change of this parameter forces recreation of the pipeline.* (Conflicts with `catalog`).
*/
readonly storage: pulumi.Output<string | undefined>;
/**
* A map of tags associated with the pipeline. These are forwarded to the cluster as cluster tags, and are therefore subject to the same limitations. A maximum of 25 tags can be added to the pipeline.
*/
readonly tags: pulumi.Output<{
[key: string]: string;
} | undefined>;
/**
* The name of a database (in either the Hive metastore or in a UC catalog) for persisting pipeline output data. Configuring the target setting allows you to view and query the pipeline output data from the Databricks UI.
*/
readonly target: pulumi.Output<string | undefined>;
readonly trigger: pulumi.Output<outputs.PipelineTrigger | undefined>;
/**
* URL of the Lakeflow Declarative Pipeline on the given workspace.
*/
readonly url: pulumi.Output<string>;
/**
* Create a Pipeline resource with the given unique name, arguments, and options.
*
* @param name The _unique_ name of the resource.
* @param args The arguments to use to populate this resource's properties.
* @param opts A bag of options that control this resource's behavior.
*/
constructor(name: string, args?: PipelineArgs, opts?: pulumi.CustomResourceOptions);
}
/**
* Input properties used for looking up and filtering Pipeline resources.
*/
export interface PipelineState {
/**
* Optional boolean flag. If false, deployment will fail if name conflicts with that of another pipeline. default is `false`.
*/
allowDuplicateNames?: pulumi.Input<boolean>;
/**
* optional string specifying ID of the budget policy for this Lakeflow Declarative Pipeline.
*/
budgetPolicyId?: pulumi.Input<string>;
/**
* The name of catalog in Unity Catalog. *Change of this parameter forces recreation of the pipeline.* (Conflicts with `storage`).
*/
catalog?: pulumi.Input<string>;
cause?: pulumi.Input<string>;
/**
* optional name of the release channel for Spark version used by Lakeflow Declarative Pipeline. Supported values are: `CURRENT` (default) and `PREVIEW`.
*/
channel?: pulumi.Input<string>;
clusterId?: pulumi.Input<string>;
/**
* blocks - Clusters to run the pipeline. If none is specified, pipelines will automatically select a default cluster configuration for the pipeline. *Please note that Lakeflow Declarative Pipeline clusters are supporting only subset of attributes as described in [documentation](https://docs.databricks.com/api/workspace/pipelines/create#clusters).* Also, note that `autoscale` block is extended with the `mode` parameter that controls the autoscaling algorithm (possible values are `ENHANCED` for new, enhanced autoscaling algorithm, or `LEGACY` for old algorithm).
*/
clusters?: pulumi.Input<pulumi.Input<inputs.PipelineCluster>[]>;
/**
* An optional list of values to apply to the entire pipeline. Elements must be formatted as key:value pairs.
*/
configuration?: pulumi.Input<{
[key: string]: pulumi.Input<string>;
}>;
/**
* A flag indicating whether to run the pipeline continuously. The default value is `false`.
*/
continuous?: pulumi.Input<boolean>;
creatorUserName?: pulumi.Input<string>;
/**
* Deployment type of this pipeline. Supports following attributes:
*/
deployment?: pulumi.Input<inputs.PipelineDeployment>;
/**
* A flag indicating whether to run the pipeline in development mode. The default value is `false`.
*/
development?: pulumi.Input<boolean>;
/**
* optional name of the [product edition](https://docs.databricks.com/aws/en/dlt/configure-pipeline#choose-a-product-edition). Supported values are: `CORE`, `PRO`, `ADVANCED` (default). Not required when `serverless` is set to `true`.
*/
edition?: pulumi.Input<string>;
environment?: pulumi.Input<inputs.PipelineEnvironment>;
/**
* an optional block specifying a table where LDP Event Log will be stored. Consists of the following fields:
*/
eventLog?: pulumi.Input<inputs.PipelineEventLog>;
expectedLastModified?: pulumi.Input<number>;
/**
* Filters on which Pipeline packages to include in the deployed graph. This block consists of following attributes:
*/
filters?: pulumi.Input<inputs.PipelineFilters>;
/**
* The definition of a gateway pipeline to support CDC. Consists of following attributes:
*/
gatewayDefinition?: pulumi.Input<inputs.PipelineGatewayDefinition>;
health?: pulumi.Input<string>;
ingestionDefinition?: pulumi.Input<inputs.PipelineIngestionDefinition>;
lastModified?: pulumi.Input<number>;
latestUpdates?: pulumi.Input<pulumi.Input<inputs.PipelineLatestUpdate>[]>;
/**
* blocks - Specifies pipeline code.
*/
libraries?: pulumi.Input<pulumi.Input<inputs.PipelineLibrary>[]>;
/**
* A user-friendly name for this pipeline. The name can be used to identify pipeline jobs in the UI.
*/
name?: pulumi.Input<string>;
notifications?: pulumi.Input<pulumi.Input<inputs.PipelineNotification>[]>;
/**
* A flag indicating whether to use Photon engine. The default value is `false`.
*/
photon?: pulumi.Input<boolean>;
restartWindow?: pulumi.Input<inputs.PipelineRestartWindow>;
/**
* An optional string specifying the root path for this pipeline. This is used as the root directory when editing the pipeline in the Databricks user interface and it is added to `sys.path` when executing Python sources during pipeline execution.
*/
rootPath?: pulumi.Input<string>;
runAs?: pulumi.Input<inputs.PipelineRunAs>;
runAsUserName?: pulumi.Input<string>;
/**
* The default schema (database) where tables are read from or published to. The presence of this attribute implies that the pipeline is in direct publishing mode.
*/
schema?: pulumi.Input<string>;
/**
* An optional flag indicating if serverless compute should be used for this Lakeflow Declarative Pipeline. Requires `catalog` to be set, as it could be used only with Unity Catalog.
*/
serverless?: pulumi.Input<boolean>;
state?: pulumi.Input<string>;
/**
* A location on cloud storage where output data and metadata required for pipeline execution are stored. By default, tables are stored in a subdirectory of this location. *Change of this parameter forces recreation of the pipeline.* (Conflicts with `catalog`).
*/
storage?: pulumi.Input<string>;
/**
* A map of tags associated with the pipeline. These are forwarded to the cluster as cluster tags, and are therefore subject to the same limitations. A maximum of 25 tags can be added to the pipeline.
*/
tags?: pulumi.Input<{
[key: string]: pulumi.Input<string>;
}>;
/**
* The name of a database (in either the Hive metastore or in a UC catalog) for persisting pipeline output data. Configuring the target setting allows you to view and query the pipeline output data from the Databricks UI.
*/
target?: pulumi.Input<string>;
trigger?: pulumi.Input<inputs.PipelineTrigger>;
/**
* URL of the Lakeflow Declarative Pipeline on the given workspace.
*/
url?: pulumi.Input<string>;
}
/**
* The set of arguments for constructing a Pipeline resource.
*/
export interface PipelineArgs {
/**
* Optional boolean flag. If false, deployment will fail if name conflicts with that of another pipeline. default is `false`.
*/
allowDuplicateNames?: pulumi.Input<boolean>;
/**
* optional string specifying ID of the budget policy for this Lakeflow Declarative Pipeline.
*/
budgetPolicyId?: pulumi.Input<string>;
/**
* The name of catalog in Unity Catalog. *Change of this parameter forces recreation of the pipeline.* (Conflicts with `storage`).
*/
catalog?: pulumi.Input<string>;
cause?: pulumi.Input<string>;
/**
* optional name of the release channel for Spark version used by Lakeflow Declarative Pipeline. Supported values are: `CURRENT` (default) and `PREVIEW`.
*/
channel?: pulumi.Input<string>;
clusterId?: pulumi.Input<string>;
/**
* blocks - Clusters to run the pipeline. If none is specified, pipelines will automatically select a default cluster configuration for the pipeline. *Please note that Lakeflow Declarative Pipeline clusters are supporting only subset of attributes as described in [documentation](https://docs.databricks.com/api/workspace/pipelines/create#clusters).* Also, note that `autoscale` block is extended with the `mode` parameter that controls the autoscaling algorithm (possible values are `ENHANCED` for new, enhanced autoscaling algorithm, or `LEGACY` for old algorithm).
*/
clusters?: pulumi.Input<pulumi.Input<inputs.PipelineCluster>[]>;
/**
* An optional list of values to apply to the entire pipeline. Elements must be formatted as key:value pairs.
*/
configuration?: pulumi.Input<{
[key: string]: pulumi.Input<string>;
}>;
/**
* A flag indicating whether to run the pipeline continuously. The default value is `false`.
*/
continuous?: pulumi.Input<boolean>;
creatorUserName?: pulumi.Input<string>;
/**
* Deployment type of this pipeline. Supports following attributes:
*/
deployment?: pulumi.Input<inputs.PipelineDeployment>;
/**
* A flag indicating whether to run the pipeline in development mode. The default value is `false`.
*/
development?: pulumi.Input<boolean>;
/**
* optional name of the [product edition](https://docs.databricks.com/aws/en/dlt/configure-pipeline#choose-a-product-edition). Supported values are: `CORE`, `PRO`, `ADVANCED` (default). Not required when `serverless` is set to `true`.
*/
edition?: pulumi.Input<string>;
environment?: pulumi.Input<inputs.PipelineEnvironment>;
/**
* an optional block specifying a table where LDP Event Log will be stored. Consists of the following fields:
*/
eventLog?: pulumi.Input<inputs.PipelineEventLog>;
expectedLastModified?: pulumi.Input<number>;
/**
* Filters on which Pipeline packages to include in the deployed graph. This block consists of following attributes:
*/
filters?: pulumi.Input<inputs.PipelineFilters>;
/**
* The definition of a gateway pipeline to support CDC. Consists of following attributes:
*/
gatewayDefinition?: pulumi.Input<inputs.PipelineGatewayDefinition>;
health?: pulumi.Input<string>;
ingestionDefinition?: pulumi.Input<inputs.PipelineIngestionDefinition>;
lastModified?: pulumi.Input<number>;
latestUpdates?: pulumi.Input<pulumi.Input<inputs.PipelineLatestUpdate>[]>;
/**
* blocks - Specifies pipeline code.
*/
libraries?: pulumi.Input<pulumi.Input<inputs.PipelineLibrary>[]>;
/**
* A user-friendly name for this pipeline. The name can be used to identify pipeline jobs in the UI.
*/
name?: pulumi.Input<string>;
notifications?: pulumi.Input<pulumi.Input<inputs.PipelineNotification>[]>;
/**
* A flag indicating whether to use Photon engine. The default value is `false`.
*/
photon?: pulumi.Input<boolean>;
restartWindow?: pulumi.Input<inputs.PipelineRestartWindow>;
/**
* An optional string specifying the root path for this pipeline. This is used as the root directory when editing the pipeline in the Databricks user interface and it is added to `sys.path` when executing Python sources during pipeline execution.
*/
rootPath?: pulumi.Input<string>;
runAs?: pulumi.Input<inputs.PipelineRunAs>;
runAsUserName?: pulumi.Input<string>;
/**
* The default schema (database) where tables are read from or published to. The presence of this attribute implies that the pipeline is in direct publishing mode.
*/
schema?: pulumi.Input<string>;
/**
* An optional flag indicating if serverless compute should be used for this Lakeflow Declarative Pipeline. Requires `catalog` to be set, as it could be used only with Unity Catalog.
*/
serverless?: pulumi.Input<boolean>;
state?: pulumi.Input<string>;
/**
* A location on cloud storage where output data and metadata required for pipeline execution are stored. By default, tables are stored in a subdirectory of this location. *Change of this parameter forces recreation of the pipeline.* (Conflicts with `catalog`).
*/
storage?: pulumi.Input<string>;
/**
* A map of tags associated with the pipeline. These are forwarded to the cluster as cluster tags, and are therefore subject to the same limitations. A maximum of 25 tags can be added to the pipeline.
*/
tags?: pulumi.Input<{
[key: string]: pulumi.Input<string>;
}>;
/**
* The name of a database (in either the Hive metastore or in a UC catalog) for persisting pipeline output data. Configuring the target setting allows you to view and query the pipeline output data from the Databricks UI.
*/
target?: pulumi.Input<string>;
trigger?: pulumi.Input<inputs.PipelineTrigger>;
/**
* URL of the Lakeflow Declarative Pipeline on the given workspace.
*/
url?: pulumi.Input<string>;
}