@pulumi/databricks
Version:
A Pulumi package for creating and managing databricks cloud resources.
411 lines (410 loc) • 20.7 kB
TypeScript
import * as pulumi from "@pulumi/pulumi";
import * as inputs from "./types/input";
import * as outputs from "./types/output";
/**
* Use `databricks.Pipeline` to deploy [Delta Live Tables](https://docs.databricks.com/data-engineering/delta-live-tables/index.html).
*
* ## Example Usage
*
* ```typescript
* import * as pulumi from "@pulumi/pulumi";
* import * as databricks from "@pulumi/databricks";
*
* const dltDemo = new databricks.Notebook("dlt_demo", {});
* const dltDemoRepo = new databricks.Repo("dlt_demo", {});
* const _this = new databricks.Pipeline("this", {
* name: "Pipeline Name",
* storage: "/test/first-pipeline",
* configuration: {
* key1: "value1",
* key2: "value2",
* },
* clusters: [
* {
* label: "default",
* numWorkers: 2,
* customTags: {
* cluster_type: "default",
* },
* },
* {
* label: "maintenance",
* numWorkers: 1,
* customTags: {
* cluster_type: "maintenance",
* },
* },
* ],
* libraries: [
* {
* notebook: {
* path: dltDemo.id,
* },
* },
* {
* file: {
* path: pulumi.interpolate`${dltDemoRepo.path}/pipeline.sql`,
* },
* },
* ],
* continuous: false,
* notifications: [{
* emailRecipients: [
* "user@domain.com",
* "user1@domain.com",
* ],
* alerts: [
* "on-update-failure",
* "on-update-fatal-failure",
* "on-update-success",
* "on-flow-failure",
* ],
* }],
* });
* ```
*
* ## Related Resources
*
* The following resources are often used in the same context:
*
* * End to end workspace management guide.
* * databricks.getPipelines to retrieve [Delta Live Tables](https://docs.databricks.com/data-engineering/delta-live-tables/index.html) pipeline data.
* * databricks.Cluster to create [Databricks Clusters](https://docs.databricks.com/clusters/index.html).
* * databricks.Job to manage [Databricks Jobs](https://docs.databricks.com/jobs.html) to run non-interactive code in a databricks_cluster.
* * databricks.Notebook to manage [Databricks Notebooks](https://docs.databricks.com/notebooks/index.html).
*
* ## Import
*
* The resource job can be imported using the id of the pipeline
*
* bash
*
* ```sh
* $ pulumi import databricks:index/pipeline:Pipeline this <pipeline-id>
* ```
*/
export declare class Pipeline extends pulumi.CustomResource {
/**
* Get an existing Pipeline resource's state with the given name, ID, and optional extra
* properties used to qualify the lookup.
*
* @param name The _unique_ name of the resulting resource.
* @param id The _unique_ provider ID of the resource to lookup.
* @param state Any extra arguments used during the lookup.
* @param opts Optional settings to control the behavior of the CustomResource.
*/
static get(name: string, id: pulumi.Input<pulumi.ID>, state?: PipelineState, opts?: pulumi.CustomResourceOptions): Pipeline;
/**
* Returns true if the given object is an instance of Pipeline. This is designed to work even
* when multiple copies of the Pulumi SDK have been loaded into the same process.
*/
static isInstance(obj: any): obj is Pipeline;
/**
* Optional boolean flag. If false, deployment will fail if name conflicts with that of another pipeline. default is `false`.
*/
readonly allowDuplicateNames: pulumi.Output<boolean | undefined>;
/**
* optional string specifying ID of the budget policy for this DLT pipeline.
*/
readonly budgetPolicyId: pulumi.Output<string | undefined>;
/**
* The name of catalog in Unity Catalog. *Change of this parameter forces recreation of the pipeline.* (Conflicts with `storage`).
*/
readonly catalog: pulumi.Output<string | undefined>;
readonly cause: pulumi.Output<string>;
/**
* optional name of the release channel for Spark version used by DLT pipeline. Supported values are: `CURRENT` (default) and `PREVIEW`.
*/
readonly channel: pulumi.Output<string | undefined>;
readonly clusterId: pulumi.Output<string>;
/**
* blocks - Clusters to run the pipeline. If none is specified, pipelines will automatically select a default cluster configuration for the pipeline. *Please note that DLT pipeline clusters are supporting only subset of attributes as described in [documentation](https://docs.databricks.com/data-engineering/delta-live-tables/delta-live-tables-api-guide.html#pipelinesnewcluster).* Also, note that `autoscale` block is extended with the `mode` parameter that controls the autoscaling algorithm (possible values are `ENHANCED` for new, enhanced autoscaling algorithm, or `LEGACY` for old algorithm).
*/
readonly clusters: pulumi.Output<outputs.PipelineCluster[] | undefined>;
/**
* An optional list of values to apply to the entire pipeline. Elements must be formatted as key:value pairs.
*/
readonly configuration: pulumi.Output<{
[key: string]: string;
} | undefined>;
/**
* A flag indicating whether to run the pipeline continuously. The default value is `false`.
*/
readonly continuous: pulumi.Output<boolean | undefined>;
readonly creatorUserName: pulumi.Output<string>;
/**
* Deployment type of this pipeline. Supports following attributes:
*/
readonly deployment: pulumi.Output<outputs.PipelineDeployment | undefined>;
/**
* A flag indicating whether to run the pipeline in development mode. The default value is `false`.
*/
readonly development: pulumi.Output<boolean | undefined>;
/**
* optional name of the [product edition](https://docs.databricks.com/data-engineering/delta-live-tables/delta-live-tables-concepts.html#editions). Supported values are: `CORE`, `PRO`, `ADVANCED` (default). Not required when `serverless` is set to `true`.
*/
readonly edition: pulumi.Output<string | undefined>;
readonly eventLog: pulumi.Output<outputs.PipelineEventLog | undefined>;
readonly expectedLastModified: pulumi.Output<number | undefined>;
/**
* Filters on which Pipeline packages to include in the deployed graph. This block consists of following attributes:
*/
readonly filters: pulumi.Output<outputs.PipelineFilters | undefined>;
/**
* The definition of a gateway pipeline to support CDC. Consists of following attributes:
*/
readonly gatewayDefinition: pulumi.Output<outputs.PipelineGatewayDefinition | undefined>;
readonly health: pulumi.Output<string>;
readonly ingestionDefinition: pulumi.Output<outputs.PipelineIngestionDefinition | undefined>;
readonly lastModified: pulumi.Output<number>;
readonly latestUpdates: pulumi.Output<outputs.PipelineLatestUpdate[]>;
/**
* blocks - Specifies pipeline code and required artifacts. Syntax resembles library configuration block with the addition of a special `notebook` & `file` library types that should have the `path` attribute. *Right now only the `notebook` & `file` types are supported.*
*/
readonly libraries: pulumi.Output<outputs.PipelineLibrary[] | undefined>;
/**
* A user-friendly name for this pipeline. The name can be used to identify pipeline jobs in the UI.
*/
readonly name: pulumi.Output<string>;
readonly notifications: pulumi.Output<outputs.PipelineNotification[] | undefined>;
/**
* A flag indicating whether to use Photon engine. The default value is `false`.
*/
readonly photon: pulumi.Output<boolean | undefined>;
readonly restartWindow: pulumi.Output<outputs.PipelineRestartWindow | undefined>;
readonly runAs: pulumi.Output<outputs.PipelineRunAs>;
readonly runAsUserName: pulumi.Output<string>;
/**
* The default schema (database) where tables are read from or published to. The presence of this attribute implies that the pipeline is in direct publishing mode.
*/
readonly schema: pulumi.Output<string | undefined>;
/**
* An optional flag indicating if serverless compute should be used for this DLT pipeline. Requires `catalog` to be set, as it could be used only with Unity Catalog.
*/
readonly serverless: pulumi.Output<boolean | undefined>;
readonly state: pulumi.Output<string>;
/**
* A location on DBFS or cloud storage where output data and metadata required for pipeline execution are stored. By default, tables are stored in a subdirectory of this location. *Change of this parameter forces recreation of the pipeline.* (Conflicts with `catalog`).
*/
readonly storage: pulumi.Output<string | undefined>;
/**
* The name of a database (in either the Hive metastore or in a UC catalog) for persisting pipeline output data. Configuring the target setting allows you to view and query the pipeline output data from the Databricks UI.
*/
readonly target: pulumi.Output<string | undefined>;
readonly trigger: pulumi.Output<outputs.PipelineTrigger | undefined>;
/**
* URL of the DLT pipeline on the given workspace.
*/
readonly url: pulumi.Output<string>;
/**
* Create a Pipeline resource with the given unique name, arguments, and options.
*
* @param name The _unique_ name of the resource.
* @param args The arguments to use to populate this resource's properties.
* @param opts A bag of options that control this resource's behavior.
*/
constructor(name: string, args?: PipelineArgs, opts?: pulumi.CustomResourceOptions);
}
/**
* Input properties used for looking up and filtering Pipeline resources.
*/
export interface PipelineState {
/**
* Optional boolean flag. If false, deployment will fail if name conflicts with that of another pipeline. default is `false`.
*/
allowDuplicateNames?: pulumi.Input<boolean>;
/**
* optional string specifying ID of the budget policy for this DLT pipeline.
*/
budgetPolicyId?: pulumi.Input<string>;
/**
* The name of catalog in Unity Catalog. *Change of this parameter forces recreation of the pipeline.* (Conflicts with `storage`).
*/
catalog?: pulumi.Input<string>;
cause?: pulumi.Input<string>;
/**
* optional name of the release channel for Spark version used by DLT pipeline. Supported values are: `CURRENT` (default) and `PREVIEW`.
*/
channel?: pulumi.Input<string>;
clusterId?: pulumi.Input<string>;
/**
* blocks - Clusters to run the pipeline. If none is specified, pipelines will automatically select a default cluster configuration for the pipeline. *Please note that DLT pipeline clusters are supporting only subset of attributes as described in [documentation](https://docs.databricks.com/data-engineering/delta-live-tables/delta-live-tables-api-guide.html#pipelinesnewcluster).* Also, note that `autoscale` block is extended with the `mode` parameter that controls the autoscaling algorithm (possible values are `ENHANCED` for new, enhanced autoscaling algorithm, or `LEGACY` for old algorithm).
*/
clusters?: pulumi.Input<pulumi.Input<inputs.PipelineCluster>[]>;
/**
* An optional list of values to apply to the entire pipeline. Elements must be formatted as key:value pairs.
*/
configuration?: pulumi.Input<{
[key: string]: pulumi.Input<string>;
}>;
/**
* A flag indicating whether to run the pipeline continuously. The default value is `false`.
*/
continuous?: pulumi.Input<boolean>;
creatorUserName?: pulumi.Input<string>;
/**
* Deployment type of this pipeline. Supports following attributes:
*/
deployment?: pulumi.Input<inputs.PipelineDeployment>;
/**
* A flag indicating whether to run the pipeline in development mode. The default value is `false`.
*/
development?: pulumi.Input<boolean>;
/**
* optional name of the [product edition](https://docs.databricks.com/data-engineering/delta-live-tables/delta-live-tables-concepts.html#editions). Supported values are: `CORE`, `PRO`, `ADVANCED` (default). Not required when `serverless` is set to `true`.
*/
edition?: pulumi.Input<string>;
eventLog?: pulumi.Input<inputs.PipelineEventLog>;
expectedLastModified?: pulumi.Input<number>;
/**
* Filters on which Pipeline packages to include in the deployed graph. This block consists of following attributes:
*/
filters?: pulumi.Input<inputs.PipelineFilters>;
/**
* The definition of a gateway pipeline to support CDC. Consists of following attributes:
*/
gatewayDefinition?: pulumi.Input<inputs.PipelineGatewayDefinition>;
health?: pulumi.Input<string>;
ingestionDefinition?: pulumi.Input<inputs.PipelineIngestionDefinition>;
lastModified?: pulumi.Input<number>;
latestUpdates?: pulumi.Input<pulumi.Input<inputs.PipelineLatestUpdate>[]>;
/**
* blocks - Specifies pipeline code and required artifacts. Syntax resembles library configuration block with the addition of a special `notebook` & `file` library types that should have the `path` attribute. *Right now only the `notebook` & `file` types are supported.*
*/
libraries?: pulumi.Input<pulumi.Input<inputs.PipelineLibrary>[]>;
/**
* A user-friendly name for this pipeline. The name can be used to identify pipeline jobs in the UI.
*/
name?: pulumi.Input<string>;
notifications?: pulumi.Input<pulumi.Input<inputs.PipelineNotification>[]>;
/**
* A flag indicating whether to use Photon engine. The default value is `false`.
*/
photon?: pulumi.Input<boolean>;
restartWindow?: pulumi.Input<inputs.PipelineRestartWindow>;
runAs?: pulumi.Input<inputs.PipelineRunAs>;
runAsUserName?: pulumi.Input<string>;
/**
* The default schema (database) where tables are read from or published to. The presence of this attribute implies that the pipeline is in direct publishing mode.
*/
schema?: pulumi.Input<string>;
/**
* An optional flag indicating if serverless compute should be used for this DLT pipeline. Requires `catalog` to be set, as it could be used only with Unity Catalog.
*/
serverless?: pulumi.Input<boolean>;
state?: pulumi.Input<string>;
/**
* A location on DBFS or cloud storage where output data and metadata required for pipeline execution are stored. By default, tables are stored in a subdirectory of this location. *Change of this parameter forces recreation of the pipeline.* (Conflicts with `catalog`).
*/
storage?: pulumi.Input<string>;
/**
* The name of a database (in either the Hive metastore or in a UC catalog) for persisting pipeline output data. Configuring the target setting allows you to view and query the pipeline output data from the Databricks UI.
*/
target?: pulumi.Input<string>;
trigger?: pulumi.Input<inputs.PipelineTrigger>;
/**
* URL of the DLT pipeline on the given workspace.
*/
url?: pulumi.Input<string>;
}
/**
* The set of arguments for constructing a Pipeline resource.
*/
export interface PipelineArgs {
/**
* Optional boolean flag. If false, deployment will fail if name conflicts with that of another pipeline. default is `false`.
*/
allowDuplicateNames?: pulumi.Input<boolean>;
/**
* optional string specifying ID of the budget policy for this DLT pipeline.
*/
budgetPolicyId?: pulumi.Input<string>;
/**
* The name of catalog in Unity Catalog. *Change of this parameter forces recreation of the pipeline.* (Conflicts with `storage`).
*/
catalog?: pulumi.Input<string>;
cause?: pulumi.Input<string>;
/**
* optional name of the release channel for Spark version used by DLT pipeline. Supported values are: `CURRENT` (default) and `PREVIEW`.
*/
channel?: pulumi.Input<string>;
clusterId?: pulumi.Input<string>;
/**
* blocks - Clusters to run the pipeline. If none is specified, pipelines will automatically select a default cluster configuration for the pipeline. *Please note that DLT pipeline clusters are supporting only subset of attributes as described in [documentation](https://docs.databricks.com/data-engineering/delta-live-tables/delta-live-tables-api-guide.html#pipelinesnewcluster).* Also, note that `autoscale` block is extended with the `mode` parameter that controls the autoscaling algorithm (possible values are `ENHANCED` for new, enhanced autoscaling algorithm, or `LEGACY` for old algorithm).
*/
clusters?: pulumi.Input<pulumi.Input<inputs.PipelineCluster>[]>;
/**
* An optional list of values to apply to the entire pipeline. Elements must be formatted as key:value pairs.
*/
configuration?: pulumi.Input<{
[key: string]: pulumi.Input<string>;
}>;
/**
* A flag indicating whether to run the pipeline continuously. The default value is `false`.
*/
continuous?: pulumi.Input<boolean>;
creatorUserName?: pulumi.Input<string>;
/**
* Deployment type of this pipeline. Supports following attributes:
*/
deployment?: pulumi.Input<inputs.PipelineDeployment>;
/**
* A flag indicating whether to run the pipeline in development mode. The default value is `false`.
*/
development?: pulumi.Input<boolean>;
/**
* optional name of the [product edition](https://docs.databricks.com/data-engineering/delta-live-tables/delta-live-tables-concepts.html#editions). Supported values are: `CORE`, `PRO`, `ADVANCED` (default). Not required when `serverless` is set to `true`.
*/
edition?: pulumi.Input<string>;
eventLog?: pulumi.Input<inputs.PipelineEventLog>;
expectedLastModified?: pulumi.Input<number>;
/**
* Filters on which Pipeline packages to include in the deployed graph. This block consists of following attributes:
*/
filters?: pulumi.Input<inputs.PipelineFilters>;
/**
* The definition of a gateway pipeline to support CDC. Consists of following attributes:
*/
gatewayDefinition?: pulumi.Input<inputs.PipelineGatewayDefinition>;
health?: pulumi.Input<string>;
ingestionDefinition?: pulumi.Input<inputs.PipelineIngestionDefinition>;
lastModified?: pulumi.Input<number>;
latestUpdates?: pulumi.Input<pulumi.Input<inputs.PipelineLatestUpdate>[]>;
/**
* blocks - Specifies pipeline code and required artifacts. Syntax resembles library configuration block with the addition of a special `notebook` & `file` library types that should have the `path` attribute. *Right now only the `notebook` & `file` types are supported.*
*/
libraries?: pulumi.Input<pulumi.Input<inputs.PipelineLibrary>[]>;
/**
* A user-friendly name for this pipeline. The name can be used to identify pipeline jobs in the UI.
*/
name?: pulumi.Input<string>;
notifications?: pulumi.Input<pulumi.Input<inputs.PipelineNotification>[]>;
/**
* A flag indicating whether to use Photon engine. The default value is `false`.
*/
photon?: pulumi.Input<boolean>;
restartWindow?: pulumi.Input<inputs.PipelineRestartWindow>;
runAs?: pulumi.Input<inputs.PipelineRunAs>;
/**
* The default schema (database) where tables are read from or published to. The presence of this attribute implies that the pipeline is in direct publishing mode.
*/
schema?: pulumi.Input<string>;
/**
* An optional flag indicating if serverless compute should be used for this DLT pipeline. Requires `catalog` to be set, as it could be used only with Unity Catalog.
*/
serverless?: pulumi.Input<boolean>;
state?: pulumi.Input<string>;
/**
* A location on DBFS or cloud storage where output data and metadata required for pipeline execution are stored. By default, tables are stored in a subdirectory of this location. *Change of this parameter forces recreation of the pipeline.* (Conflicts with `catalog`).
*/
storage?: pulumi.Input<string>;
/**
* The name of a database (in either the Hive metastore or in a UC catalog) for persisting pipeline output data. Configuring the target setting allows you to view and query the pipeline output data from the Databricks UI.
*/
target?: pulumi.Input<string>;
trigger?: pulumi.Input<inputs.PipelineTrigger>;
/**
* URL of the DLT pipeline on the given workspace.
*/
url?: pulumi.Input<string>;
}