@pulumi/gcp

import * as pulumi from "@pulumi/pulumi"; import * as inputs from "../types/input"; import * as outputs from "../types/output"; /** * Represents a user-visible job which provides the insights for the related data source. * * To get more information about Datascan, see: * * * [API documentation](https://cloud.google.com/dataplex/docs/reference/rest) * * How-to Guides * * [Official Documentation](https://cloud.google.com/dataplex/docs) * * ## Example Usage * * ### Dataplex Datascan Basic Profile * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const basicProfile = new gcp.dataplex.Datascan("basic_profile", { * location: "us-central1", * dataScanId: "dataprofile-basic", * data: { * resource: "//bigquery.googleapis.com/projects/bigquery-public-data/datasets/samples/tables/shakespeare", * }, * executionSpec: { * trigger: { * onDemand: {}, * }, * }, * dataProfileSpec: {}, * project: "my-project-name", * }); * ``` * ### Dataplex Datascan Full Profile * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const source = new gcp.bigquery.Dataset("source", { * datasetId: "dataplex_dataset", * friendlyName: "test", * description: "This is a test description", * location: "US", * deleteContentsOnDestroy: true, * }); * const fullProfile = new gcp.dataplex.Datascan("full_profile", { * location: "us-central1", * displayName: "Full Datascan Profile", * dataScanId: "dataprofile-full", * description: "Example resource - Full Datascan Profile", * labels: { * author: "billing", * }, * data: { * resource: "//bigquery.googleapis.com/projects/bigquery-public-data/datasets/samples/tables/shakespeare", * }, * executionSpec: { * trigger: { * schedule: { * cron: "TZ=America/New_York 1 1 * * *", * }, * }, * }, * dataProfileSpec: { * samplingPercent: 80, * rowFilter: "word_count > 10", * includeFields: { * fieldNames: ["word_count"], * }, * excludeFields: { * fieldNames: ["property_type"], * }, * postScanActions: { * bigqueryExport: { * resultsTable: "//bigquery.googleapis.com/projects/my-project-name/datasets/dataplex_dataset/tables/profile_export", * }, * }, * catalogPublishingEnabled: true, * }, * project: "my-project-name", * }, { * dependsOn: [source], * }); * ``` * ### Dataplex Datascan Onetime Profile * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const onetimeProfile = new gcp.dataplex.Datascan("onetime_profile", { * location: "us-central1", * dataScanId: "dataprofile-onetime", * data: { * resource: "//bigquery.googleapis.com/projects/bigquery-public-data/datasets/samples/tables/shakespeare", * }, * executionSpec: { * trigger: { * oneTime: { * ttlAfterScanCompletion: "120s", * }, * }, * }, * dataProfileSpec: {}, * project: "my-project-name", * }); * ``` * ### Dataplex Datascan Basic Quality * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const basicQuality = new gcp.dataplex.Datascan("basic_quality", { * location: "us-central1", * dataScanId: "dataquality-basic", * data: { * resource: "//bigquery.googleapis.com/projects/bigquery-public-data/datasets/samples/tables/shakespeare", * }, * executionSpec: { * trigger: { * onDemand: {}, * }, * }, * dataQualitySpec: { * rules: [{ * dimension: "VALIDITY", * name: "rule1", * description: "rule 1 for validity dimension", * tableConditionExpectation: { * sqlExpression: "COUNT(*) > 0", * }, * }], * }, * project: "my-project-name", * }); * ``` * ### Dataplex Datascan Full Quality * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const fullQuality = new gcp.dataplex.Datascan("full_quality", { * location: "us-central1", * displayName: "Full Datascan Quality", * dataScanId: "dataquality-full", * description: "Example resource - Full Datascan Quality", * labels: { * author: "billing", * }, * data: { * resource: "//bigquery.googleapis.com/projects/bigquery-public-data/datasets/austin_bikeshare/tables/bikeshare_stations", * }, * executionSpec: { * trigger: { * schedule: { * cron: "TZ=America/New_York 1 1 * * *", * }, * }, * field: "modified_date", * }, * dataQualitySpec: { * samplingPercent: 5, * rowFilter: "station_id > 1000", * catalogPublishingEnabled: true, * filter: "attributes.priority = 'high'", * postScanActions: { * notificationReport: { * recipients: { * emails: ["jane.doe@example.com"], * }, * scoreThresholdTrigger: { * scoreThreshold: 86, * }, * }, * }, * rules: [ * { * column: "address", * dimension: "VALIDITY", * threshold: 0.99, * attributes: { * priority: "high", * }, * nonNullExpectation: {}, * }, * { * column: "council_district", * dimension: "VALIDITY", * ignoreNull: true, * threshold: 0.9, * rangeExpectation: { * minValue: "1", * maxValue: "10", * strictMinEnabled: true, * strictMaxEnabled: false, * }, * }, * { * column: "power_type", * dimension: "VALIDITY", * ignoreNull: false, * regexExpectation: { * regex: ".*solar.*", * }, * }, * { * column: "property_type", * dimension: "VALIDITY", * ignoreNull: false, * setExpectation: { * values: [ * "sidewalk", * "parkland", * ], * }, * }, * { * column: "address", * dimension: "UNIQUENESS", * uniquenessExpectation: {}, * }, * { * column: "number_of_docks", * dimension: "VALIDITY", * statisticRangeExpectation: { * statistic: "MEAN", * minValue: "5", * maxValue: "15", * strictMinEnabled: true, * strictMaxEnabled: true, * }, * }, * { * column: "footprint_length", * dimension: "VALIDITY", * rowConditionExpectation: { * sqlExpression: "footprint_length > 0 AND footprint_length <= 10", * }, * }, * { * dimension: "VALIDITY", * tableConditionExpectation: { * sqlExpression: "COUNT(*) > 0", * }, * }, * { * dimension: "VALIDITY", * sqlAssertion: { * sqlStatement: "select * from bigquery-public-data.austin_bikeshare.bikeshare_stations where station_id is null", * }, * }, * ], * }, * project: "my-project-name", * }); * ``` * ### Dataplex Datascan Onetime Quality * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const onetimeQuality = new gcp.dataplex.Datascan("onetime_quality", { * location: "us-central1", * dataScanId: "dataquality-onetime", * data: { * resource: "//bigquery.googleapis.com/projects/bigquery-public-data/datasets/samples/tables/shakespeare", * }, * executionSpec: { * trigger: { * oneTime: { * ttlAfterScanCompletion: "120s", * }, * }, * }, * dataQualitySpec: { * rules: [{ * dimension: "VALIDITY", * name: "rule1", * description: "rule 1 for validity dimension", * tableConditionExpectation: { * sqlExpression: "COUNT(*) > 0", * }, * }], * }, * project: "my-project-name", * }); * ``` * ### Dataplex Datascan Basic Discovery * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const tfTestBucket = new gcp.storage.Bucket("tf_test_bucket", { * name: "tf-test-bucket-name-_13293", * location: "us-west1", * uniformBucketLevelAccess: true, * }); * const basicDiscovery = new gcp.dataplex.Datascan("basic_discovery", { * location: "us-central1", * dataScanId: "datadiscovery-basic", * data: { * resource: pulumi.interpolate`//storage.googleapis.com/projects/${tfTestBucket.project}/buckets/${tfTestBucket.name}`, * }, * executionSpec: { * trigger: { * onDemand: {}, * }, * }, * dataDiscoverySpec: {}, * project: "my-project-name", * }); * ``` * ### Dataplex Datascan Full Discovery * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const tfTestBucket = new gcp.storage.Bucket("tf_test_bucket", { * name: "tf-test-bucket-name-_40289", * location: "us-west1", * uniformBucketLevelAccess: true, * }); * const tfTestConnection = new gcp.bigquery.Connection("tf_test_connection", { * connectionId: "tf-test-connection-_33395", * location: "us-central1", * friendlyName: "tf-test-connection-_76044", * description: "a bigquery connection for tf test", * cloudResource: {}, * }); * const fullDiscovery = new gcp.dataplex.Datascan("full_discovery", { * location: "us-central1", * displayName: "Full Datascan Discovery", * dataScanId: "datadiscovery-full", * description: "Example resource - Full Datascan Discovery", * labels: { * author: "billing", * }, * data: { * resource: pulumi.interpolate`//storage.googleapis.com/projects/${tfTestBucket.project}/buckets/${tfTestBucket.name}`, * }, * executionSpec: { * trigger: { * schedule: { * cron: "TZ=America/New_York 1 1 * * *", * }, * }, * }, * dataDiscoverySpec: { * bigqueryPublishingConfig: { * tableType: "BIGLAKE", * connection: pulumi.all([tfTestConnection.project, tfTestConnection.location, tfTestConnection.connectionId]).apply(([project, location, connectionId]) => `projects/${project}/locations/${location}/connections/${connectionId}`), * location: tfTestBucket.location, * project: pulumi.interpolate`projects/${tfTestBucket.project}`, * }, * storageConfig: { * includePatterns: [ * "ai*", * "ml*", * ], * excludePatterns: [ * "doc*", * "gen*", * ], * csvOptions: { * headerRows: 5, * delimiter: ",", * encoding: "UTF-8", * typeInferenceDisabled: false, * quote: "'", * }, * jsonOptions: { * encoding: "UTF-8", * typeInferenceDisabled: false, * }, * }, * }, * project: "my-project-name", * }); * ``` * ### Dataplex Datascan Onetime Discovery * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const tfTestBucket = new gcp.storage.Bucket("tf_test_bucket", { * name: "tf-test-bucket-name-_69391", * location: "us-west1", * uniformBucketLevelAccess: true, * }); * const onetimeDiscovery = new gcp.dataplex.Datascan("onetime_discovery", { * location: "us-central1", * dataScanId: "datadiscovery-onetime", * data: { * resource: pulumi.interpolate`//storage.googleapis.com/projects/${tfTestBucket.project}/buckets/${tfTestBucket.name}`, * }, * executionSpec: { * trigger: { * oneTime: { * ttlAfterScanCompletion: "120s", * }, * }, * }, * dataDiscoverySpec: {}, * project: "my-project-name", * }); * ``` * ### Dataplex Datascan Documentation * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const tfDataplexTestDataset = new gcp.bigquery.Dataset("tf_dataplex_test_dataset", { * datasetId: "tf_dataplex_test_dataset_id__8270", * defaultTableExpirationMs: 3600000, * }); * const tfDataplexTestTable = new gcp.bigquery.Table("tf_dataplex_test_table", { * datasetId: tfDataplexTestDataset.datasetId, * tableId: "tf_dataplex_test_table_id__41150", * deletionProtection: false, * schema: ` [ * { * \\"name\\": \\"name\\", * \\"type\\": \\"STRING\\", * \\"mode\\": \\"NULLABLE\\" * }, * { * \\"name\\": \\"station_id\\", * \\"type\\": \\"INTEGER\\", * \\"mode\\": \\"NULLABLE\\", * \\"description\\": \\"The id of the bike station\\" * }, * { * \\"name\\": \\"address\\", * \\"type\\": \\"STRING\\", * \\"mode\\": \\"NULLABLE\\", * \\"description\\": \\"The address of the bike station\\" * }, * { * \\"name\\": \\"power_type\\", * \\"type\\": \\"STRING\\", * \\"mode\\": \\"NULLABLE\\", * \\"description\\": \\"The powert type of the bike station\\" * }, * { * \\"name\\": \\"property_type\\", * \\"type\\": \\"STRING\\", * \\"mode\\": \\"NULLABLE\\", * \\"description\\": \\"The type of the property\\" * }, * { * \\"name\\": \\"number_of_docks\\", * \\"type\\": \\"INTEGER\\", * \\"mode\\": \\"NULLABLE\\", * \\"description\\": \\"The number of docks the property have\\" * }, * { * \\"name\\": \\"footprint_length\\", * \\"type\\": \\"INTEGER\\", * \\"mode\\": \\"NULLABLE\\", * \\"description\\": \\"The footpring lenght of the property\\" * }, * { * \\"name\\": \\"council_district\\", * \\"type\\": \\"INTEGER\\", * \\"mode\\": \\"NULLABLE\\", * \\"description\\": \\"The council district the property is in\\" * } * ] * `, * }); * const documentation = new gcp.dataplex.Datascan("documentation", { * location: "us-central1", * dataScanId: "datadocumentation", * data: { * resource: pulumi.interpolate`//bigquery.googleapis.com/projects/my-project-name/datasets/${tfDataplexTestDataset.datasetId}/tables/${tfDataplexTestTable.tableId}`, * }, * executionSpec: { * trigger: { * onDemand: {}, * }, * }, * dataDocumentationSpec: {}, * project: "my-project-name", * }); * ``` * ### Dataplex Datascan Onetime Documentation * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const tfDataplexTestDataset = new gcp.bigquery.Dataset("tf_dataplex_test_dataset", { * datasetId: "tf_dataplex_test_dataset_id__89313", * defaultTableExpirationMs: 3600000, * }); * const tfDataplexTestTable = new gcp.bigquery.Table("tf_dataplex_test_table", { * datasetId: tfDataplexTestDataset.datasetId, * tableId: "tf_dataplex_test_table_id__60646", * deletionProtection: false, * schema: ` [ * { * \\"name\\": \\"name\\", * \\"type\\": \\"STRING\\", * \\"mode\\": \\"NULLABLE\\" * }, * { * \\"name\\": \\"station_id\\", * \\"type\\": \\"INTEGER\\", * \\"mode\\": \\"NULLABLE\\", * \\"description\\": \\"The id of the bike station\\" * }, * { * \\"name\\": \\"address\\", * \\"type\\": \\"STRING\\", * \\"mode\\": \\"NULLABLE\\", * \\"description\\": \\"The address of the bike station\\" * }, * { * \\"name\\": \\"power_type\\", * \\"type\\": \\"STRING\\", * \\"mode\\": \\"NULLABLE\\", * \\"description\\": \\"The powert type of the bike station\\" * }, * { * \\"name\\": \\"property_type\\", * \\"type\\": \\"STRING\\", * \\"mode\\": \\"NULLABLE\\", * \\"description\\": \\"The type of the property\\" * }, * { * \\"name\\": \\"number_of_docks\\", * \\"type\\": \\"INTEGER\\", * \\"mode\\": \\"NULLABLE\\", * \\"description\\": \\"The number of docks the property have\\" * }, * { * \\"name\\": \\"footprint_length\\", * \\"type\\": \\"INTEGER\\", * \\"mode\\": \\"NULLABLE\\", * \\"description\\": \\"The footpring lenght of the property\\" * }, * { * \\"name\\": \\"council_district\\", * \\"type\\": \\"INTEGER\\", * \\"mode\\": \\"NULLABLE\\", * \\"description\\": \\"The council district the property is in\\" * } * ] * `, * }); * const onetimeDocumentation = new gcp.dataplex.Datascan("onetime_documentation", { * location: "us-central1", * dataScanId: "datadocumentation-onetime", * data: { * resource: pulumi.interpolate`//bigquery.googleapis.com/projects/my-project-name/datasets/${tfDataplexTestDataset.datasetId}/tables/${tfDataplexTestTable.tableId}`, * }, * executionSpec: { * trigger: { * oneTime: { * ttlAfterScanCompletion: "120s", * }, * }, * }, * dataDocumentationSpec: {}, * project: "my-project-name", * }); * ``` * ### Dataplex Datascan Execution Identity User Credential * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const tfTestDataset = new gcp.bigquery.Dataset("tf_test_dataset", { * datasetId: "tf_test_ds__9394", * defaultTableExpirationMs: 3600000, * deleteContentsOnDestroy: true, * project: "my-project-name", * }); * const tfTestTable = new gcp.bigquery.Table("tf_test_table", { * datasetId: tfTestDataset.datasetId, * tableId: "tf_test_tbl__11380", * deletionProtection: false, * project: "my-project-name", * schema: ` [ * { * \\"name\\": \\"word\\", * \\"type\\": \\"STRING\\", * \\"mode\\": \\"REQUIRED\\" * }, * { * \\"name\\": \\"word_count\\", * \\"type\\": \\"INTEGER\\", * \\"mode\\": \\"REQUIRED\\" * } * ] * `, * }); * const identityUserCredential = new gcp.dataplex.Datascan("identity_user_credential", { * location: "us-central1", * dataScanId: "dataplex-id-user-cred", * data: { * resource: pulumi.interpolate`//bigquery.googleapis.com/projects/my-project-name/datasets/${tfTestDataset.datasetId}/tables/${tfTestTable.tableId}`, * }, * executionSpec: { * trigger: { * oneTime: {}, * }, * }, * executionIdentity: { * userCredential: {}, * }, * dataProfileSpec: {}, * project: "my-project-name", * }, { * dependsOn: [tfTestTable], * }); * ``` * ### Dataplex Datascan Execution Identity Service Account * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * import * as time from "@pulumiverse/time"; * * const project = gcp.organizations.getProject({ * projectId: "my-project-name", * }); * const sa = new gcp.serviceaccount.Account("sa", { * accountId: "tf-test-sa-_35305", * displayName: "DataScan Service Account", * project: "my-project-name", * }); * const dataplexSaImpersonate = new gcp.serviceaccount.IAMMember("dataplex_sa_impersonate", { * serviceAccountId: sa.name, * role: "roles/iam.serviceAccountTokenCreator", * member: project.then(project => `serviceAccount:service-${project.number}@gcp-sa-dataplex.iam.gserviceaccount.com`), * }); * const wait120Seconds = new time.Sleep("wait_120_seconds", {createDuration: "120s"}, { * dependsOn: [dataplexSaImpersonate], * }); * const saBqDataViewer = new gcp.projects.IAMMember("sa_bq_data_viewer", { * project: "my-project-name", * role: "roles/bigquery.dataViewer", * member: pulumi.interpolate`serviceAccount:${sa.email}`, * }); * const saBqJobUser = new gcp.projects.IAMMember("sa_bq_job_user", { * project: "my-project-name", * role: "roles/bigquery.jobUser", * member: pulumi.interpolate`serviceAccount:${sa.email}`, * }); * const tfTestDataset = new gcp.bigquery.Dataset("tf_test_dataset", { * datasetId: "tf_test_ds__62793", * defaultTableExpirationMs: 3600000, * deleteContentsOnDestroy: true, * project: "my-project-name", * }, { * dependsOn: [ * dataplexSaImpersonate, * saBqDataViewer, * saBqJobUser, * ], * }); * const tfTestTable = new gcp.bigquery.Table("tf_test_table", { * datasetId: tfTestDataset.datasetId, * tableId: "tf_test_tbl__55438", * deletionProtection: false, * project: "my-project-name", * schema: ` [ * { * \\"name\\": \\"word\\", * \\"type\\": \\"STRING\\", * \\"mode\\": \\"REQUIRED\\" * }, * { * \\"name\\": \\"word_count\\", * \\"type\\": \\"INTEGER\\", * \\"mode\\": \\"REQUIRED\\" * } * ] * `, * }); * const identityServiceAccount = new gcp.dataplex.Datascan("identity_service_account", { * location: "us-central1", * dataScanId: "dataplex-id-sa", * data: { * resource: pulumi.interpolate`//bigquery.googleapis.com/projects/my-project-name/datasets/${tfTestDataset.datasetId}/tables/${tfTestTable.tableId}`, * }, * executionSpec: { * trigger: { * onDemand: {}, * }, * }, * executionIdentity: { * serviceAccount: { * email: sa.email, * }, * }, * dataProfileSpec: {}, * project: "my-project-name", * }, { * dependsOn: [ * tfTestTable, * wait120Seconds, * ], * }); * ``` * ### Dataplex Datascan Quality Reusable Rules Catalog Based * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * import * as time from "@pulumiverse/time"; * * const project = gcp.organizations.getProject({ * projectId: "my-project-name", * }); * const sa = new gcp.serviceaccount.Account("sa", { * accountId: "tf-test-sa-_32706", * displayName: "DataScan Service Account", * project: "my-project-name", * }); * const dataplexSaImpersonate = new gcp.serviceaccount.IAMMember("dataplex_sa_impersonate", { * serviceAccountId: sa.name, * role: "roles/iam.serviceAccountTokenCreator", * member: project.then(project => `serviceAccount:service-${project.number}@gcp-sa-dataplex.iam.gserviceaccount.com`), * }); * const saBqDataViewer = new gcp.projects.IAMMember("sa_bq_data_viewer", { * project: "my-project-name", * role: "roles/bigquery.dataViewer", * member: pulumi.interpolate`serviceAccount:${sa.email}`, * }); * const saBqJobUser = new gcp.projects.IAMMember("sa_bq_job_user", { * project: "my-project-name", * role: "roles/bigquery.jobUser", * member: pulumi.interpolate`serviceAccount:${sa.email}`, * }); * const tfTestDataset = new gcp.bigquery.Dataset("tf_test_dataset", { * datasetId: "tf_test_dataset_id__49082", * defaultTableExpirationMs: 3600000, * deleteContentsOnDestroy: true, * project: "my-project-name", * location: "us-central1", * }, { * dependsOn: [ * dataplexSaImpersonate, * saBqDataViewer, * saBqJobUser, * ], * }); * const tfTestTable = new gcp.bigquery.Table("tf_test_table", { * datasetId: tfTestDataset.datasetId, * tableId: "tf_test_table_id__60365", * deletionProtection: false, * project: "my-project-name", * schema: ` [ * { * "name": "name", * "type": "STRING", * "mode": "NULLABLE" * } * ] * `, * }); * const testGroup = new gcp.dataplex.EntryGroup("test_group", { * location: "us-central1", * entryGroupId: "test-group-_80215", * project: "my-project-name", * }); * const testEntry = new gcp.dataplex.Entry("test_entry", { * location: "us-central1", * entryGroupId: testGroup.entryGroupId, * entryId: "test-entry-_59033", * entryType: "projects/655216118709/locations/global/entryTypes/data-quality-rule-template", * project: project.then(project => project.number), * aspects: [{ * aspectKey: "655216118709.global.data-quality-rule-template", * aspect: { * data: JSON.stringify({ * dimension: "VALIDITY", * sqlCollection: [{ * query: "SELECT * FROM ${param(table_name)} WHERE ${param(column_name)} IS NULL", * }], * inputParameters: { * table_name: { * description: "Table Name", * }, * column_name: { * description: "Column Name", * }, * }, * }), * }, * }], * }); * const waitForBqSync = new time.Sleep("wait_for_bq_sync", {createDuration: "300s"}, { * dependsOn: [tfTestTable], * }); * const bqTableEntry = new gcp.dataplex.Entry("bq_table_entry", { * entryGroupId: "@bigquery", * project: project.then(project => project.projectId), * location: "us-central1", * entryId: pulumi.all([project, tfTestDataset.datasetId, tfTestTable.tableId]).apply(([project, datasetId, tableId]) => `bigquery.googleapis.com/projects/${project.projectId}/datasets/${datasetId}/tables/${tableId}`), * entryType: "projects/655216118709/locations/global/entryTypes/bigquery-table", * fullyQualifiedName: pulumi.all([project, tfTestDataset.datasetId, tfTestTable.tableId]).apply(([project, datasetId, tableId]) => `bigquery:${project.projectId}.${datasetId}.${tableId}`), * parentEntry: pulumi.all([project, project, tfTestDataset.datasetId]).apply(([project, project1, datasetId]) => `projects/${project.projectId}/locations/us-central1/entryGroups/@bigquery/entries/bigquery.googleapis.com/projects/${project1.projectId}/datasets/${datasetId}`), * aspects: [{ * aspectKey: "655216118709.global.data-rules@Schema.name", * aspect: { * data: pulumi.jsonStringify({ * rules: [ * { * name: "rule-to-filter-out", * dimension: "VALIDITY", * type: "TEMPLATE_REFERENCE", * templateReference: { * name: testEntry.name, * values: { * table_name: { * value: pulumi.all([project, tfTestDataset.datasetId, tfTestTable.tableId]).apply(([project, datasetId, tableId]) => ``${project.projectId}.${datasetId}.${tableId}``), * }, * column_name: { * value: "name", * }, * }, * }, * attributes: { * priority: "low", * }, * }, * { * name: "non-null-check-name-manual", * dimension: "VALIDITY", * type: "TEMPLATE_REFERENCE", * templateReference: { * name: testEntry.name, * values: { * table_name: { * value: pulumi.all([project, tfTestDataset.datasetId, tfTestTable.tableId]).apply(([project, datasetId, tableId]) => ``${project.projectId}.${datasetId}.${tableId}``), * }, * column_name: { * value: "name", * }, * }, * }, * attributes: { * priority: "high", * }, * }, * ], * }), * }, * }], * }, { * dependsOn: [ * waitForBqSync, * testEntry, * ], * }); * const waitForAspectPropagation = new time.Sleep("wait_for_aspect_propagation", {createDuration: "300s"}, { * dependsOn: [bqTableEntry], * }); * const reusableRulesCatalogBased = new gcp.dataplex.Datascan("reusable_rules_catalog_based", { * location: "us-central1", * dataScanId: "dataquality-catalog", * displayName: "Catalog Datascan Quality", * description: "Example resource - Catalog Datascan Quality", * data: { * resource: pulumi.all([project, tfTestDataset.datasetId, tfTestTable.tableId]).apply(([project, datasetId, tableId]) => `//bigquery.googleapis.com/projects/${project.projectId}/datasets/${datasetId}/tables/${tableId}`), * }, * executionSpec: { * trigger: { * onDemand: {}, * }, * }, * executionIdentity: { * serviceAccount: { * email: sa.email, * }, * }, * dataQualitySpec: { * enableCatalogBasedRules: true, * filter: "attributes.priority = \"high\"", * }, * project: project.then(project => project.projectId), * }, { * dependsOn: [waitForAspectPropagation], * }); * ``` * ### Dataplex Datascan Data Quality Template Reference * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * import * as time from "@pulumiverse/time"; * * const project = gcp.organizations.getProject({ * projectId: "my-project-name", * }); * const sa = new gcp.serviceaccount.Account("sa", { * accountId: "tf-test-sa-_32081", * displayName: "DataScan Service Account", * project: project.then(project => project.projectId), * }); * const dataplexSaImpersonate = new gcp.serviceaccount.IAMMember("dataplex_sa_impersonate", { * serviceAccountId: sa.name, * role: "roles/iam.serviceAccountTokenCreator", * member: project.then(project => `serviceAccount:service-${project.number}@gcp-sa-dataplex.iam.gserviceaccount.com`), * }); * const wait120Seconds = new time.Sleep("wait_120_seconds", {createDuration: "120s"}, { * dependsOn: [dataplexSaImpersonate], * }); * const saBqDataViewer = new gcp.projects.IAMMember("sa_bq_data_viewer", { * project: project.then(project => project.projectId), * role: "roles/bigquery.dataViewer", * member: pulumi.interpolate`serviceAccount:${sa.email}`, * }); * const saBqJobUser = new gcp.projects.IAMMember("sa_bq_job_user", { * project: project.then(project => project.projectId), * role: "roles/bigquery.jobUser", * member: pulumi.interpolate`serviceAccount:${sa.email}`, * }); * const testGroup = new gcp.dataplex.EntryGroup("test_group", { * location: "us-central1", * entryGroupId: "test-group-_10393", * project: project.then(project => project.projectId), * }); * const testEntry = new gcp.dataplex.Entry("test_entry", { * location: "us-central1", * entryGroupId: testGroup.entryGroupId, * entryId: "test-entry-_33052", * entryType: "projects/655216118709/locations/global/entryTypes/data-quality-rule-template", * project: project.then(project => project.number), * aspects: [{ * aspectKey: "655216118709.global.data-quality-rule-template", * aspect: { * data: JSON.stringify({ * dimension: "VALIDITY", * sqlCollection: [{ * query: "SELECT * FROM ${data()} WHERE ${column()} IS NOT NULL", * }], * }), * }, * }], * }); * const tfTestDataset = new gcp.bigquery.Dataset("tf_test_dataset", { * datasetId: "tf_test_dataset_id__3684", * defaultTableExpirationMs: 3600000, * location: "us-central1", * project: project.then(project => project.projectId), * }, { * dependsOn: [ * dataplexSaImpersonate, * saBqDataViewer, * saBqJobUser, * ], * }); * const tfTestTable = new gcp.bigquery.Table("tf_test_table", { * datasetId: tfTestDataset.datasetId, * tableId: "tf_test_table_id__10719", * deletionProtection: false, * project: project.then(project => project.projectId), * schema: ` [ * { * \\"name\\": \\"name\\", * \\"type\\": \\"STRING\\", * \\"mode\\": \\"NULLABLE\\" * } * ] * `, * }); * const dataQualityTemplateReference = new gcp.dataplex.Datascan("data_quality_template_reference", { * location: "us-central1", * displayName: "Data Quality Template Reference", * dataScanId: "dataquality-template", * data: { * resource: pulumi.all([project, tfTestDataset.datasetId, tfTestTable.tableId]).apply(([project, datasetId, tableId]) => `//bigquery.googleapis.com/projects/${project.projectId}/datasets/${datasetId}/tables/${tableId}`), * }, * executionSpec: { * trigger: { * onDemand: {}, * }, * }, * executionIdentity: { * serviceAccount: { * email: sa.email, * }, * }, * dataQualitySpec: { * rules: [{ * column: "name", * dimension: "VALIDITY", * templateReference: { * name: testEntry.name, * values: [{ * name: "min_length", * value: "10", * }], * }, * }], * }, * project: project.then(project => project.projectId), * }, { * dependsOn: [ * tfTestTable, * wait120Seconds, * ], * }); * ``` * * ## Import * * Datascan can be imported using any of these accepted formats: * * * `projects/{{project}}/locations/{{location}}/dataScans/{{data_scan_id}}` * * `{{project}}/{{location}}/{{data_scan_id}}` * * `{{location}}/{{data_scan_id}}` * * `{{data_scan_id}}` * * When using the `pulumi import` command, Datascan can be imported using one of the formats above. For example: * * ```sh * $ pulumi import gcp:dataplex/datascan:Datascan default projects/{{project}}/locations/{{location}}/dataScans/{{data_scan_id}} * $ pulumi import gcp:dataplex/datascan:Datascan default {{project}}/{{location}}/{{data_scan_id}} * $ pulumi import gcp:dataplex/datascan:Datascan default {{location}}/{{data_scan_id}} * $ pulumi import gcp:dataplex/datascan:Datascan default {{data_scan_id}} * ``` */ export declare class Datascan extends pulumi.CustomResource { /** * Get an existing Datascan resource's state with the given name, ID, and optional extra * properties used to qualify the lookup. * * @param name The _unique_ name of the resulting resource. * @param id The _unique_ provider ID of the resource to lookup. * @param state Any extra arguments used during the lookup. * @param opts Optional settings to control the behavior of the CustomResource. */ static get(name: string, id: pulumi.Input<pulumi.ID>, state?: DatascanState, opts?: pulumi.CustomResourceOptions): Datascan; /** * Returns true if the given object is an instance of Datascan. This is designed to work even * when multiple copies of the Pulumi SDK have been loaded into the same process. */ static isInstance(obj: any): obj is Datascan; /** * The time when the scan was created. */ readonly createTime: pulumi.Output<string>; /** * The data source for DataScan. * Structure is documented below. */ readonly data: pulumi.Output<outputs.dataplex.DatascanData>; /** * DataDiscoveryScan related setting. * Structure is documented below. */ readonly dataDiscoverySpec: pulumi.Output<outputs.dataplex.DatascanDataDiscoverySpec | undefined>; /** * DataDocumentationScan related setting. */ readonly dataDocumentationSpec: pulumi.Output<outputs.dataplex.DatascanDataDocumentationSpec | undefined>; /** * DataProfileScan related setting. * Structure is documented below. */ readonly dataProfileSpec: pulumi.Output<outputs.dataplex.DatascanDataProfileSpec | undefined>; /** * DataQualityScan related setting. * Structure is documented below. */ readonly dataQualitySpec: pulumi.Output<outputs.dataplex.DatascanDataQualitySpec | undefined>; /** * DataScan identifier. Must contain only lowercase letters, numbers and hyphens. Must start with a letter. Must end with a number or a letter. */ readonly dataScanId: pulumi.Output<string>; /** * Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. * When a 'terraform destroy' or 'pulumi up' would delete the resource, * the command will fail if this field is set to "PREVENT" in Terraform state. * When set to "ABANDON", the command will remove the resource from Terraform * management without updating or deleting the resource in the API. * When set to "DELETE", deleting the resource is allowed. */ readonly deletionPolicy: pulumi.Output<string>; /** * Description of the scan. */ readonly description: pulumi.Output<string | undefined>; /** * User friendly display name. */ readonly displayName: pulumi.Output<string | undefined>; /** * All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services. */ readonly effectiveLabels: pulumi.Output<{ [key: string]: string; }>; /** * The identity to run the datascan. If not specified, defaults to the Dataplex Service Agent. * Structure is documented below. */ readonly executionIdentity: pulumi.Output<outputs.dataplex.DatascanExecutionIdentity | undefined>; /** * DataScan execution settings. * Structure is documented below. */ readonly executionSpec: pulumi.Output<outputs.dataplex.DatascanExecutionSpec>; /** * Status of the data scan execution. * Structure is documented below. */ readonly executionStatuses: pulumi.Output<outputs.dataplex.DatascanExecutionStatus[]>; /** * User-defined labels for the scan. A list of key->value pairs. * * **Note**: This field is non-authoritative, and will only manage the labels present in your configuration. * Please refer to the field `effectiveLabels` for all of the labels present on the resource. */ readonly labels: pulumi.Output<{ [key: string]: string; } | undefined>; /** * The location where the data scan should reside. */ readonly location: pulumi.Output<string>; /** * The relative resource name of the scan, of the form: projects/{project}/locations/{locationId}/dataScans/{datascan_id}, where project refers to a projectId or projectNumber and locationId refers to a GCP region. */ readonly name: pulumi.Output<string>; /** * The ID of the project in which the resource belongs. * If it is not provided, the provider project is used. */ readonly project: pulumi.Output<string>; /** * The combination of labels configured directly on the resource * and default labels configured on the provider. */ readonly pulumiLabels: pulumi.Output<{ [key: string]: string; }>; /** * Current state of the DataScan. */ readonly state: pulumi.Output<string>; /** * The type of DataScan. */ readonly type: pulumi.Output<string>; /** * System generated globally unique ID for the scan. This ID will be different if the scan is deleted and re-created with the same name. */ readonly uid: pulumi.Output<string>; /** * The time when the scan was last updated. */ readonly updateTime: pulumi.Output<string>; /** * Create a Datascan resource with the given unique name, arguments, and options. * * @param name The _unique_ name of the resource. * @param args The arguments to use to populate this resource's properties. * @param opts A bag of options that control this resource's behavior. */ constructor(name: string, args: DatascanArgs, opts?: pulumi.CustomResourceOptions); } /** * Input properties used for looking up and filtering Datascan resources. */ export interface DatascanState { /** * The time when the scan was created. */ createTime?: pulumi.Input<string | undefined>; /** * The data source for DataScan. * Structure is documented below. */ data?: pulumi.Input<inputs.dataplex.DatascanData | undefined>; /** * DataDiscoveryScan related setting. * Structure is documented below. */ dataDiscoverySpec?: pulumi.Input<inputs.dataplex.DatascanDataDiscoverySpec | undefined>; /** * DataDocumentationScan related setting. */ dataDocumentationSpec?: pulumi.Input<inputs.dataplex.DatascanDataDocumentationSpec | undefined>; /** * DataProfileScan related setting. * Structure is documented below. */ dataProfileSpec?: pulumi.Input<inputs.dataplex.DatascanDataProfileSpec | undefined>; /** * DataQualityScan related setting. * Structure is documented below. */ dataQualitySpec?: pulumi.Input<inputs.dataplex.DatascanDataQualitySpec | undefined>; /** * DataScan identifier. Must contain only lowercase letters, numbers and hyphens. Must start with a letter. Must end with a number or a letter. */ dataScanId?: pulumi.Input<string | undefined>; /** * Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. * When a 'terraform destroy' or 'pulumi up' would delete the resource, * the command will fail if this field is set to "PREVENT" in Terraform state. * When set to "ABANDON", the command will remove the resource from Terraform * management without updating or deleting the resource in the API. * When set to "DELETE", deleting the resource is allowed. */ deletionPolicy?: pulumi.Input<string | undefined>; /** * Description of the scan. */ description?: pulumi.Input<string | undefined>; /** * User friendly display name. */ displayName?: pulumi.Input<string | undefined>; /** * All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services. */ effectiveLabels?: pulumi.Input<{ [key: string]: pulumi.Input<string>; } | undefined>; /** * The identity to run the datascan. If not specified, defaults to the Dataplex Service Agent. * Structure is documented below. */ executionIdentity?: pulumi.Input<inputs.dataplex.DatascanExecutionIdentity | undefined>; /** * DataScan execution settings. * Structure is documented below. */ executionSpec?: pulumi.Input<inputs.dataplex.DatascanExecutionSpec | undefined>; /** * Status of the data scan execution. * Structure is documented below. */ executionStatuses?: pulumi.Input<pulumi.Input<inputs.dataplex.DatascanExecutionStatus>[] | undefined>; /** * User-defined labels for the scan. A list of key->value pairs. * * **Note**: This field is non-authoritative, and will only manage the labels present in your configuration. * Please refer to the field `effectiveLabels` for all of the labels present on the resource. */ labels?: pulumi.Input<{ [key: string]: pulumi.Input<string>; } | undefined>; /** * The location where the data scan should reside. */ location?: pulumi.Input<string | undefined>; /** * The relative resource name of the scan, of the form: projects/{project}/locations/{locationId}/dataScans/{datascan_id}, where project refers to a projectId or projectNumber and locationId refers to a GCP region. */ name?: pulumi.Input<string | undefined>; /** * The ID of the project in which the resource belongs. * If it is not provided, the provider project is used. */ project?: pulumi.Input<string | undefined>; /** * The combination of labels configured directly on the resource * and default labels configured on the provider. */ pulumiLabels?: pulumi.Input<{ [key: string]: pulumi.Input<string>; } | undefined>; /** * Current state of the DataScan. */ state?: pulumi.Input<string | undefined>; /** * The type of DataScan. */ type?: pulumi.Input<string | undefined>; /** * System generated globally unique ID for the scan. This ID will be different if the scan is deleted and re-created with the same name. */ uid?: pulumi.Input<string | undefined>; /** * The time when the scan was last updated. */ updateTime?: pulumi.Input<string | undefined>; } /** * The set of arguments for constructing a Datascan resource. */ export interface DatascanArgs { /** * The data source for DataScan. * Structure is documented below. */ data: pulumi.Input<inputs.dataplex.DatascanData>; /** * DataDiscoveryScan related setting. * Structure is documented below. */ dataDiscoverySpec?: pulumi.Input<inputs.dataplex.DatascanDataDiscoverySpec | undefined>; /** * DataDocumentationScan related setting. */ dataDocumentationSpec?: pulumi.Input<inputs.dataplex.DatascanDataDocumentationSpec | undefined>; /** * DataProfileScan related setting. * Structure is documented below. */ dataProfileSpec?: pulumi.Input<inputs.dataplex.DatascanDataProfileSpec | undefined>; /** * DataQualityScan related setting. * Structure is documented below. */ dataQualitySpec?: pulumi.Input<inputs.dataplex.DatascanDataQualitySpec | undefined>; /** * DataScan identifier. Must contain only lowercase letters, numbers and hyphens. Must start with a letter. Must end with a number or a letter. */ dataScanId: pulumi.Input<string>; /** * Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. * When a 'terraform destroy' or 'pulumi up' would delete the resource, * the command will fail if this field is set to "PREVENT" in Terraform state. * When set to "ABANDON", the command will remove the resource from Terraform * management without updating or deleting the resource in the API. * When set to "DELETE", deleting the resource is allowed. */ deletionPolicy?: pulumi.Input<string | undefined>; /** * Description of the scan. */ description?: pulumi.Input<string | undefined>; /** * User friendly display name. */ displayName?: pulumi.Input<string | undefined>; /** * The identity to run the datascan. If not specified, defaults to the Dataplex Service Agent. * Structure is documented below. */ executionIdentity?: pulumi.Input<inputs.dataplex.DatascanExecutionIdentity | undefined>; /** * DataScan execution settings. * Structure is documented below. */ executionSpec: pulumi.Input<inputs.dataplex.DatascanExecutionSpec>; /** * User-defined labels for the scan. A list of key->value pairs. * * **Note**: This field is non-authoritative, and will only manage the labels present in your configuration. * Please refer to the field `effectiveLabels` for all of the labels present on the resource. */ labels?: pulumi.Input<{ [key: string]: pulumi.Input<string>; } | undefined>; /** * The location where the data scan should reside. */ location: pulumi.Input<string>; /** * The ID of the project in which the resource belongs. * If it is not provided, the provider project is used. */ project?: pulumi.Input<string | undefined>; } //# sourceMappingURL=datascan.d.ts.map