@pulumi/gcp
Version:
A Pulumi package for creating and managing Google Cloud Platform resources.
1,444 lines • 49.5 kB
JavaScript
"use strict";
// *** WARNING: this file was generated by pulumi-language-nodejs. ***
// *** Do not edit by hand unless you're certain you know what you are doing! ***
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.Stream = void 0;
const pulumi = __importStar(require("@pulumi/pulumi"));
const utilities = __importStar(require("../utilities"));
/**
* A resource representing streaming data from a source to a destination.
*
* To get more information about Stream, see:
*
* * [API documentation](https://cloud.google.com/datastream/docs/reference/rest/v1/projects.locations.streams)
* * How-to Guides
* * [Official Documentation](https://cloud.google.com/datastream/docs/create-a-stream)
*
* ## Example Usage
*
* ### Datastream Stream Full
*
* ```typescript
* import * as pulumi from "@pulumi/pulumi";
* import * as gcp from "@pulumi/gcp";
* import * as random from "@pulumi/random";
*
* const project = gcp.organizations.getProject({});
* const instance = new gcp.sql.DatabaseInstance("instance", {
* name: "my-instance",
* databaseVersion: "MYSQL_8_0",
* region: "us-central1",
* settings: {
* tier: "db-f1-micro",
* backupConfiguration: {
* enabled: true,
* binaryLogEnabled: true,
* },
* ipConfiguration: {
* authorizedNetworks: [
* {
* value: "34.71.242.81",
* },
* {
* value: "34.72.28.29",
* },
* {
* value: "34.67.6.157",
* },
* {
* value: "34.67.234.134",
* },
* {
* value: "34.72.239.218",
* },
* ],
* },
* },
* deletionProtection: true,
* });
* const db = new gcp.sql.Database("db", {
* instance: instance.name,
* name: "db",
* });
* const pwd = new random.index.Password("pwd", {
* length: 16,
* special: false,
* });
* const user = new gcp.sql.User("user", {
* name: "user",
* instance: instance.name,
* host: "%",
* password: pwd.result,
* });
* const sourceConnectionProfile = new gcp.datastream.ConnectionProfile("source_connection_profile", {
* displayName: "Source connection profile",
* location: "us-central1",
* connectionProfileId: "source-profile",
* mysqlProfile: {
* hostname: instance.publicIpAddress,
* username: user.name,
* password: user.password,
* },
* });
* const bucket = new gcp.storage.Bucket("bucket", {
* name: "my-bucket",
* location: "US",
* uniformBucketLevelAccess: true,
* });
* const viewer = new gcp.storage.BucketIAMMember("viewer", {
* bucket: bucket.name,
* role: "roles/storage.objectViewer",
* member: project.then(project => `serviceAccount:service-${project.number}@gcp-sa-datastream.iam.gserviceaccount.com`),
* });
* const creator = new gcp.storage.BucketIAMMember("creator", {
* bucket: bucket.name,
* role: "roles/storage.objectCreator",
* member: project.then(project => `serviceAccount:service-${project.number}@gcp-sa-datastream.iam.gserviceaccount.com`),
* });
* const reader = new gcp.storage.BucketIAMMember("reader", {
* bucket: bucket.name,
* role: "roles/storage.legacyBucketReader",
* member: project.then(project => `serviceAccount:service-${project.number}@gcp-sa-datastream.iam.gserviceaccount.com`),
* });
* const keyUser = new gcp.kms.CryptoKeyIAMMember("key_user", {
* cryptoKeyId: "kms-name",
* role: "roles/cloudkms.cryptoKeyEncrypterDecrypter",
* member: project.then(project => `serviceAccount:service-${project.number}@gcp-sa-datastream.iam.gserviceaccount.com`),
* });
* const destinationConnectionProfile = new gcp.datastream.ConnectionProfile("destination_connection_profile", {
* displayName: "Connection profile",
* location: "us-central1",
* connectionProfileId: "destination-profile",
* gcsProfile: {
* bucket: bucket.name,
* rootPath: "/path",
* },
* });
* const _default = new gcp.datastream.Stream("default", {
* streamId: "my-stream",
* desiredState: "NOT_STARTED",
* location: "us-central1",
* displayName: "my stream",
* labels: {
* key: "value",
* },
* sourceConfig: {
* sourceConnectionProfile: sourceConnectionProfile.id,
* mysqlSourceConfig: {
* includeObjects: {
* mysqlDatabases: [{
* database: "my-database",
* mysqlTables: [
* {
* table: "includedTable",
* mysqlColumns: [{
* column: "includedColumn",
* dataType: "VARCHAR",
* collation: "utf8mb4",
* primaryKey: false,
* nullable: false,
* ordinalPosition: 0,
* }],
* },
* {
* table: "includedTable_2",
* },
* ],
* }],
* },
* excludeObjects: {
* mysqlDatabases: [{
* database: "my-database",
* mysqlTables: [{
* table: "excludedTable",
* mysqlColumns: [{
* column: "excludedColumn",
* dataType: "VARCHAR",
* collation: "utf8mb4",
* primaryKey: false,
* nullable: false,
* ordinalPosition: 0,
* }],
* }],
* }],
* },
* maxConcurrentCdcTasks: 5,
* },
* },
* destinationConfig: {
* destinationConnectionProfile: destinationConnectionProfile.id,
* gcsDestinationConfig: {
* path: "mydata",
* fileRotationMb: 200,
* fileRotationInterval: "60s",
* jsonFileFormat: {
* schemaFileFormat: "NO_SCHEMA_FILE",
* compression: "GZIP",
* },
* },
* },
* backfillAll: {
* mysqlExcludedObjects: {
* mysqlDatabases: [{
* database: "my-database",
* mysqlTables: [{
* table: "excludedTable",
* mysqlColumns: [{
* column: "excludedColumn",
* dataType: "VARCHAR",
* collation: "utf8mb4",
* primaryKey: false,
* nullable: false,
* ordinalPosition: 0,
* }],
* }],
* }],
* },
* },
* customerManagedEncryptionKey: "kms-name",
* }, {
* dependsOn: [keyUser],
* });
* ```
* ### Datastream Stream Postgresql
*
* ```typescript
* import * as pulumi from "@pulumi/pulumi";
* import * as gcp from "@pulumi/gcp";
*
* const source = new gcp.datastream.ConnectionProfile("source", {
* displayName: "Postgresql Source",
* location: "us-central1",
* connectionProfileId: "source-profile",
* postgresqlProfile: {
* hostname: "hostname",
* port: 5432,
* username: "user",
* password: "pass",
* database: "postgres",
* },
* });
* const destination = new gcp.datastream.ConnectionProfile("destination", {
* displayName: "BigQuery Destination",
* location: "us-central1",
* connectionProfileId: "destination-profile",
* bigqueryProfile: {},
* });
* const _default = new gcp.datastream.Stream("default", {
* displayName: "Postgres to BigQuery",
* location: "us-central1",
* streamId: "my-stream",
* desiredState: "RUNNING",
* sourceConfig: {
* sourceConnectionProfile: source.id,
* postgresqlSourceConfig: {
* maxConcurrentBackfillTasks: 12,
* publication: "publication",
* replicationSlot: "replication_slot",
* includeObjects: {
* postgresqlSchemas: [{
* schema: "schema",
* postgresqlTables: [{
* table: "table",
* postgresqlColumns: [{
* column: "column",
* }],
* }],
* }],
* },
* excludeObjects: {
* postgresqlSchemas: [{
* schema: "schema",
* postgresqlTables: [{
* table: "table",
* postgresqlColumns: [{
* column: "column",
* }],
* }],
* }],
* },
* },
* },
* destinationConfig: {
* destinationConnectionProfile: destination.id,
* bigqueryDestinationConfig: {
* dataFreshness: "900s",
* sourceHierarchyDatasets: {
* datasetTemplate: {
* location: "us-central1",
* },
* },
* },
* },
* backfillAll: {
* postgresqlExcludedObjects: {
* postgresqlSchemas: [{
* schema: "schema",
* postgresqlTables: [{
* table: "table",
* postgresqlColumns: [{
* column: "column",
* }],
* }],
* }],
* },
* },
* });
* ```
* ### Datastream Stream Oracle
*
* ```typescript
* import * as pulumi from "@pulumi/pulumi";
* import * as gcp from "@pulumi/gcp";
*
* const source = new gcp.datastream.ConnectionProfile("source", {
* displayName: "Oracle Source",
* location: "us-central1",
* connectionProfileId: "source-profile",
* oracleProfile: {
* hostname: "hostname",
* port: 1521,
* username: "user",
* password: "pass",
* databaseService: "ORCL",
* },
* });
* const destination = new gcp.datastream.ConnectionProfile("destination", {
* displayName: "BigQuery Destination",
* location: "us-central1",
* connectionProfileId: "destination-profile",
* bigqueryProfile: {},
* });
* const stream5 = new gcp.datastream.Stream("stream5", {
* displayName: "Oracle to BigQuery",
* location: "us-central1",
* streamId: "my-stream",
* desiredState: "RUNNING",
* sourceConfig: {
* sourceConnectionProfile: source.id,
* oracleSourceConfig: {
* maxConcurrentCdcTasks: 8,
* maxConcurrentBackfillTasks: 12,
* includeObjects: {
* oracleSchemas: [{
* schema: "schema",
* oracleTables: [{
* table: "table",
* oracleColumns: [{
* column: "column",
* }],
* }],
* }],
* },
* excludeObjects: {
* oracleSchemas: [{
* schema: "schema",
* oracleTables: [{
* table: "table",
* oracleColumns: [{
* column: "column",
* }],
* }],
* }],
* },
* dropLargeObjects: {},
* },
* },
* destinationConfig: {
* destinationConnectionProfile: destination.id,
* bigqueryDestinationConfig: {
* dataFreshness: "900s",
* sourceHierarchyDatasets: {
* datasetTemplate: {
* location: "us-central1",
* },
* },
* },
* },
* backfillAll: {
* oracleExcludedObjects: {
* oracleSchemas: [{
* schema: "schema",
* oracleTables: [{
* table: "table",
* oracleColumns: [{
* column: "column",
* }],
* }],
* }],
* },
* },
* });
* ```
* ### Datastream Stream Sql Server
*
* ```typescript
* import * as pulumi from "@pulumi/pulumi";
* import * as gcp from "@pulumi/gcp";
*
* const instance = new gcp.sql.DatabaseInstance("instance", {
* name: "sql-server",
* databaseVersion: "SQLSERVER_2022_STANDARD",
* region: "us-central1",
* rootPassword: "root-password",
* deletionProtection: true,
* settings: {
* tier: "db-custom-2-4096",
* ipConfiguration: {
* authorizedNetworks: [
* {
* value: "34.71.242.81",
* },
* {
* value: "34.72.28.29",
* },
* {
* value: "34.67.6.157",
* },
* {
* value: "34.67.234.134",
* },
* {
* value: "34.72.239.218",
* },
* ],
* },
* },
* });
* const user = new gcp.sql.User("user", {
* name: "user",
* instance: instance.name,
* password: "password",
* });
* const db = new gcp.sql.Database("db", {
* name: "db",
* instance: instance.name,
* }, {
* dependsOn: [user],
* });
* const source = new gcp.datastream.ConnectionProfile("source", {
* displayName: "SQL Server Source",
* location: "us-central1",
* connectionProfileId: "source-profile",
* sqlServerProfile: {
* hostname: instance.publicIpAddress,
* port: 1433,
* username: user.name,
* password: user.password,
* database: db.name,
* },
* });
* const destination = new gcp.datastream.ConnectionProfile("destination", {
* displayName: "BigQuery Destination",
* location: "us-central1",
* connectionProfileId: "destination-profile",
* bigqueryProfile: {},
* });
* const _default = new gcp.datastream.Stream("default", {
* displayName: "SQL Server to BigQuery",
* location: "us-central1",
* streamId: "stream",
* sourceConfig: {
* sourceConnectionProfile: source.id,
* sqlServerSourceConfig: {
* includeObjects: {
* schemas: [{
* schema: "schema",
* tables: [{
* table: "table",
* }],
* }],
* },
* transactionLogs: {},
* },
* },
* destinationConfig: {
* destinationConnectionProfile: destination.id,
* bigqueryDestinationConfig: {
* dataFreshness: "900s",
* sourceHierarchyDatasets: {
* datasetTemplate: {
* location: "us-central1",
* },
* },
* },
* },
* backfillNone: {},
* });
* ```
* ### Datastream Stream Sql Server Change Tables
*
* ```typescript
* import * as pulumi from "@pulumi/pulumi";
* import * as gcp from "@pulumi/gcp";
*
* const instance = new gcp.sql.DatabaseInstance("instance", {
* name: "sql-server",
* databaseVersion: "SQLSERVER_2022_STANDARD",
* region: "us-central1",
* rootPassword: "root-password",
* deletionProtection: true,
* settings: {
* tier: "db-custom-2-4096",
* ipConfiguration: {
* authorizedNetworks: [
* {
* value: "34.71.242.81",
* },
* {
* value: "34.72.28.29",
* },
* {
* value: "34.67.6.157",
* },
* {
* value: "34.67.234.134",
* },
* {
* value: "34.72.239.218",
* },
* ],
* },
* },
* });
* const user = new gcp.sql.User("user", {
* name: "user",
* instance: instance.name,
* password: "password",
* });
* const db = new gcp.sql.Database("db", {
* name: "db",
* instance: instance.name,
* }, {
* dependsOn: [user],
* });
* const source = new gcp.datastream.ConnectionProfile("source", {
* displayName: "SQL Server Source",
* location: "us-central1",
* connectionProfileId: "source-profile",
* sqlServerProfile: {
* hostname: instance.publicIpAddress,
* port: 1433,
* username: user.name,
* password: user.password,
* database: db.name,
* },
* });
* const destination = new gcp.datastream.ConnectionProfile("destination", {
* displayName: "BigQuery Destination",
* location: "us-central1",
* connectionProfileId: "destination-profile",
* bigqueryProfile: {},
* });
* const _default = new gcp.datastream.Stream("default", {
* displayName: "SQL Server to BigQuery",
* location: "us-central1",
* streamId: "stream",
* sourceConfig: {
* sourceConnectionProfile: source.id,
* sqlServerSourceConfig: {
* includeObjects: {
* schemas: [{
* schema: "schema",
* tables: [{
* table: "table",
* }],
* }],
* },
* changeTables: {},
* },
* },
* destinationConfig: {
* destinationConnectionProfile: destination.id,
* bigqueryDestinationConfig: {
* dataFreshness: "900s",
* sourceHierarchyDatasets: {
* datasetTemplate: {
* location: "us-central1",
* },
* },
* },
* },
* backfillNone: {},
* });
* ```
* ### Datastream Stream Mysql Gtid
*
* ```typescript
* import * as pulumi from "@pulumi/pulumi";
* import * as gcp from "@pulumi/gcp";
*
* const instance = new gcp.sql.DatabaseInstance("instance", {
* name: "<%= ctx[:vars]['mysql_name'] %>",
* databaseVersion: "MYSQL_8_0",
* region: "us-central1",
* rootPassword: "<%= ctx[:vars]['mysql_root_password'] %>",
* deletionProtection: "<%= ctx[:vars]['deletion_protection'] %>" === "true",
* settings: {
* tier: "db-custom-2-4096",
* ipConfiguration: {
* authorizedNetworks: [
* {
* value: "34.71.242.81",
* },
* {
* value: "34.72.28.29",
* },
* {
* value: "34.67.6.157",
* },
* {
* value: "34.67.234.134",
* },
* {
* value: "34.72.239.218",
* },
* ],
* },
* },
* });
* const user = new gcp.sql.User("user", {
* name: "<%= ctx[:vars]['database_user'] %>",
* instance: instance.name,
* password: "<%= ctx[:vars]['database_password'] %>",
* });
* const db = new gcp.sql.Database("db", {
* name: "<%= ctx[:vars]['database_name'] %>",
* instance: instance.name,
* }, {
* dependsOn: [user],
* });
* const source = new gcp.datastream.ConnectionProfile("source", {
* displayName: "MySQL Source",
* location: "us-central1",
* connectionProfileId: "<%= ctx[:vars]['source_connection_profile_id'] %>",
* mysqlProfile: {
* hostname: instance.publicIpAddress,
* port: 1433,
* username: user.name,
* password: user.password,
* database: db.name,
* },
* });
* const destination = new gcp.datastream.ConnectionProfile("destination", {
* displayName: "BigQuery Destination",
* location: "us-central1",
* connectionProfileId: "<%= ctx[:vars]['destination_connection_profile_id'] %>",
* bigqueryProfile: {},
* });
* const _default = new gcp.datastream.Stream("default", {
* displayName: "MySQL to BigQuery",
* location: "us-central1",
* streamId: "<%= ctx[:vars]['stream_id'] %>",
* sourceConfig: {
* sourceConnectionProfile: source.id,
* mysqlSourceConfig: {
* includeObjects: {
* schemas: [{
* schema: "schema",
* tables: [{
* table: "table",
* }],
* }],
* },
* gtid: {},
* },
* },
* destinationConfig: {
* destinationConnectionProfile: destination.id,
* bigqueryDestinationConfig: {
* dataFreshness: "900s",
* sourceHierarchyDatasets: {
* datasetTemplate: {
* location: "us-central1",
* },
* },
* },
* },
* backfillNone: {},
* });
* ```
* ### Datastream Stream Postgresql Bigquery Dataset Id
*
* ```typescript
* import * as pulumi from "@pulumi/pulumi";
* import * as gcp from "@pulumi/gcp";
* import * as random from "@pulumi/random";
*
* const postgres = new gcp.bigquery.Dataset("postgres", {
* datasetId: "postgres",
* friendlyName: "postgres",
* description: "Database of postgres",
* location: "us-central1",
* });
* const destinationConnectionProfile2 = new gcp.datastream.ConnectionProfile("destination_connection_profile2", {
* displayName: "Connection profile",
* location: "us-central1",
* connectionProfileId: "dest-profile",
* bigqueryProfile: {},
* });
* const instance = new gcp.sql.DatabaseInstance("instance", {
* name: "instance-name",
* databaseVersion: "MYSQL_8_0",
* region: "us-central1",
* settings: {
* tier: "db-f1-micro",
* backupConfiguration: {
* enabled: true,
* binaryLogEnabled: true,
* },
* ipConfiguration: {
* authorizedNetworks: [
* {
* value: "34.71.242.81",
* },
* {
* value: "34.72.28.29",
* },
* {
* value: "34.67.6.157",
* },
* {
* value: "34.67.234.134",
* },
* {
* value: "34.72.239.218",
* },
* ],
* },
* },
* deletionProtection: false,
* });
* const pwd = new random.index.Password("pwd", {
* length: 16,
* special: false,
* });
* const user = new gcp.sql.User("user", {
* name: "my-user",
* instance: instance.name,
* host: "%",
* password: pwd.result,
* });
* const sourceConnectionProfile = new gcp.datastream.ConnectionProfile("source_connection_profile", {
* displayName: "Source connection profile",
* location: "us-central1",
* connectionProfileId: "source-profile",
* mysqlProfile: {
* hostname: instance.publicIpAddress,
* username: user.name,
* password: user.password,
* },
* });
* const _default = new gcp.datastream.Stream("default", {
* displayName: "postgres to bigQuery",
* location: "us-central1",
* streamId: "postgres-bigquery",
* sourceConfig: {
* sourceConnectionProfile: sourceConnectionProfile.id,
* mysqlSourceConfig: {},
* },
* destinationConfig: {
* destinationConnectionProfile: destinationConnectionProfile2.id,
* bigqueryDestinationConfig: {
* dataFreshness: "900s",
* singleTargetDataset: {
* datasetId: postgres.id,
* },
* },
* },
* backfillAll: {},
* });
* const db = new gcp.sql.Database("db", {
* instance: instance.name,
* name: "db",
* });
* ```
* ### Datastream Stream Bigquery
*
* ```typescript
* import * as pulumi from "@pulumi/pulumi";
* import * as gcp from "@pulumi/gcp";
* import * as random from "@pulumi/random";
*
* const project = gcp.organizations.getProject({});
* const instance = new gcp.sql.DatabaseInstance("instance", {
* name: "my-instance",
* databaseVersion: "MYSQL_8_0",
* region: "us-central1",
* settings: {
* tier: "db-f1-micro",
* backupConfiguration: {
* enabled: true,
* binaryLogEnabled: true,
* },
* ipConfiguration: {
* authorizedNetworks: [
* {
* value: "34.71.242.81",
* },
* {
* value: "34.72.28.29",
* },
* {
* value: "34.67.6.157",
* },
* {
* value: "34.67.234.134",
* },
* {
* value: "34.72.239.218",
* },
* ],
* },
* },
* deletionProtection: true,
* });
* const db = new gcp.sql.Database("db", {
* instance: instance.name,
* name: "db",
* });
* const pwd = new random.index.Password("pwd", {
* length: 16,
* special: false,
* });
* const user = new gcp.sql.User("user", {
* name: "user",
* instance: instance.name,
* host: "%",
* password: pwd.result,
* });
* const sourceConnectionProfile = new gcp.datastream.ConnectionProfile("source_connection_profile", {
* displayName: "Source connection profile",
* location: "us-central1",
* connectionProfileId: "source-profile",
* mysqlProfile: {
* hostname: instance.publicIpAddress,
* username: user.name,
* password: user.password,
* },
* });
* const bqSa = gcp.bigquery.getDefaultServiceAccount({});
* const bigqueryKeyUser = new gcp.kms.CryptoKeyIAMMember("bigquery_key_user", {
* cryptoKeyId: "bigquery-kms-name",
* role: "roles/cloudkms.cryptoKeyEncrypterDecrypter",
* member: bqSa.then(bqSa => `serviceAccount:${bqSa.email}`),
* });
* const destinationConnectionProfile = new gcp.datastream.ConnectionProfile("destination_connection_profile", {
* displayName: "Connection profile",
* location: "us-central1",
* connectionProfileId: "destination-profile",
* bigqueryProfile: {},
* });
* const _default = new gcp.datastream.Stream("default", {
* streamId: "my-stream",
* location: "us-central1",
* displayName: "my stream",
* sourceConfig: {
* sourceConnectionProfile: sourceConnectionProfile.id,
* mysqlSourceConfig: {},
* },
* destinationConfig: {
* destinationConnectionProfile: destinationConnectionProfile.id,
* bigqueryDestinationConfig: {
* sourceHierarchyDatasets: {
* datasetTemplate: {
* location: "us-central1",
* kmsKeyName: "bigquery-kms-name",
* },
* },
* },
* },
* backfillNone: {},
* }, {
* dependsOn: [bigqueryKeyUser],
* });
* ```
* ### Datastream Stream Bigquery Cross Project Source Hierachy
*
* ```typescript
* import * as pulumi from "@pulumi/pulumi";
* import * as gcp from "@pulumi/gcp";
* import * as random from "@pulumi/random";
* import * as time from "@pulumiverse/time";
*
* const project = gcp.organizations.getProject({});
* const cross_project_dataset = new gcp.organizations.Project("cross-project-dataset", {
* projectId: "tf-test_11171",
* name: "tf-test_40472",
* orgId: "123456789",
* billingAccount: "000000-0000000-0000000-000000",
* deletionPolicy: "DELETE",
* });
* const wait60Seconds = new time.Sleep("wait_60_seconds", {createDuration: "60s"}, {
* dependsOn: [cross_project_dataset],
* });
* const bigquery = new gcp.projects.Service("bigquery", {
* project: cross_project_dataset.projectId,
* service: "bigquery.googleapis.com",
* disableOnDestroy: false,
* }, {
* dependsOn: [wait60Seconds],
* });
* const datastreamBigqueryAdmin = new gcp.projects.IAMMember("datastream_bigquery_admin", {
* project: cross_project_dataset.projectId,
* role: "roles/bigquery.admin",
* member: project.then(project => `serviceAccount:service-${project.number}@gcp-sa-datastream.iam.gserviceaccount.com`),
* }, {
* dependsOn: [wait60Seconds],
* });
* const instance = new gcp.sql.DatabaseInstance("instance", {
* name: "my-instance",
* databaseVersion: "MYSQL_8_0",
* region: "us-central1",
* settings: {
* tier: "db-f1-micro",
* backupConfiguration: {
* enabled: true,
* binaryLogEnabled: true,
* },
* ipConfiguration: {
* authorizedNetworks: [
* {
* value: "34.71.242.81",
* },
* {
* value: "34.72.28.29",
* },
* {
* value: "34.67.6.157",
* },
* {
* value: "34.67.234.134",
* },
* {
* value: "34.72.239.218",
* },
* ],
* },
* },
* deletionProtection: true,
* });
* const db = new gcp.sql.Database("db", {
* instance: instance.name,
* name: "db",
* });
* const pwd = new random.index.Password("pwd", {
* length: 16,
* special: false,
* });
* const user = new gcp.sql.User("user", {
* name: "user",
* instance: instance.name,
* host: "%",
* password: pwd.result,
* });
* const sourceConnectionProfile = new gcp.datastream.ConnectionProfile("source_connection_profile", {
* displayName: "Source connection profile",
* location: "us-central1",
* connectionProfileId: "source-profile",
* mysqlProfile: {
* hostname: instance.publicIpAddress,
* username: user.name,
* password: user.password,
* },
* });
* const destinationConnectionProfile = new gcp.datastream.ConnectionProfile("destination_connection_profile", {
* displayName: "Connection profile",
* location: "us-central1",
* connectionProfileId: "destination-profile",
* bigqueryProfile: {},
* });
* const _default = new gcp.datastream.Stream("default", {
* streamId: "my-stream",
* location: "us-central1",
* displayName: "my stream",
* sourceConfig: {
* sourceConnectionProfile: sourceConnectionProfile.id,
* mysqlSourceConfig: {},
* },
* destinationConfig: {
* destinationConnectionProfile: destinationConnectionProfile.id,
* bigqueryDestinationConfig: {
* sourceHierarchyDatasets: {
* datasetTemplate: {
* location: "us-central1",
* },
* projectId: cross_project_dataset.projectId,
* },
* },
* },
* backfillNone: {},
* });
* ```
* ### Datastream Stream Bigquery Append Only
*
* ```typescript
* import * as pulumi from "@pulumi/pulumi";
* import * as gcp from "@pulumi/gcp";
* import * as random from "@pulumi/random";
*
* const project = gcp.organizations.getProject({});
* const instance = new gcp.sql.DatabaseInstance("instance", {
* name: "my-instance",
* databaseVersion: "MYSQL_8_0",
* region: "us-central1",
* settings: {
* tier: "db-f1-micro",
* backupConfiguration: {
* enabled: true,
* binaryLogEnabled: true,
* },
* ipConfiguration: {
* authorizedNetworks: [
* {
* value: "34.71.242.81",
* },
* {
* value: "34.72.28.29",
* },
* {
* value: "34.67.6.157",
* },
* {
* value: "34.67.234.134",
* },
* {
* value: "34.72.239.218",
* },
* ],
* },
* },
* deletionProtection: true,
* });
* const db = new gcp.sql.Database("db", {
* instance: instance.name,
* name: "db",
* });
* const pwd = new random.index.Password("pwd", {
* length: 16,
* special: false,
* });
* const user = new gcp.sql.User("user", {
* name: "user",
* instance: instance.name,
* host: "%",
* password: pwd.result,
* });
* const sourceConnectionProfile = new gcp.datastream.ConnectionProfile("source_connection_profile", {
* displayName: "Source connection profile",
* location: "us-central1",
* connectionProfileId: "source-profile",
* mysqlProfile: {
* hostname: instance.publicIpAddress,
* username: user.name,
* password: user.password,
* },
* });
* const destinationConnectionProfile = new gcp.datastream.ConnectionProfile("destination_connection_profile", {
* displayName: "Connection profile",
* location: "us-central1",
* connectionProfileId: "destination-profile",
* bigqueryProfile: {},
* });
* const _default = new gcp.datastream.Stream("default", {
* streamId: "my-stream",
* location: "us-central1",
* displayName: "my stream",
* sourceConfig: {
* sourceConnectionProfile: sourceConnectionProfile.id,
* mysqlSourceConfig: {},
* },
* destinationConfig: {
* destinationConnectionProfile: destinationConnectionProfile.id,
* bigqueryDestinationConfig: {
* sourceHierarchyDatasets: {
* datasetTemplate: {
* location: "us-central1",
* },
* },
* appendOnly: {},
* },
* },
* backfillNone: {},
* });
* ```
* ### Datastream Stream Bigquery Blmt
*
* ```typescript
* import * as pulumi from "@pulumi/pulumi";
* import * as gcp from "@pulumi/gcp";
* import * as random from "@pulumi/random";
*
* const project = gcp.organizations.getProject({});
* const instance = new gcp.sql.DatabaseInstance("instance", {
* name: "blmt-instance",
* databaseVersion: "MYSQL_8_0",
* region: "us-central1",
* settings: {
* tier: "db-f1-micro",
* ipConfiguration: {
* authorizedNetworks: [
* {
* value: "34.71.242.81",
* },
* {
* value: "34.72.28.29",
* },
* {
* value: "34.67.6.157",
* },
* {
* value: "34.67.234.134",
* },
* {
* value: "34.72.239.218",
* },
* ],
* },
* },
* deletionProtection: true,
* });
* const db = new gcp.sql.Database("db", {
* instance: instance.name,
* name: "db",
* });
* const pwd = new random.index.Password("pwd", {
* length: 16,
* special: false,
* });
* const user = new gcp.sql.User("user", {
* name: "user",
* instance: instance.name,
* host: "%",
* password: pwd.result,
* });
* const blmtBucket = new gcp.storage.Bucket("blmt_bucket", {
* name: "blmt-bucket",
* location: "us-central1",
* forceDestroy: true,
* });
* const blmtConnection = new gcp.bigquery.Connection("blmt_connection", {
* project: project.then(project => project.projectId),
* location: "us-central1",
* connectionId: "blmt-connection",
* friendlyName: "Datastream BLMT Test Connection",
* description: "Connection for Datastream BLMT test",
* cloudResource: {},
* });
* const blmtConnectionBucketAdmin = new gcp.storage.BucketIAMMember("blmt_connection_bucket_admin", {
* bucket: blmtBucket.name,
* role: "roles/storage.admin",
* member: blmtConnection.cloudResource.apply(cloudResource => `serviceAccount:${cloudResource?.serviceAccountId}`),
* });
* const sourceConnectionProfile = new gcp.datastream.ConnectionProfile("source_connection_profile", {
* displayName: "Source connection profile",
* location: "us-central1",
* connectionProfileId: "blmt-source-profile",
* mysqlProfile: {
* hostname: instance.publicIpAddress,
* username: user.name,
* password: user.password,
* },
* });
* const destinationConnectionProfile = new gcp.datastream.ConnectionProfile("destination_connection_profile", {
* displayName: "Connection profile",
* location: "us-central1",
* connectionProfileId: "blmt-destination-profile",
* bigqueryProfile: {},
* });
* const _default = new gcp.datastream.Stream("default", {
* streamId: "blmt-stream",
* location: "us-central1",
* displayName: "My BLMT stream",
* sourceConfig: {
* sourceConnectionProfile: sourceConnectionProfile.id,
* mysqlSourceConfig: {},
* },
* destinationConfig: {
* destinationConnectionProfile: destinationConnectionProfile.id,
* bigqueryDestinationConfig: {
* sourceHierarchyDatasets: {
* datasetTemplate: {
* location: "us-central1",
* },
* },
* blmtConfig: {
* bucket: blmtBucket.name,
* connectionName: pulumi.all([blmtConnection.project, blmtConnection.location, blmtConnection.connectionId]).apply(([project, location, connectionId]) => `${project}.${location}.${connectionId}`),
* fileFormat: "PARQUET",
* tableFormat: "ICEBERG",
* rootPath: "/",
* },
* appendOnly: {},
* },
* },
* backfillNone: {},
* });
* ```
* ### Datastream Stream Rule Sets Bigquery
*
* ```typescript
* import * as pulumi from "@pulumi/pulumi";
* import * as gcp from "@pulumi/gcp";
*
* const project = gcp.organizations.getProject({});
* const stream = new gcp.datastream.Stream("stream", {
* streamId: "rules-stream",
* location: "us-central1",
* displayName: "BigQuery Stream with Rules",
* sourceConfig: {
* sourceConnectionProfile: "rules-source-profile",
* mysqlSourceConfig: {
* includeObjects: {
* mysqlDatabases: [{
* database: "my_database",
* }],
* },
* binaryLogPosition: {},
* },
* },
* destinationConfig: {
* destinationConnectionProfile: "rules-dest-profile",
* bigqueryDestinationConfig: {
* singleTargetDataset: {
* datasetId: "rules-project:rules-dataset",
* },
* },
* },
* backfillNone: {},
* ruleSets: [
* {
* objectFilter: {
* sourceObjectIdentifier: {
* mysqlIdentifier: {
* database: "test_database",
* table: "test_table_1",
* },
* },
* },
* customizationRules: [
* {
* bigqueryClustering: {
* columns: ["user_id"],
* },
* },
* {
* bigqueryPartitioning: {
* ingestionTimePartition: {},
* },
* },
* ],
* },
* {
* objectFilter: {
* sourceObjectIdentifier: {
* mysqlIdentifier: {
* database: "test_database",
* table: "test_table_2",
* },
* },
* },
* customizationRules: [
* {
* bigqueryClustering: {
* columns: ["event_time"],
* },
* },
* {
* bigqueryPartitioning: {
* timeUnitPartition: {
* column: "event_time",
* partitioningTimeGranularity: "PARTITIONING_TIME_GRANULARITY_DAY",
* },
* },
* },
* ],
* },
* ],
* });
* ```
* ### Datastream Stream Mongodb
*
* ```typescript
* import * as pulumi from "@pulumi/pulumi";
* import * as gcp from "@pulumi/gcp";
*
* const _default = new gcp.datastream.Stream("default", {
* displayName: "Mongodb to BigQuery",
* location: "us-central1",
* streamId: "mongodb-stream",
* sourceConfig: {
* sourceConnectionProfile: "source-profile",
* mongodbSourceConfig: {
* includeObjects: {
* databases: [{
* database: "mydb",
* collections: [
* {
* collection: "mycollection1",
* },
* {
* collection: "mycollection2",
* },
* ],
* }],
* },
* excludeeObjects: [{
* databases: [{
* database: "mydb",
* collections: [{
* fields: [{
* field: "excludedField",
* }],
* }],
* }],
* }],
* },
* },
* destinationConfig: {
* destinationConnectionProfile: "destination-profile",
* bigqueryDestinationConfig: {
* dataFreshness: "900s",
* sourceHierarchyDatasets: {
* datasetTemplate: {
* location: "us-central1",
* },
* },
* },
* },
* backfillNone: {},
* });
* ```
*
* ## Import
*
* Stream can be imported using any of these accepted formats:
*
* * `projects/{{project}}/locations/{{location}}/streams/{{stream_id}}`
* * `{{project}}/{{location}}/{{stream_id}}`
* * `{{location}}/{{stream_id}}`
*
* When using the `pulumi import` command, Stream can be imported using one of the formats above. For example:
*
* ```sh
* $ pulumi import gcp:datastream/stream:Stream default projects/{{project}}/locations/{{location}}/streams/{{stream_id}}
* $ pulumi import gcp:datastream/stream:Stream default {{project}}/{{location}}/{{stream_id}}
* $ pulumi import gcp:datastream/stream:Stream default {{location}}/{{stream_id}}
* ```
*/
class Stream extends pulumi.CustomResource {
/**
* Get an existing Stream resource's state with the given name, ID, and optional extra
* properties used to qualify the lookup.
*
* @param name The _unique_ name of the resulting resource.
* @param id The _unique_ provider ID of the resource to lookup.
* @param state Any extra arguments used during the lookup.
* @param opts Optional settings to control the behavior of the CustomResource.
*/
static get(name, id, state, opts) {
return new Stream(name, state, { ...opts, id: id });
}
/** @internal */
static __pulumiType = 'gcp:datastream/stream:Stream';
/**
* Returns true if the given object is an instance of Stream. This is designed to work even
* when multiple copies of the Pulumi SDK have been loaded into the same process.
*/
static isInstance(obj) {
if (obj === undefined || obj === null) {
return false;
}
return obj['__pulumiType'] === Stream.__pulumiType;
}
constructor(name, argsOrState, opts) {
let resourceInputs = {};
opts = opts || {};
if (opts.id) {
const state = argsOrState;
resourceInputs["backfillAll"] = state?.backfillAll;
resourceInputs["backfillNone"] = state?.backfillNone;
resourceInputs["createWithoutValidation"] = state?.createWithoutValidation;
resourceInputs["customerManagedEncryptionKey"] = state?.customerManagedEncryptionKey;
resourceInputs["deletionPolicy"] = state?.deletionPolicy;
resourceInputs["desiredState"] = state?.desiredState;
resourceInputs["destinationConfig"] = state?.destinationConfig;
resourceInputs["displayName"] = state?.displayName;
resourceInputs["effectiveLabels"] = state?.effectiveLabels;
resourceInputs["labels"] = state?.labels;
resourceInputs["location"] = state?.location;
resourceInputs["name"] = state?.name;
resourceInputs["project"] = state?.project;
resourceInputs["pulumiLabels"] = state?.pulumiLabels;
resourceInputs["ruleSets"] = state?.ruleSets;
resourceInputs["sourceConfig"] = state?.sourceConfig;
resourceInputs["state"] = state?.state;
resourceInputs["streamId"] = state?.streamId;
}
else {
const args = argsOrState;
if (args?.destinationConfig === undefined && !opts.urn) {
throw new Error("Missing required property 'destinationConfig'");
}
if (args?.displayName === undefined && !opts.urn) {
throw new Error("Missing required property 'displayName'");
}
if (args?.location === undefined && !opts.urn) {
throw new Error("Missing required property 'location'");
}
if (args?.sourceConfig === undefined && !opts.urn) {
throw new Error("Missing required property 'sourceConfig'");
}
if (args?.streamId === undefined && !opts.urn) {
throw new Error("Missing required property 'streamId'");
}
resourceInputs["backfillAll"] = args?.backfillAll;
resourceInputs["backfillNone"] = args?.backfillNone;
resourceInputs["createWithoutValidation"] = args?.createWithoutValidation;
resourceInputs["customerManagedEncryptionKey"] = args?.customerManagedEncryptionKey;
resourceInputs["deletionPolicy"] = args?.deletionPolicy;
resourceInputs["desiredState"] = args?.desiredState;
resourceInputs["destinationConfig"] = args?.destinationConfig;
resourceInputs["displayName"] = args?.displayName;
resourceInputs["labels"] = args?.labels;
resourceInputs["location"] = args?.location;
resourceInputs["project"] = args?.project;
resourceInputs["ruleSets"] = args?.ruleSets;
resourceInputs["sourceConfig"] = args?.sourceConfig;
resourceInputs["streamId"] = args?.streamId;
resourceInputs["effectiveLabels"] = undefined /*out*/;
resourceInputs["name"] = undefined /*out*/;
resourceInputs["pulumiLabels"] = undefined /*out*/;
resourceInputs["state"] = undefined /*out*/;
}
opts = pulumi.mergeOptions(utilities.resourceOptsDefaults(), opts);
const secretOpts = { additionalSecretOutputs: ["effectiveLabels", "pulumiLabels"] };
opts = pulumi.mergeOptions(opts, secretOpts);
super(Stream.__pulumiType, name, resourceInputs, opts);
}
}
exports.Stream = Stream;
//# sourceMappingURL=stream.js.map