UNPKG

@pulumi/aws

Version:

A Pulumi package for creating and managing Amazon Web Services (AWS) cloud resources.

744 lines • 27.3 kB
"use strict"; // *** WARNING: this file was generated by pulumi-language-nodejs. *** // *** Do not edit by hand unless you're certain you know what you are doing! *** Object.defineProperty(exports, "__esModule", { value: true }); exports.FirehoseDeliveryStream = void 0; const pulumi = require("@pulumi/pulumi"); const utilities = require("../utilities"); /** * Provides a Kinesis Firehose Delivery Stream resource. Amazon Kinesis Firehose is a fully managed, elastic service to easily deliver real-time data streams to destinations such as Amazon S3 , Amazon Redshift and Snowflake. * * For more details, see the [Amazon Kinesis Firehose Documentation](https://aws.amazon.com/documentation/firehose/). * * ## Example Usage * * ### Extended S3 Destination * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as aws from "@pulumi/aws"; * * const bucket = new aws.s3.Bucket("bucket", {bucket: "tf-test-bucket"}); * const firehoseAssumeRole = aws.iam.getPolicyDocument({ * statements: [{ * effect: "Allow", * principals: [{ * type: "Service", * identifiers: ["firehose.amazonaws.com"], * }], * actions: ["sts:AssumeRole"], * }], * }); * const firehoseRole = new aws.iam.Role("firehose_role", { * name: "firehose_test_role", * assumeRolePolicy: firehoseAssumeRole.then(firehoseAssumeRole => firehoseAssumeRole.json), * }); * const lambdaAssumeRole = aws.iam.getPolicyDocument({ * statements: [{ * effect: "Allow", * principals: [{ * type: "Service", * identifiers: ["lambda.amazonaws.com"], * }], * actions: ["sts:AssumeRole"], * }], * }); * const lambdaIam = new aws.iam.Role("lambda_iam", { * name: "lambda_iam", * assumeRolePolicy: lambdaAssumeRole.then(lambdaAssumeRole => lambdaAssumeRole.json), * }); * const lambdaProcessor = new aws.lambda.Function("lambda_processor", { * code: new pulumi.asset.FileArchive("lambda.zip"), * name: "firehose_lambda_processor", * role: lambdaIam.arn, * handler: "exports.handler", * runtime: aws.lambda.Runtime.NodeJS20dX, * }); * const extendedS3Stream = new aws.kinesis.FirehoseDeliveryStream("extended_s3_stream", { * name: "kinesis-firehose-extended-s3-test-stream", * destination: "extended_s3", * extendedS3Configuration: { * roleArn: firehoseRole.arn, * bucketArn: bucket.arn, * processingConfiguration: { * enabled: true, * processors: [{ * type: "Lambda", * parameters: [{ * parameterName: "LambdaArn", * parameterValue: pulumi.interpolate`${lambdaProcessor.arn}:$LATEST`, * }], * }], * }, * }, * }); * const bucketAcl = new aws.s3.BucketAcl("bucket_acl", { * bucket: bucket.id, * acl: "private", * }); * ``` * * ### Extended S3 Destination with dynamic partitioning * * These examples use built-in Firehose functionality, rather than requiring a lambda. * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as aws from "@pulumi/aws"; * * const extendedS3Stream = new aws.kinesis.FirehoseDeliveryStream("extended_s3_stream", { * name: "kinesis-firehose-extended-s3-test-stream", * destination: "extended_s3", * extendedS3Configuration: { * roleArn: firehoseRole.arn, * bucketArn: bucket.arn, * bufferingSize: 64, * dynamicPartitioningConfiguration: { * enabled: true, * }, * prefix: "data/customer_id=!{partitionKeyFromQuery:customer_id}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/", * errorOutputPrefix: "errors/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/!{firehose:error-output-type}/", * processingConfiguration: { * enabled: true, * processors: [ * { * type: "RecordDeAggregation", * parameters: [{ * parameterName: "SubRecordType", * parameterValue: "JSON", * }], * }, * { * type: "AppendDelimiterToRecord", * }, * { * type: "MetadataExtraction", * parameters: [ * { * parameterName: "JsonParsingEngine", * parameterValue: "JQ-1.6", * }, * { * parameterName: "MetadataExtractionQuery", * parameterValue: "{customer_id:.customer_id}", * }, * ], * }, * ], * }, * }, * }); * ``` * * Multiple Dynamic Partitioning Keys (maximum of 50) can be added by comma separating the `parameterValue`. * * The following example adds the Dynamic Partitioning Keys: `storeId` and `customerId` to the S3 prefix. * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as aws from "@pulumi/aws"; * * const extendedS3Stream = new aws.kinesis.FirehoseDeliveryStream("extended_s3_stream", { * name: "kinesis-firehose-extended-s3-test-stream", * destination: "extended_s3", * extendedS3Configuration: { * roleArn: firehoseRole.arn, * bucketArn: bucket.arn, * bufferingSize: 64, * dynamicPartitioningConfiguration: { * enabled: true, * }, * prefix: "data/store_id=!{partitionKeyFromQuery:store_id}/customer_id=!{partitionKeyFromQuery:customer_id}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/", * errorOutputPrefix: "errors/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/!{firehose:error-output-type}/", * processingConfiguration: { * enabled: true, * processors: [{ * type: "MetadataExtraction", * parameters: [ * { * parameterName: "JsonParsingEngine", * parameterValue: "JQ-1.6", * }, * { * parameterName: "MetadataExtractionQuery", * parameterValue: "{store_id:.store_id,customer_id:.customer_id}", * }, * ], * }], * }, * }, * }); * ``` * * ### Redshift Destination * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as aws from "@pulumi/aws"; * * const testCluster = new aws.redshift.Cluster("test_cluster", { * clusterIdentifier: "tf-redshift-cluster", * databaseName: "test", * masterUsername: "testuser", * masterPassword: "T3stPass", * nodeType: "dc1.large", * clusterType: "single-node", * }); * const testStream = new aws.kinesis.FirehoseDeliveryStream("test_stream", { * name: "kinesis-firehose-test-stream", * destination: "redshift", * redshiftConfiguration: { * roleArn: firehoseRole.arn, * clusterJdbcurl: pulumi.interpolate`jdbc:redshift://${testCluster.endpoint}/${testCluster.databaseName}`, * username: "testuser", * password: "T3stPass", * dataTableName: "test-table", * copyOptions: "delimiter '|'", * dataTableColumns: "test-col", * s3BackupMode: "Enabled", * s3Configuration: { * roleArn: firehoseRole.arn, * bucketArn: bucket.arn, * bufferingSize: 10, * bufferingInterval: 400, * compressionFormat: "GZIP", * }, * s3BackupConfiguration: { * roleArn: firehoseRole.arn, * bucketArn: bucket.arn, * bufferingSize: 15, * bufferingInterval: 300, * compressionFormat: "GZIP", * }, * }, * }); * ``` * * ### Elasticsearch Destination * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as aws from "@pulumi/aws"; * * const testCluster = new aws.elasticsearch.Domain("test_cluster", {domainName: "firehose-es-test"}); * const testStream = new aws.kinesis.FirehoseDeliveryStream("test_stream", { * name: "kinesis-firehose-test-stream", * destination: "elasticsearch", * elasticsearchConfiguration: { * domainArn: testCluster.arn, * roleArn: firehoseRole.arn, * indexName: "test", * typeName: "test", * s3Configuration: { * roleArn: firehoseRole.arn, * bucketArn: bucket.arn, * bufferingSize: 10, * bufferingInterval: 400, * compressionFormat: "GZIP", * }, * processingConfiguration: { * enabled: true, * processors: [{ * type: "Lambda", * parameters: [{ * parameterName: "LambdaArn", * parameterValue: `${lambdaProcessor.arn}:$LATEST`, * }], * }], * }, * }, * }); * ``` * * ### Elasticsearch Destination With VPC * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as aws from "@pulumi/aws"; * * const testCluster = new aws.elasticsearch.Domain("test_cluster", { * domainName: "es-test", * clusterConfig: { * instanceCount: 2, * zoneAwarenessEnabled: true, * instanceType: "t2.small.elasticsearch", * }, * ebsOptions: { * ebsEnabled: true, * volumeSize: 10, * }, * vpcOptions: { * securityGroupIds: [first.id], * subnetIds: [ * firstAwsSubnet.id, * second.id, * ], * }, * }); * const firehose_elasticsearch = aws.iam.getPolicyDocumentOutput({ * statements: [ * { * effect: "Allow", * actions: ["es:*"], * resources: [ * testCluster.arn, * pulumi.interpolate`${testCluster.arn}/*`, * ], * }, * { * effect: "Allow", * actions: [ * "ec2:DescribeVpcs", * "ec2:DescribeVpcAttribute", * "ec2:DescribeSubnets", * "ec2:DescribeSecurityGroups", * "ec2:DescribeNetworkInterfaces", * "ec2:CreateNetworkInterface", * "ec2:CreateNetworkInterfacePermission", * "ec2:DeleteNetworkInterface", * ], * resources: ["*"], * }, * ], * }); * const firehose_elasticsearchRolePolicy = new aws.iam.RolePolicy("firehose-elasticsearch", { * name: "elasticsearch", * role: firehose.id, * policy: firehose_elasticsearch.apply(firehose_elasticsearch => firehose_elasticsearch.json), * }); * const test = new aws.kinesis.FirehoseDeliveryStream("test", { * name: "kinesis-firehose-es", * destination: "elasticsearch", * elasticsearchConfiguration: { * domainArn: testCluster.arn, * roleArn: firehose.arn, * indexName: "test", * typeName: "test", * s3Configuration: { * roleArn: firehose.arn, * bucketArn: bucket.arn, * }, * vpcConfig: { * subnetIds: [ * firstAwsSubnet.id, * second.id, * ], * securityGroupIds: [first.id], * roleArn: firehose.arn, * }, * }, * }, { * dependsOn: [firehose_elasticsearchRolePolicy], * }); * ``` * * ### OpenSearch Destination * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as aws from "@pulumi/aws"; * * const testCluster = new aws.opensearch.Domain("test_cluster", {domainName: "firehose-os-test"}); * const testStream = new aws.kinesis.FirehoseDeliveryStream("test_stream", { * name: "kinesis-firehose-test-stream", * destination: "opensearch", * opensearchConfiguration: { * domainArn: testCluster.arn, * roleArn: firehoseRole.arn, * indexName: "test", * s3Configuration: { * roleArn: firehoseRole.arn, * bucketArn: bucket.arn, * bufferingSize: 10, * bufferingInterval: 400, * compressionFormat: "GZIP", * }, * processingConfiguration: { * enabled: true, * processors: [{ * type: "Lambda", * parameters: [{ * parameterName: "LambdaArn", * parameterValue: `${lambdaProcessor.arn}:$LATEST`, * }], * }], * }, * }, * }); * ``` * * ### OpenSearch Destination With VPC * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as aws from "@pulumi/aws"; * * const testCluster = new aws.opensearch.Domain("test_cluster", { * domainName: "es-test", * clusterConfig: { * instanceCount: 2, * zoneAwarenessEnabled: true, * instanceType: "m4.large.search", * }, * ebsOptions: { * ebsEnabled: true, * volumeSize: 10, * }, * vpcOptions: { * securityGroupIds: [first.id], * subnetIds: [ * firstAwsSubnet.id, * second.id, * ], * }, * }); * const firehose_opensearch = new aws.iam.RolePolicy("firehose-opensearch", { * name: "opensearch", * role: firehose.id, * policy: pulumi.interpolate`{ * "Version": "2012-10-17", * "Statement": [ * { * "Effect": "Allow", * "Action": [ * "es:*" * ], * "Resource": [ * "${testCluster.arn}", * "${testCluster.arn}/*" * ] * }, * { * "Effect": "Allow", * "Action": [ * "ec2:DescribeVpcs", * "ec2:DescribeVpcAttribute", * "ec2:DescribeSubnets", * "ec2:DescribeSecurityGroups", * "ec2:DescribeNetworkInterfaces", * "ec2:CreateNetworkInterface", * "ec2:CreateNetworkInterfacePermission", * "ec2:DeleteNetworkInterface" * ], * "Resource": [ * "*" * ] * } * ] * } * `, * }); * const test = new aws.kinesis.FirehoseDeliveryStream("test", { * name: "pulumi-kinesis-firehose-os", * destination: "opensearch", * opensearchConfiguration: { * domainArn: testCluster.arn, * roleArn: firehose.arn, * indexName: "test", * s3Configuration: { * roleArn: firehose.arn, * bucketArn: bucket.arn, * }, * vpcConfig: { * subnetIds: [ * firstAwsSubnet.id, * second.id, * ], * securityGroupIds: [first.id], * roleArn: firehose.arn, * }, * }, * }, { * dependsOn: [firehose_opensearch], * }); * ``` * * ### OpenSearch Serverless Destination * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as aws from "@pulumi/aws"; * * const testCollection = new aws.opensearch.ServerlessCollection("test_collection", {name: "firehose-osserverless-test"}); * const testStream = new aws.kinesis.FirehoseDeliveryStream("test_stream", { * name: "kinesis-firehose-test-stream", * destination: "opensearchserverless", * opensearchserverlessConfiguration: { * collectionEndpoint: testCollection.collectionEndpoint, * roleArn: firehoseRole.arn, * indexName: "test", * s3Configuration: { * roleArn: firehoseRole.arn, * bucketArn: bucket.arn, * bufferingSize: 10, * bufferingInterval: 400, * compressionFormat: "GZIP", * }, * processingConfiguration: { * enabled: true, * processors: [{ * type: "Lambda", * parameters: [{ * parameterName: "LambdaArn", * parameterValue: `${lambdaProcessor.arn}:$LATEST`, * }], * }], * }, * }, * }); * ``` * * ### Iceberg Destination * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as aws from "@pulumi/aws"; * * const current = aws.getCallerIdentity({}); * const currentGetPartition = aws.getPartition({}); * const currentGetRegion = aws.getRegion({}); * const bucket = new aws.s3.Bucket("bucket", { * bucket: "test-bucket", * forceDestroy: true, * }); * const test = new aws.glue.CatalogDatabase("test", {name: "test"}); * const testCatalogTable = new aws.glue.CatalogTable("test", { * name: "test", * databaseName: test.name, * parameters: { * format: "parquet", * }, * tableType: "EXTERNAL_TABLE", * openTableFormatInput: { * icebergInput: { * metadataOperation: "CREATE", * version: "2", * }, * }, * storageDescriptor: { * location: pulumi.interpolate`s3://${bucket.id}`, * columns: [{ * name: "my_column_1", * type: "int", * }], * }, * }); * const testStream = new aws.kinesis.FirehoseDeliveryStream("test_stream", { * name: "kinesis-firehose-test-stream", * destination: "iceberg", * icebergConfiguration: { * roleArn: firehoseRole.arn, * catalogArn: Promise.all([currentGetPartition, currentGetRegion, current]).then(([currentGetPartition, currentGetRegion, current]) => `arn:${currentGetPartition.partition}:glue:${currentGetRegion.region}:${current.accountId}:catalog`), * bufferingSize: 10, * bufferingInterval: 400, * s3Configuration: { * roleArn: firehoseRole.arn, * bucketArn: bucket.arn, * }, * destinationTableConfigurations: [{ * databaseName: test.name, * tableName: testCatalogTable.name, * }], * processingConfiguration: { * enabled: true, * processors: [{ * type: "Lambda", * parameters: [{ * parameterName: "LambdaArn", * parameterValue: `${lambdaProcessor.arn}:$LATEST`, * }], * }], * }, * }, * }); * ``` * * ### Splunk Destination * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as aws from "@pulumi/aws"; * * const testStream = new aws.kinesis.FirehoseDeliveryStream("test_stream", { * name: "kinesis-firehose-test-stream", * destination: "splunk", * splunkConfiguration: { * hecEndpoint: "https://http-inputs-mydomain.splunkcloud.com:443", * hecToken: "51D4DA16-C61B-4F5F-8EC7-ED4301342A4A", * hecAcknowledgmentTimeout: 600, * hecEndpointType: "Event", * s3BackupMode: "FailedEventsOnly", * s3Configuration: { * roleArn: firehose.arn, * bucketArn: bucket.arn, * bufferingSize: 10, * bufferingInterval: 400, * compressionFormat: "GZIP", * }, * }, * }); * ``` * * ### HTTP Endpoint (e.g., New Relic) Destination * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as aws from "@pulumi/aws"; * * const testStream = new aws.kinesis.FirehoseDeliveryStream("test_stream", { * name: "kinesis-firehose-test-stream", * destination: "http_endpoint", * httpEndpointConfiguration: { * url: "https://aws-api.newrelic.com/firehose/v1", * name: "New Relic", * accessKey: "my-key", * bufferingSize: 15, * bufferingInterval: 600, * roleArn: firehose.arn, * s3BackupMode: "FailedDataOnly", * s3Configuration: { * roleArn: firehose.arn, * bucketArn: bucket.arn, * bufferingSize: 10, * bufferingInterval: 400, * compressionFormat: "GZIP", * }, * requestConfiguration: { * contentEncoding: "GZIP", * commonAttributes: [ * { * name: "testname", * value: "testvalue", * }, * { * name: "testname2", * value: "testvalue2", * }, * ], * }, * }, * }); * ``` * * ### Snowflake Destination * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as aws from "@pulumi/aws"; * * const exampleSnowflakeDestination = new aws.kinesis.FirehoseDeliveryStream("example_snowflake_destination", { * name: "example-snowflake-destination", * destination: "snowflake", * snowflakeConfiguration: { * accountUrl: "https://example.snowflakecomputing.com", * bufferingSize: 15, * bufferingInterval: 600, * database: "example-db", * privateKey: "...", * roleArn: firehose.arn, * schema: "example-schema", * table: "example-table", * user: "example-usr", * s3Configuration: { * roleArn: firehose.arn, * bucketArn: bucket.arn, * bufferingSize: 10, * bufferingInterval: 400, * compressionFormat: "GZIP", * }, * }, * }); * ``` * * ## Import * * Using `pulumi import`, import Kinesis Firehose Delivery streams using the stream ARN. For example: * * ```sh * $ pulumi import aws:kinesis/firehoseDeliveryStream:FirehoseDeliveryStream foo arn:aws:firehose:us-east-1:XXX:deliverystream/example * ``` * Note: Import does not work for stream destination `s3`. Consider using `extended_s3` since `s3` destination is deprecated. */ class FirehoseDeliveryStream extends pulumi.CustomResource { /** * Get an existing FirehoseDeliveryStream resource's state with the given name, ID, and optional extra * properties used to qualify the lookup. * * @param name The _unique_ name of the resulting resource. * @param id The _unique_ provider ID of the resource to lookup. * @param state Any extra arguments used during the lookup. * @param opts Optional settings to control the behavior of the CustomResource. */ static get(name, id, state, opts) { return new FirehoseDeliveryStream(name, state, { ...opts, id: id }); } /** * Returns true if the given object is an instance of FirehoseDeliveryStream. This is designed to work even * when multiple copies of the Pulumi SDK have been loaded into the same process. */ static isInstance(obj) { if (obj === undefined || obj === null) { return false; } return obj['__pulumiType'] === FirehoseDeliveryStream.__pulumiType; } constructor(name, argsOrState, opts) { let resourceInputs = {}; opts = opts || {}; if (opts.id) { const state = argsOrState; resourceInputs["arn"] = state?.arn; resourceInputs["destination"] = state?.destination; resourceInputs["destinationId"] = state?.destinationId; resourceInputs["elasticsearchConfiguration"] = state?.elasticsearchConfiguration; resourceInputs["extendedS3Configuration"] = state?.extendedS3Configuration; resourceInputs["httpEndpointConfiguration"] = state?.httpEndpointConfiguration; resourceInputs["icebergConfiguration"] = state?.icebergConfiguration; resourceInputs["kinesisSourceConfiguration"] = state?.kinesisSourceConfiguration; resourceInputs["mskSourceConfiguration"] = state?.mskSourceConfiguration; resourceInputs["name"] = state?.name; resourceInputs["opensearchConfiguration"] = state?.opensearchConfiguration; resourceInputs["opensearchserverlessConfiguration"] = state?.opensearchserverlessConfiguration; resourceInputs["redshiftConfiguration"] = state?.redshiftConfiguration; resourceInputs["region"] = state?.region; resourceInputs["serverSideEncryption"] = state?.serverSideEncryption; resourceInputs["snowflakeConfiguration"] = state?.snowflakeConfiguration; resourceInputs["splunkConfiguration"] = state?.splunkConfiguration; resourceInputs["tags"] = state?.tags; resourceInputs["tagsAll"] = state?.tagsAll; resourceInputs["versionId"] = state?.versionId; } else { const args = argsOrState; if (args?.destination === undefined && !opts.urn) { throw new Error("Missing required property 'destination'"); } resourceInputs["arn"] = args?.arn; resourceInputs["destination"] = args?.destination; resourceInputs["destinationId"] = args?.destinationId; resourceInputs["elasticsearchConfiguration"] = args?.elasticsearchConfiguration; resourceInputs["extendedS3Configuration"] = args?.extendedS3Configuration; resourceInputs["httpEndpointConfiguration"] = args?.httpEndpointConfiguration; resourceInputs["icebergConfiguration"] = args?.icebergConfiguration; resourceInputs["kinesisSourceConfiguration"] = args?.kinesisSourceConfiguration; resourceInputs["mskSourceConfiguration"] = args?.mskSourceConfiguration; resourceInputs["name"] = args?.name; resourceInputs["opensearchConfiguration"] = args?.opensearchConfiguration; resourceInputs["opensearchserverlessConfiguration"] = args?.opensearchserverlessConfiguration; resourceInputs["redshiftConfiguration"] = args?.redshiftConfiguration; resourceInputs["region"] = args?.region; resourceInputs["serverSideEncryption"] = args?.serverSideEncryption; resourceInputs["snowflakeConfiguration"] = args?.snowflakeConfiguration; resourceInputs["splunkConfiguration"] = args?.splunkConfiguration; resourceInputs["tags"] = args?.tags; resourceInputs["versionId"] = args?.versionId; resourceInputs["tagsAll"] = undefined /*out*/; } opts = pulumi.mergeOptions(utilities.resourceOptsDefaults(), opts); super(FirehoseDeliveryStream.__pulumiType, name, resourceInputs, opts); } } exports.FirehoseDeliveryStream = FirehoseDeliveryStream; /** @internal */ FirehoseDeliveryStream.__pulumiType = 'aws:kinesis/firehoseDeliveryStream:FirehoseDeliveryStream'; //# sourceMappingURL=firehoseDeliveryStream.js.map