@punchcard/data-lake/lib/data-pipeline.js

Data Lake Construct

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.DataPipeline = void 0;
const aws_kinesis_1 = require("@aws-cdk/aws-kinesis");
const punchcard_1 = require("punchcard");
const build_1 = require("punchcard/lib/core/build");
const cdk_1 = require("punchcard/lib/core/cdk");
const period_1 = require("./period");
class DataPipeline {
    constructor(dataLake, id, props) {
        this.dataLake = dataLake;
        const scope = cdk_1.CDK.chain(({ core }) => dataLake.database.map(dataLake => new core.Construct(dataLake, id)));
        this.stream = new punchcard_1.Kinesis.Stream(scope, 'Stream', {
            shape: props.schema.shape,
            streamProps: build_1.Build.of({
                encryption: aws_kinesis_1.StreamEncryption.KMS
            })
        });
        this.index = dataLake.domain.addIndex({
            _id: props.schema.id,
            indexName: props.schema.schemaName,
            mappings: props.schema.shape,
            settings: props.indexSettings || {
                number_of_shards: 1
            }
        });
        this.bucket = new punchcard_1.S3.Bucket(cdk_1.CDK.chain(({ s3 }) => scope.map(scope => new s3.Bucket(scope, 'Bucket', {
            encryption: s3.BucketEncryption.KMS_MANAGED,
        }))));
        this.table = this.stream
            .toFirehoseDeliveryStream(scope, 'ToS3').objects()
            .toGlueTable(scope, 'ToGlue', {
            bucket: this.bucket.resource,
            database: dataLake.database,
            tableName: props.schema.schemaName,
            columns: props.schema.shape,
            partition: {
                keys: period_1.Period.PT1M.schema,
                get(record) {
                    const ts = props.schema.timestamp(record);
                    return new period_1.PT1M({
                        year: ts.getUTCFullYear(),
                        month: ts.getUTCMonth(),
                        day: ts.getUTCDate(),
                        hour: ts.getUTCHours(),
                        minute: ts.getUTCMinutes()
                    });
                }
            }
        });
    }
}
exports.DataPipeline = DataPipeline;
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZGF0YS1waXBlbGluZS5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3NyYy9kYXRhLXBpcGVsaW5lLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7OztBQUFBLHNEQUF3RDtBQUV4RCx5Q0FBNkQ7QUFDN0Qsb0RBQWlEO0FBQ2pELGdEQUE2QztBQUc3QyxxQ0FBd0M7QUFPeEMsTUFBYSxZQUFZO0lBTXZCLFlBQTRCLFFBQWtCLEVBQUUsRUFBVSxFQUFFLEtBQW1DO1FBQW5FLGFBQVEsR0FBUixRQUFRLENBQVU7UUFDNUMsTUFBTSxLQUFLLEdBQUcsU0FBRyxDQUFDLEtBQUssQ0FBQyxDQUFDLEVBQUMsSUFBSSxFQUFDLEVBQUUsRUFBRSxDQUFDLFFBQVEsQ0FBQyxRQUFRLENBQUMsR0FBRyxDQUFDLFFBQVEsQ0FBQyxFQUFFLENBQUMsSUFBSSxJQUFJLENBQUMsU0FBUyxDQUFDLFFBQVEsRUFBRSxFQUFFLENBQUMsQ0FBQyxDQUFDLENBQUM7UUFFekcsSUFBSSxDQUFDLE1BQU0sR0FBRyxJQUFJLG1CQUFPLENBQUMsTUFBTSxDQUFDLEtBQUssRUFBRSxRQUFRLEVBQUU7WUFDaEQsS0FBSyxFQUFFLEtBQUssQ0FBQyxNQUFNLENBQUMsS0FBSztZQUN6QixXQUFXLEVBQUUsYUFBSyxDQUFDLEVBQUUsQ0FBQztnQkFDcEIsVUFBVSxFQUFFLDhCQUFnQixDQUFDLEdBQUc7YUFDakMsQ0FBQztTQUNILENBQUMsQ0FBQztRQUVILElBQUksQ0FBQyxLQUFLLEdBQUcsUUFBUSxDQUFDLE1BQU0sQ0FBQyxRQUFRLENBQUM7WUFDcEMsR0FBRyxFQUFFLEtBQUssQ0FBQyxNQUFNLENBQUMsRUFBRTtZQUNwQixTQUFTLEVBQUUsS0FBSyxDQUFDLE1BQU0sQ0FBQyxVQUFVO1lBQ2xDLFFBQVEsRUFBRSxLQUFLLENBQUMsTUFBTSxDQUFDLEtBQUs7WUFDNUIsUUFBUSxFQUFFLEtBQUssQ0FBQyxhQUFhLElBQUk7Z0JBQy9CLGdCQUFnQixFQUFFLENBQUM7YUFDcEI7U0FDRixDQUFDLENBQUM7UUFFSCxJQUFJLENBQUMsTUFBTSxHQUFHLElBQUksY0FBRSxDQUFDLE1BQU0sQ0FBQyxTQUFHLENBQUMsS0FBSyxDQUFDLENBQUMsRUFBQyxFQUFFLEVBQUMsRUFBRSxFQUFFLENBQUMsS0FBSyxDQUFDLEdBQUcsQ0FBQyxLQUFLLENBQUMsRUFBRSxDQUFDLElBQUksRUFBRSxDQUFDLE1BQU0sQ0FBQyxLQUFLLEVBQUUsUUFBUSxFQUFFO1lBQ2hHLFVBQVUsRUFBRSxFQUFFLENBQUMsZ0JBQWdCLENBQUMsV0FBVztTQUM1QyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUM7UUFFTixJQUFJLENBQUMsS0FBSyxHQUFHLElBQUksQ0FBQyxNQUFNO2FBQ3JCLHdCQUF3QixDQUFDLEtBQUssRUFBRSxNQUFNLENBQUMsQ0FBQyxPQUFPLEVBQUU7YUFDakQsV0FBVyxDQUFDLEtBQUssRUFBRSxRQUFRLEVBQUU7WUFDNUIsTUFBTSxFQUFFLElBQUksQ0FBQyxNQUFNLENBQUMsUUFBUTtZQUM1QixRQUFRLEVBQUUsUUFBUSxDQUFDLFFBQVE7WUFDM0IsU0FBUyxFQUFFLEtBQUssQ0FBQyxNQUFNLENBQUMsVUFBVTtZQUNsQyxPQUFPLEVBQUUsS0FBSyxDQUFDLE1BQU0sQ0FBQyxLQUFLO1lBQzNCLFNBQVMsRUFBRTtnQkFDVCxJQUFJLEVBQUUsZUFBTSxDQUFDLElBQUksQ0FBQyxNQUFNO2dCQUN4QixHQUFHLENBQUMsTUFBbUI7b0JBQ3JCLE1BQU0sRUFBRSxHQUFHLEtBQUssQ0FBQyxNQUFNLENBQUMsU0FBUyxDQUFDLE1BQU0sQ0FBQyxDQUFDO29CQUMxQyxPQUFPLElBQUksYUFBSSxDQUFDO3dCQUNkLElBQUksRUFBRSxFQUFFLENBQUMsY0FBYyxFQUFFO3dCQUN6QixLQUFLLEVBQUUsRUFBRSxDQUFDLFdBQVcsRUFBRTt3QkFDdkIsR0FBRyxFQUFFLEVBQUUsQ0FBQyxVQUFVLEVBQUU7d0JBQ3BCLElBQUksRUFBRSxFQUFFLENBQUMsV0FBVyxFQUFFO3dCQUN0QixNQUFNLEVBQUUsRUFBRSxDQUFDLGFBQWEsRUFBRTtxQkFDM0IsQ0FBQyxDQUFDO2dCQUNMLENBQUM7YUFDRjtTQUNGLENBQUMsQ0FBQztJQUNQLENBQUM7Q0FDRjtBQW5ERCxvQ0FtREMifQ==