@cdklabs/aws-data-solutions-framework/lib/processing/examples/pyspark-application-emr-serverless.lit.js

L3 CDK Constructs used to build data solutions with AWS

"use strict";
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0
Object.defineProperty(exports, "__esModule", { value: true });
const cdk = require("aws-cdk-lib");
const aws_iam_1 = require("aws-cdk-lib/aws-iam");
const dsf = require("../../index");
const aws_s3_1 = require("aws-cdk-lib/aws-s3");
class ExampleSparkJobEmrServerlessStack extends cdk.Stack {
    constructor(scope, id) {
        super(scope, id);
        /// !show
        const runtime = new dsf.processing.SparkEmrServerlessRuntime(this, 'SparkRuntime', {
            name: 'mySparkRuntime',
        });
        /// !hide
        const s3ReadPolicy = new aws_iam_1.PolicyDocument({
            statements: [
                aws_iam_1.PolicyStatement.fromJson({
                    actions: ['s3:GetObject'],
                    resources: ['arn:aws:s3:::bucket_name', 'arn:aws:s3:::bucket_name/*'],
                }),
            ],
        });
        const executionRole = dsf.processing.SparkEmrServerlessRuntime.createExecutionRole(this, 'EmrServerlessExecutionRole', s3ReadPolicy);
        /// !show
        const applicationPackage = new dsf.processing.PySparkApplicationPackage(this, 'PySparkApplicationPackage', {
            applicationName: 'nightly-job-aggregation',
            entrypointPath: './../spark/src/entrypoint.py',
            dependenciesFolder: './../spark',
            venvArchivePath: '/venv-package/pyspark-env.tar.gz',
        });
        new dsf.processing.SparkEmrServerlessJob(this, 'SparkNightlyJob', {
            applicationId: runtime.application.attrApplicationId,
            name: 'nightly_job',
            executionRole: executionRole,
            executionTimeout: cdk.Duration.minutes(15),
            s3LogBucket: aws_s3_1.Bucket.fromBucketArn(this, 'LogBucket', 'emr-job-logs-EXAMPLE'),
            s3LogPrefix: 'logs',
            sparkSubmitEntryPoint: applicationPackage.entrypointUri, // use the application package entrypoint
            sparkSubmitParameters: '--conf spark.executor.instances=2 --conf spark.executor.memory=2G --conf spark.driver.memory=2G --conf spark.executor.cores=2 {sparkEnvConf}',
        });
        /// !hide
    }
}
const app = new cdk.App();
new ExampleSparkJobEmrServerlessStack(app, 'ExampleSparkJobEmrServerlessStack');
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicHlzcGFyay1hcHBsaWNhdGlvbi1lbXItc2VydmVybGVzcy5saXQuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvcHJvY2Vzc2luZy9leGFtcGxlcy9weXNwYXJrLWFwcGxpY2F0aW9uLWVtci1zZXJ2ZXJsZXNzLmxpdC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiO0FBQUEscUVBQXFFO0FBQ3JFLHNDQUFzQzs7QUFFdEMsbUNBQW1DO0FBQ25DLGlEQUFzRTtBQUV0RSxtQ0FBbUM7QUFDbkMsK0NBQTRDO0FBRTVDLE1BQU0saUNBQWtDLFNBQVEsR0FBRyxDQUFDLEtBQUs7SUFDdkQsWUFBWSxLQUFnQixFQUFFLEVBQVU7UUFDdEMsS0FBSyxDQUFDLEtBQUssRUFBRSxFQUFFLENBQUMsQ0FBQztRQUNqQixTQUFTO1FBQ1QsTUFBTSxPQUFPLEdBQUcsSUFBSSxHQUFHLENBQUMsVUFBVSxDQUFDLHlCQUF5QixDQUFDLElBQUksRUFBRSxjQUFjLEVBQUU7WUFDakYsSUFBSSxFQUFFLGdCQUFnQjtTQUN2QixDQUFDLENBQUM7UUFFSCxTQUFTO1FBQ1QsTUFBTSxZQUFZLEdBQUcsSUFBSSx3QkFBYyxDQUFDO1lBQ3RDLFVBQVUsRUFBRTtnQkFDVix5QkFBZSxDQUFDLFFBQVEsQ0FBQztvQkFDdkIsT0FBTyxFQUFFLENBQUMsY0FBYyxDQUFDO29CQUN6QixTQUFTLEVBQUUsQ0FBQywwQkFBMEIsRUFBRSw0QkFBNEIsQ0FBQztpQkFDdEUsQ0FBQzthQUNIO1NBQ0YsQ0FBQyxDQUFDO1FBQ0gsTUFBTSxhQUFhLEdBQUcsR0FBRyxDQUFDLFVBQVUsQ0FBQyx5QkFBeUIsQ0FBQyxtQkFBbUIsQ0FBQyxJQUFJLEVBQUUsNEJBQTRCLEVBQUUsWUFBWSxDQUFDLENBQUM7UUFFckksU0FBUztRQUNULE1BQU0sa0JBQWtCLEdBQUcsSUFBSSxHQUFHLENBQUMsVUFBVSxDQUFDLHlCQUF5QixDQUFDLElBQUksRUFBRSwyQkFBMkIsRUFBRTtZQUN6RyxlQUFlLEVBQUUseUJBQXlCO1lBQzFDLGNBQWMsRUFBRSw4QkFBOEI7WUFDOUMsa0JBQWtCLEVBQUUsWUFBWTtZQUNoQyxlQUFlLEVBQUUsa0NBQWtDO1NBQ3BELENBQUMsQ0FBQztRQUVILElBQUksR0FBRyxDQUFDLFVBQVUsQ0FBQyxxQkFBcUIsQ0FBQyxJQUFJLEVBQUUsaUJBQWlCLEVBQUU7WUFDaEUsYUFBYSxFQUFFLE9BQU8sQ0FBQyxXQUFXLENBQUMsaUJBQWlCO1lBQ3BELElBQUksRUFBRSxhQUFhO1lBQ25CLGFBQWEsRUFBRSxhQUFhO1lBQzVCLGdCQUFnQixFQUFFLEdBQUcsQ0FBQyxRQUFRLENBQUMsT0FBTyxDQUFDLEVBQUUsQ0FBQztZQUMxQyxXQUFXLEVBQUUsZUFBTSxDQUFDLGFBQWEsQ0FBQyxJQUFJLEVBQUUsV0FBVyxFQUFFLHNCQUFzQixDQUFDO1lBQzVFLFdBQVcsRUFBRSxNQUFNO1lBQ25CLHFCQUFxQixFQUFFLGtCQUFrQixDQUFDLGFBQWEsRUFBRSx5Q0FBeUM7WUFDbEcscUJBQXFCLEVBQUUsOElBQThJO1NBQ3RLLENBQUMsQ0FBQztRQUNILFNBQVM7SUFDWCxDQUFDO0NBQ0Y7QUFFRCxNQUFNLEdBQUcsR0FBRyxJQUFJLEdBQUcsQ0FBQyxHQUFHLEVBQUUsQ0FBQztBQUMxQixJQUFJLGlDQUFpQyxDQUFDLEdBQUcsRUFBRSxtQ0FBbUMsQ0FBQyxDQUFDIiwic291cmNlc0NvbnRlbnQiOlsiLy8gQ29weXJpZ2h0IEFtYXpvbi5jb20sIEluYy4gb3IgaXRzIGFmZmlsaWF0ZXMuIEFsbCBSaWdodHMgUmVzZXJ2ZWQuXG4vLyBTUERYLUxpY2Vuc2UtSWRlbnRpZmllcjogQXBhY2hlLTIuMFxuXG5pbXBvcnQgKiBhcyBjZGsgZnJvbSAnYXdzLWNkay1saWInO1xuaW1wb3J0IHsgUG9saWN5RG9jdW1lbnQsIFBvbGljeVN0YXRlbWVudCB9IGZyb20gJ2F3cy1jZGstbGliL2F3cy1pYW0nO1xuaW1wb3J0IHsgQ29uc3RydWN0IH0gZnJvbSAnY29uc3RydWN0cyc7XG5pbXBvcnQgKiBhcyBkc2YgZnJvbSAnLi4vLi4vaW5kZXgnO1xuaW1wb3J0IHsgQnVja2V0IH0gZnJvbSAnYXdzLWNkay1saWIvYXdzLXMzJztcblxuY2xhc3MgRXhhbXBsZVNwYXJrSm9iRW1yU2VydmVybGVzc1N0YWNrIGV4dGVuZHMgY2RrLlN0YWNrIHtcbiAgY29uc3RydWN0b3Ioc2NvcGU6IENvbnN0cnVjdCwgaWQ6IHN0cmluZykge1xuICAgIHN1cGVyKHNjb3BlLCBpZCk7XG4gICAgLy8vICFzaG93XG4gICAgY29uc3QgcnVudGltZSA9IG5ldyBkc2YucHJvY2Vzc2luZy5TcGFya0VtclNlcnZlcmxlc3NSdW50aW1lKHRoaXMsICdTcGFya1J1bnRpbWUnLCB7XG4gICAgICBuYW1lOiAnbXlTcGFya1J1bnRpbWUnLFxuICAgIH0pO1xuICAgIFxuICAgIC8vLyAhaGlkZVxuICAgIGNvbnN0IHMzUmVhZFBvbGljeSA9IG5ldyBQb2xpY3lEb2N1bWVudCh7XG4gICAgICBzdGF0ZW1lbnRzOiBbXG4gICAgICAgIFBvbGljeVN0YXRlbWVudC5mcm9tSnNvbih7XG4gICAgICAgICAgYWN0aW9uczogWydzMzpHZXRPYmplY3QnXSxcbiAgICAgICAgICByZXNvdXJjZXM6IFsnYXJuOmF3czpzMzo6OmJ1Y2tldF9uYW1lJywgJ2Fybjphd3M6czM6OjpidWNrZXRfbmFtZS8qJ10sXG4gICAgICAgIH0pLFxuICAgICAgXSxcbiAgICB9KTtcbiAgICBjb25zdCBleGVjdXRpb25Sb2xlID0gZHNmLnByb2Nlc3NpbmcuU3BhcmtFbXJTZXJ2ZXJsZXNzUnVudGltZS5jcmVhdGVFeGVjdXRpb25Sb2xlKHRoaXMsICdFbXJTZXJ2ZXJsZXNzRXhlY3V0aW9uUm9sZScsIHMzUmVhZFBvbGljeSk7XG4gICAgXG4gICAgLy8vICFzaG93XG4gICAgY29uc3QgYXBwbGljYXRpb25QYWNrYWdlID0gbmV3IGRzZi5wcm9jZXNzaW5nLlB5U3BhcmtBcHBsaWNhdGlvblBhY2thZ2UodGhpcywgJ1B5U3BhcmtBcHBsaWNhdGlvblBhY2thZ2UnLCB7XG4gICAgICBhcHBsaWNhdGlvbk5hbWU6ICduaWdodGx5LWpvYi1hZ2dyZWdhdGlvbicsXG4gICAgICBlbnRyeXBvaW50UGF0aDogJy4vLi4vc3Bhcmsvc3JjL2VudHJ5cG9pbnQucHknLFxuICAgICAgZGVwZW5kZW5jaWVzRm9sZGVyOiAnLi8uLi9zcGFyaycsXG4gICAgICB2ZW52QXJjaGl2ZVBhdGg6ICcvdmVudi1wYWNrYWdlL3B5c3BhcmstZW52LnRhci5neicsXG4gICAgfSk7XG4gICAgXG4gICAgbmV3IGRzZi5wcm9jZXNzaW5nLlNwYXJrRW1yU2VydmVybGVzc0pvYih0aGlzLCAnU3BhcmtOaWdodGx5Sm9iJywge1xuICAgICAgYXBwbGljYXRpb25JZDogcnVudGltZS5hcHBsaWNhdGlvbi5hdHRyQXBwbGljYXRpb25JZCxcbiAgICAgIG5hbWU6ICduaWdodGx5X2pvYicsXG4gICAgICBleGVjdXRpb25Sb2xlOiBleGVjdXRpb25Sb2xlLFxuICAgICAgZXhlY3V0aW9uVGltZW91dDogY2RrLkR1cmF0aW9uLm1pbnV0ZXMoMTUpLFxuICAgICAgczNMb2dCdWNrZXQ6IEJ1Y2tldC5mcm9tQnVja2V0QXJuKHRoaXMsICdMb2dCdWNrZXQnLCAnZW1yLWpvYi1sb2dzLUVYQU1QTEUnKSxcbiAgICAgIHMzTG9nUHJlZml4OiAnbG9ncycsXG4gICAgICBzcGFya1N1Ym1pdEVudHJ5UG9pbnQ6IGFwcGxpY2F0aW9uUGFja2FnZS5lbnRyeXBvaW50VXJpLCAvLyB1c2UgdGhlIGFwcGxpY2F0aW9uIHBhY2thZ2UgZW50cnlwb2ludFxuICAgICAgc3BhcmtTdWJtaXRQYXJhbWV0ZXJzOiAnLS1jb25mIHNwYXJrLmV4ZWN1dG9yLmluc3RhbmNlcz0yIC0tY29uZiBzcGFyay5leGVjdXRvci5tZW1vcnk9MkcgLS1jb25mIHNwYXJrLmRyaXZlci5tZW1vcnk9MkcgLS1jb25mIHNwYXJrLmV4ZWN1dG9yLmNvcmVzPTIge3NwYXJrRW52Q29uZn0nLFxuICAgIH0pO1xuICAgIC8vLyAhaGlkZVxuICB9XG59XG5cbmNvbnN0IGFwcCA9IG5ldyBjZGsuQXBwKCk7XG5uZXcgRXhhbXBsZVNwYXJrSm9iRW1yU2VydmVybGVzc1N0YWNrKGFwcCwgJ0V4YW1wbGVTcGFya0pvYkVtclNlcnZlcmxlc3NTdGFjaycpO1xuIl19