@cdklabs/aws-data-solutions-framework
Version:
L3 CDK Constructs used to build data solutions with AWS
45 lines • 6.31 kB
JavaScript
;
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0
Object.defineProperty(exports, "__esModule", { value: true });
const cdk = require("aws-cdk-lib");
const aws_cdk_lib_1 = require("aws-cdk-lib");
const aws_iam_1 = require("aws-cdk-lib/aws-iam");
const dsf = require("../../index");
const aws_s3_1 = require("aws-cdk-lib/aws-s3");
/// !show
class ExampleSparkJobEmrServerlessStack extends cdk.Stack {
constructor(scope, id) {
super(scope, id);
/// !hide
const runtime = new dsf.processing.SparkEmrServerlessRuntime(this, 'SparkRuntime', {
name: 'mySparkRuntime',
});
const s3ReadPolicy = new aws_iam_1.PolicyDocument({
statements: [
aws_iam_1.PolicyStatement.fromJson({
actions: ['s3:GetObject'],
resources: ['arn:aws:s3:::bucket_name', 'arn:aws:s3:::bucket_name/*'],
}),
],
});
const executionRole = dsf.processing.SparkEmrServerlessRuntime.createExecutionRole(this, 'EmrServerlessExecutionRole', s3ReadPolicy);
/// !show
const nightJob = new dsf.processing.SparkEmrServerlessJob(this, 'PiJob', {
applicationId: runtime.application.attrApplicationId,
name: 'PiCalculation',
executionRole: executionRole,
executionTimeout: cdk.Duration.minutes(15),
s3LogBucket: aws_s3_1.Bucket.fromBucketName(this, 'LogBucket', 'emr-job-logs-EXAMPLE'),
s3LogPrefix: 'logs',
sparkSubmitEntryPoint: 'local:///usr/lib/spark/examples/src/main/python/pi.py',
});
new aws_cdk_lib_1.CfnOutput(this, 'job-state-machine', {
value: nightJob.stateMachine.stateMachineArn,
});
/// !hide
}
}
const app = new cdk.App();
new ExampleSparkJobEmrServerlessStack(app, 'ExampleSparkJobEmrServerlessStack');
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic3Bhcmstam9iLWVtci1zZXJ2ZXJsZXNzLmxpdC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uL3NyYy9wcm9jZXNzaW5nL2V4YW1wbGVzL3NwYXJrLWpvYi1lbXItc2VydmVybGVzcy5saXQudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IjtBQUFBLHFFQUFxRTtBQUNyRSxzQ0FBc0M7O0FBRXRDLG1DQUFtQztBQUNuQyw2Q0FBd0M7QUFDeEMsaURBQXNFO0FBRXRFLG1DQUFtQztBQUNuQywrQ0FBNEM7QUFFNUMsU0FBUztBQUNULE1BQU0saUNBQWtDLFNBQVEsR0FBRyxDQUFDLEtBQUs7SUFDdkQsWUFBWSxLQUFnQixFQUFFLEVBQVU7UUFDdEMsS0FBSyxDQUFDLEtBQUssRUFBRSxFQUFFLENBQUMsQ0FBQztRQUVqQixTQUFTO1FBQ1QsTUFBTSxPQUFPLEdBQUcsSUFBSSxHQUFHLENBQUMsVUFBVSxDQUFDLHlCQUF5QixDQUFDLElBQUksRUFBRSxjQUFjLEVBQUU7WUFDakYsSUFBSSxFQUFFLGdCQUFnQjtTQUN2QixDQUFDLENBQUM7UUFFSCxNQUFNLFlBQVksR0FBRyxJQUFJLHdCQUFjLENBQUM7WUFDdEMsVUFBVSxFQUFFO2dCQUNWLHlCQUFlLENBQUMsUUFBUSxDQUFDO29CQUN2QixPQUFPLEVBQUUsQ0FBQyxjQUFjLENBQUM7b0JBQ3pCLFNBQVMsRUFBRSxDQUFDLDBCQUEwQixFQUFFLDRCQUE0QixDQUFDO2lCQUN0RSxDQUFDO2FBQ0g7U0FDRixDQUFDLENBQUM7UUFFSCxNQUFNLGFBQWEsR0FBRyxHQUFHLENBQUMsVUFBVSxDQUFDLHlCQUF5QixDQUFDLG1CQUFtQixDQUFDLElBQUksRUFBRSw0QkFBNEIsRUFBRSxZQUFZLENBQUMsQ0FBQztRQUNySSxTQUFTO1FBQ1QsTUFBTSxRQUFRLEdBQUcsSUFBSSxHQUFHLENBQUMsVUFBVSxDQUFDLHFCQUFxQixDQUFDLElBQUksRUFBRSxPQUFPLEVBQUU7WUFDdkUsYUFBYSxFQUFFLE9BQU8sQ0FBQyxXQUFXLENBQUMsaUJBQWlCO1lBQ3BELElBQUksRUFBRSxlQUFlO1lBQ3JCLGFBQWEsRUFBRSxhQUFhO1lBQzVCLGdCQUFnQixFQUFFLEdBQUcsQ0FBQyxRQUFRLENBQUMsT0FBTyxDQUFDLEVBQUUsQ0FBQztZQUMxQyxXQUFXLEVBQUUsZUFBTSxDQUFDLGNBQWMsQ0FBQyxJQUFJLEVBQUUsV0FBVyxFQUFFLHNCQUFzQixDQUFDO1lBQzdFLFdBQVcsRUFBRSxNQUFNO1lBQ25CLHFCQUFxQixFQUFFLHVEQUF1RDtTQUMvRSxDQUFDLENBQUM7UUFFSCxJQUFJLHVCQUFTLENBQUMsSUFBSSxFQUFFLG1CQUFtQixFQUFFO1lBQ3ZDLEtBQUssRUFBRSxRQUFRLENBQUMsWUFBYSxDQUFDLGVBQWU7U0FDOUMsQ0FBQyxDQUFDO1FBQ0gsU0FBUztJQUNYLENBQUM7Q0FDRjtBQUVELE1BQU0sR0FBRyxHQUFHLElBQUksR0FBRyxDQUFDLEdBQUcsRUFBRSxDQUFDO0FBQzFCLElBQUksaUNBQWlDLENBQUMsR0FBRyxFQUFFLG1DQUFtQyxDQUFDLENBQUMiLCJzb3VyY2VzQ29udGVudCI6WyIvLyBDb3B5cmlnaHQgQW1hem9uLmNvbSwgSW5jLiBvciBpdHMgYWZmaWxpYXRlcy4gQWxsIFJpZ2h0cyBSZXNlcnZlZC5cbi8vIFNQRFgtTGljZW5zZS1JZGVudGlmaWVyOiBBcGFjaGUtMi4wXG5cbmltcG9ydCAqIGFzIGNkayBmcm9tICdhd3MtY2RrLWxpYic7XG5pbXBvcnQgeyBDZm5PdXRwdXQgfSBmcm9tICdhd3MtY2RrLWxpYic7XG5pbXBvcnQgeyBQb2xpY3lEb2N1bWVudCwgUG9saWN5U3RhdGVtZW50IH0gZnJvbSAnYXdzLWNkay1saWIvYXdzLWlhbSc7XG5pbXBvcnQgeyBDb25zdHJ1Y3QgfSBmcm9tICdjb25zdHJ1Y3RzJztcbmltcG9ydCAqIGFzIGRzZiBmcm9tICcuLi8uLi9pbmRleCc7XG5pbXBvcnQgeyBCdWNrZXQgfSBmcm9tICdhd3MtY2RrLWxpYi9hd3MtczMnO1xuXG4vLy8gIXNob3dcbmNsYXNzIEV4YW1wbGVTcGFya0pvYkVtclNlcnZlcmxlc3NTdGFjayBleHRlbmRzIGNkay5TdGFjayB7XG4gIGNvbnN0cnVjdG9yKHNjb3BlOiBDb25zdHJ1Y3QsIGlkOiBzdHJpbmcpIHtcbiAgICBzdXBlcihzY29wZSwgaWQpO1xuXG4gICAgLy8vICFoaWRlXG4gICAgY29uc3QgcnVudGltZSA9IG5ldyBkc2YucHJvY2Vzc2luZy5TcGFya0VtclNlcnZlcmxlc3NSdW50aW1lKHRoaXMsICdTcGFya1J1bnRpbWUnLCB7XG4gICAgICBuYW1lOiAnbXlTcGFya1J1bnRpbWUnLFxuICAgIH0pO1xuICAgIFxuICAgIGNvbnN0IHMzUmVhZFBvbGljeSA9IG5ldyBQb2xpY3lEb2N1bWVudCh7XG4gICAgICBzdGF0ZW1lbnRzOiBbXG4gICAgICAgIFBvbGljeVN0YXRlbWVudC5mcm9tSnNvbih7XG4gICAgICAgICAgYWN0aW9uczogWydzMzpHZXRPYmplY3QnXSxcbiAgICAgICAgICByZXNvdXJjZXM6IFsnYXJuOmF3czpzMzo6OmJ1Y2tldF9uYW1lJywgJ2Fybjphd3M6czM6OjpidWNrZXRfbmFtZS8qJ10sXG4gICAgICAgIH0pLFxuICAgICAgXSxcbiAgICB9KTtcbiAgICBcbiAgICBjb25zdCBleGVjdXRpb25Sb2xlID0gZHNmLnByb2Nlc3NpbmcuU3BhcmtFbXJTZXJ2ZXJsZXNzUnVudGltZS5jcmVhdGVFeGVjdXRpb25Sb2xlKHRoaXMsICdFbXJTZXJ2ZXJsZXNzRXhlY3V0aW9uUm9sZScsIHMzUmVhZFBvbGljeSk7XG4gICAgLy8vICFzaG93XG4gICAgY29uc3QgbmlnaHRKb2IgPSBuZXcgZHNmLnByb2Nlc3NpbmcuU3BhcmtFbXJTZXJ2ZXJsZXNzSm9iKHRoaXMsICdQaUpvYicsIHtcbiAgICAgIGFwcGxpY2F0aW9uSWQ6IHJ1bnRpbWUuYXBwbGljYXRpb24uYXR0ckFwcGxpY2F0aW9uSWQsXG4gICAgICBuYW1lOiAnUGlDYWxjdWxhdGlvbicsXG4gICAgICBleGVjdXRpb25Sb2xlOiBleGVjdXRpb25Sb2xlLFxuICAgICAgZXhlY3V0aW9uVGltZW91dDogY2RrLkR1cmF0aW9uLm1pbnV0ZXMoMTUpLFxuICAgICAgczNMb2dCdWNrZXQ6IEJ1Y2tldC5mcm9tQnVja2V0TmFtZSh0aGlzLCAnTG9nQnVja2V0JywgJ2Vtci1qb2ItbG9ncy1FWEFNUExFJyksXG4gICAgICBzM0xvZ1ByZWZpeDogJ2xvZ3MnLFxuICAgICAgc3BhcmtTdWJtaXRFbnRyeVBvaW50OiAnbG9jYWw6Ly8vdXNyL2xpYi9zcGFyay9leGFtcGxlcy9zcmMvbWFpbi9weXRob24vcGkucHknLFxuICAgIH0pO1xuICAgIFxuICAgIG5ldyBDZm5PdXRwdXQodGhpcywgJ2pvYi1zdGF0ZS1tYWNoaW5lJywge1xuICAgICAgdmFsdWU6IG5pZ2h0Sm9iLnN0YXRlTWFjaGluZSEuc3RhdGVNYWNoaW5lQXJuLFxuICAgIH0pO1xuICAgIC8vLyAhaGlkZVxuICB9XG59XG5cbmNvbnN0IGFwcCA9IG5ldyBjZGsuQXBwKCk7XG5uZXcgRXhhbXBsZVNwYXJrSm9iRW1yU2VydmVybGVzc1N0YWNrKGFwcCwgJ0V4YW1wbGVTcGFya0pvYkVtclNlcnZlcmxlc3NTdGFjaycpO1xuIl19