@cloudsnorkel/cdk-github-runners
Version:
CDK construct to create GitHub Actions self-hosted runners. Creates ephemeral runners on demand. Easy to deploy and highly customizable.
636 lines • 116 kB
JavaScript
"use strict";
var _a;
Object.defineProperty(exports, "__esModule", { value: true });
exports.GitHubRunners = void 0;
const JSII_RTTI_SYMBOL_1 = Symbol.for("jsii.rtti");
const fs = require("fs");
const os = require("os");
const path = require("path");
const cdk = require("aws-cdk-lib");
const aws_cdk_lib_1 = require("aws-cdk-lib");
const constructs_1 = require("constructs");
const access_1 = require("./access");
const delete_failed_runner_function_1 = require("./delete-failed-runner-function");
const idle_runner_repear_function_1 = require("./idle-runner-repear-function");
const providers_1 = require("./providers");
const secrets_1 = require("./secrets");
const setup_function_1 = require("./setup-function");
const status_function_1 = require("./status-function");
const token_retriever_function_1 = require("./token-retriever-function");
const utils_1 = require("./utils");
const webhook_1 = require("./webhook");
const webhook_redelivery_1 = require("./webhook-redelivery");
/**
* Create all the required infrastructure to provide self-hosted GitHub runners. It creates a webhook, secrets, and a step function to orchestrate all runs. Secrets are not automatically filled. See README.md for instructions on how to setup GitHub integration.
*
* By default, this will create a runner provider of each available type with the defaults. This is good enough for the initial setup stage when you just want to get GitHub integration working.
*
* ```typescript
* new GitHubRunners(this, 'runners');
* ```
*
* Usually you'd want to configure the runner providers so the runners can run in a certain VPC or have certain permissions.
*
* ```typescript
* const vpc = ec2.Vpc.fromLookup(this, 'vpc', { vpcId: 'vpc-1234567' });
* const runnerSg = new ec2.SecurityGroup(this, 'runner security group', { vpc: vpc });
* const dbSg = ec2.SecurityGroup.fromSecurityGroupId(this, 'database security group', 'sg-1234567');
* const bucket = new s3.Bucket(this, 'runner bucket');
*
* // create a custom CodeBuild provider
* const myProvider = new CodeBuildRunnerProvider(
* this, 'codebuild runner',
* {
* labels: ['my-codebuild'],
* vpc: vpc,
* securityGroups: [runnerSg],
* },
* );
* // grant some permissions to the provider
* bucket.grantReadWrite(myProvider);
* dbSg.connections.allowFrom(runnerSg, ec2.Port.tcp(3306), 'allow runners to connect to MySQL database');
*
* // create the runner infrastructure
* new GitHubRunners(
* this,
* 'runners',
* {
* providers: [myProvider],
* }
* );
* ```
*/
class GitHubRunners extends constructs_1.Construct {
constructor(scope, id, props) {
super(scope, id);
this.props = props;
this.extraLambdaEnv = {};
this.jobsCompletedMetricFiltersInitialized = false;
this.secrets = new secrets_1.Secrets(this, 'Secrets');
this.extraLambdaProps = {
vpc: this.props?.vpc,
vpcSubnets: this.props?.vpcSubnets,
allowPublicSubnet: this.props?.allowPublicSubnet,
securityGroups: this.lambdaSecurityGroups(),
layers: [],
};
this.connections = new aws_cdk_lib_1.aws_ec2.Connections({ securityGroups: this.extraLambdaProps.securityGroups });
this.createCertificateLayer(scope);
if (this.props?.providers) {
this.providers = this.props.providers;
}
else {
this.providers = [
new providers_1.CodeBuildRunnerProvider(this, 'CodeBuild'),
new providers_1.LambdaRunnerProvider(this, 'Lambda'),
new providers_1.FargateRunnerProvider(this, 'Fargate'),
];
}
if (this.providers.length == 0) {
throw new Error('At least one runner provider is required');
}
this.checkIntersectingLabels();
this.orchestrator = this.stateMachine(props);
this.webhook = new webhook_1.GithubWebhookHandler(this, 'Webhook Handler', {
orchestrator: this.orchestrator,
secrets: this.secrets,
access: this.props?.webhookAccess ?? access_1.LambdaAccess.lambdaUrl(),
providers: this.providers.reduce((acc, p) => {
acc[p.node.path] = p.labels;
return acc;
}, {}),
requireSelfHostedLabel: this.props?.requireSelfHostedLabel ?? true,
providerSelector: this.props?.providerSelector,
extraLambdaProps: this.extraLambdaProps,
extraLambdaEnv: this.extraLambdaEnv,
});
this.redeliverer = new webhook_redelivery_1.GithubWebhookRedelivery(this, 'Webhook Redelivery', {
secrets: this.secrets,
extraLambdaProps: this.extraLambdaProps,
extraLambdaEnv: this.extraLambdaEnv,
});
this.setupUrl = this.setupFunction();
this.statusFunction();
}
stateMachine(props) {
const tokenRetrieverTask = new aws_cdk_lib_1.aws_stepfunctions_tasks.LambdaInvoke(this, 'Get Runner Token', {
lambdaFunction: this.tokenRetriever(),
payloadResponseOnly: true,
resultPath: '$.runner',
});
let deleteFailedRunnerFunction = this.deleteFailedRunner();
const deleteFailedRunnerTask = new aws_cdk_lib_1.aws_stepfunctions_tasks.LambdaInvoke(this, 'Delete Failed Runner', {
lambdaFunction: deleteFailedRunnerFunction,
payloadResponseOnly: true,
resultPath: '$.delete',
payload: aws_cdk_lib_1.aws_stepfunctions.TaskInput.fromObject({
runnerName: aws_cdk_lib_1.aws_stepfunctions.JsonPath.stringAt('$$.Execution.Name'),
owner: aws_cdk_lib_1.aws_stepfunctions.JsonPath.stringAt('$.owner'),
repo: aws_cdk_lib_1.aws_stepfunctions.JsonPath.stringAt('$.repo'),
installationId: aws_cdk_lib_1.aws_stepfunctions.JsonPath.numberAt('$.installationId'),
error: aws_cdk_lib_1.aws_stepfunctions.JsonPath.objectAt('$.error'),
}),
});
deleteFailedRunnerTask.addRetry({
errors: [
'RunnerBusy',
],
interval: cdk.Duration.minutes(1),
backoffRate: 1,
maxAttempts: 60,
});
const idleReaper = this.idleReaper();
const queueIdleReaperTask = new aws_cdk_lib_1.aws_stepfunctions_tasks.SqsSendMessage(this, 'Queue Idle Reaper', {
queue: this.idleReaperQueue(idleReaper),
messageBody: aws_cdk_lib_1.aws_stepfunctions.TaskInput.fromObject({
executionArn: aws_cdk_lib_1.aws_stepfunctions.JsonPath.stringAt('$$.Execution.Id'),
runnerName: aws_cdk_lib_1.aws_stepfunctions.JsonPath.stringAt('$$.Execution.Name'),
owner: aws_cdk_lib_1.aws_stepfunctions.JsonPath.stringAt('$.owner'),
repo: aws_cdk_lib_1.aws_stepfunctions.JsonPath.stringAt('$.repo'),
installationId: aws_cdk_lib_1.aws_stepfunctions.JsonPath.numberAt('$.installationId'),
maxIdleSeconds: (props?.idleTimeout ?? cdk.Duration.minutes(5)).toSeconds(),
}),
resultPath: aws_cdk_lib_1.aws_stepfunctions.JsonPath.DISCARD,
});
const providerChooser = new aws_cdk_lib_1.aws_stepfunctions.Choice(this, 'Choose provider');
for (const provider of this.providers) {
const providerTask = provider.getStepFunctionTask({
runnerTokenPath: aws_cdk_lib_1.aws_stepfunctions.JsonPath.stringAt('$.runner.token'),
runnerNamePath: aws_cdk_lib_1.aws_stepfunctions.JsonPath.stringAt('$$.Execution.Name'),
githubDomainPath: aws_cdk_lib_1.aws_stepfunctions.JsonPath.stringAt('$.runner.domain'),
ownerPath: aws_cdk_lib_1.aws_stepfunctions.JsonPath.stringAt('$.owner'),
repoPath: aws_cdk_lib_1.aws_stepfunctions.JsonPath.stringAt('$.repo'),
registrationUrl: aws_cdk_lib_1.aws_stepfunctions.JsonPath.stringAt('$.runner.registrationUrl'),
labelsPath: aws_cdk_lib_1.aws_stepfunctions.JsonPath.stringAt('$.labels'),
});
providerChooser.when(aws_cdk_lib_1.aws_stepfunctions.Condition.and(aws_cdk_lib_1.aws_stepfunctions.Condition.stringEquals('$.provider', provider.node.path)), providerTask, {
comment: `Labels: ${provider.labels.join(', ')}`,
});
}
providerChooser.otherwise(new aws_cdk_lib_1.aws_stepfunctions.Succeed(this, 'Unknown label'));
const runProviders = new aws_cdk_lib_1.aws_stepfunctions.Parallel(this, 'Run Providers').branch(new aws_cdk_lib_1.aws_stepfunctions.Parallel(this, 'Error Handler').branch(
// we get a token for every retry because the token can expire faster than the job can timeout
tokenRetrieverTask.next(providerChooser)).addCatch(
// delete runner on failure as it won't remove itself and there is a limit on the number of registered runners
deleteFailedRunnerTask, {
resultPath: '$.error',
}));
if (props?.retryOptions?.retry ?? true) {
const interval = props?.retryOptions?.interval ?? cdk.Duration.minutes(1);
const maxAttempts = props?.retryOptions?.maxAttempts ?? 23;
const backoffRate = props?.retryOptions?.backoffRate ?? 1.3;
const totalSeconds = interval.toSeconds() * backoffRate ** maxAttempts / (backoffRate - 1);
if (totalSeconds >= cdk.Duration.days(1).toSeconds()) {
// https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/about-self-hosted-runners#usage-limits
// "Job queue time - Each job for self-hosted runners can be queued for a maximum of 24 hours. If a self-hosted runner does not start executing the job within this limit, the job is terminated and fails to complete."
aws_cdk_lib_1.Annotations.of(this).addWarning(`Total retry time is greater than 24 hours (${Math.floor(totalSeconds / 60 / 60)} hours). Jobs expire after 24 hours so it would be a waste of resources to retry further.`);
}
runProviders.addRetry({
interval,
maxAttempts,
backoffRate,
// we retry on everything
// deleted idle runners will also fail, but the reaper will stop this step function to avoid endless retries
});
}
let logOptions;
if (this.props?.logOptions) {
this.stateMachineLogGroup = new aws_cdk_lib_1.aws_logs.LogGroup(this, 'Logs', {
logGroupName: props?.logOptions?.logGroupName,
retention: props?.logOptions?.logRetention ?? aws_cdk_lib_1.aws_logs.RetentionDays.ONE_MONTH,
removalPolicy: cdk.RemovalPolicy.DESTROY,
});
logOptions = {
destination: this.stateMachineLogGroup,
includeExecutionData: props?.logOptions?.includeExecutionData ?? true,
level: props?.logOptions?.level ?? aws_cdk_lib_1.aws_stepfunctions.LogLevel.ALL,
};
}
const stateMachine = new aws_cdk_lib_1.aws_stepfunctions.StateMachine(this, 'Runner Orchestrator', {
definitionBody: aws_cdk_lib_1.aws_stepfunctions.DefinitionBody.fromChainable(queueIdleReaperTask.next(runProviders)),
logs: logOptions,
});
stateMachine.grantRead(idleReaper);
stateMachine.grantExecution(idleReaper, 'states:StopExecution');
for (const provider of this.providers) {
provider.grantStateMachine(stateMachine);
}
return stateMachine;
}
tokenRetriever() {
const func = new token_retriever_function_1.TokenRetrieverFunction(this, 'token-retriever', {
description: 'Get token from GitHub Actions used to start new self-hosted runner',
environment: {
GITHUB_SECRET_ARN: this.secrets.github.secretArn,
GITHUB_PRIVATE_KEY_SECRET_ARN: this.secrets.githubPrivateKey.secretArn,
...this.extraLambdaEnv,
},
timeout: cdk.Duration.seconds(30),
logGroup: (0, utils_1.singletonLogGroup)(this, utils_1.SingletonLogType.ORCHESTRATOR),
loggingFormat: aws_cdk_lib_1.aws_lambda.LoggingFormat.JSON,
...this.extraLambdaProps,
});
this.secrets.github.grantRead(func);
this.secrets.githubPrivateKey.grantRead(func);
return func;
}
deleteFailedRunner() {
const func = new delete_failed_runner_function_1.DeleteFailedRunnerFunction(this, 'delete-runner', {
description: 'Delete failed GitHub Actions runner on error',
environment: {
GITHUB_SECRET_ARN: this.secrets.github.secretArn,
GITHUB_PRIVATE_KEY_SECRET_ARN: this.secrets.githubPrivateKey.secretArn,
...this.extraLambdaEnv,
},
timeout: cdk.Duration.seconds(30),
logGroup: (0, utils_1.singletonLogGroup)(this, utils_1.SingletonLogType.ORCHESTRATOR),
loggingFormat: aws_cdk_lib_1.aws_lambda.LoggingFormat.JSON,
...this.extraLambdaProps,
});
this.secrets.github.grantRead(func);
this.secrets.githubPrivateKey.grantRead(func);
return func;
}
statusFunction() {
const statusFunction = new status_function_1.StatusFunction(this, 'status', {
description: 'Provide user with status about self-hosted GitHub Actions runners',
environment: {
WEBHOOK_SECRET_ARN: this.secrets.webhook.secretArn,
GITHUB_SECRET_ARN: this.secrets.github.secretArn,
GITHUB_PRIVATE_KEY_SECRET_ARN: this.secrets.githubPrivateKey.secretArn,
SETUP_SECRET_ARN: this.secrets.setup.secretArn,
WEBHOOK_URL: this.webhook.url,
WEBHOOK_HANDLER_ARN: this.webhook.handler.latestVersion.functionArn,
STEP_FUNCTION_ARN: this.orchestrator.stateMachineArn,
STEP_FUNCTION_LOG_GROUP: this.stateMachineLogGroup?.logGroupName ?? '',
SETUP_FUNCTION_URL: this.setupUrl,
...this.extraLambdaEnv,
},
timeout: cdk.Duration.minutes(3),
logGroup: (0, utils_1.singletonLogGroup)(this, utils_1.SingletonLogType.SETUP),
loggingFormat: aws_cdk_lib_1.aws_lambda.LoggingFormat.JSON,
...this.extraLambdaProps,
});
const providers = this.providers.flatMap(provider => {
const status = provider.status(statusFunction);
// Composite providers return an array, regular providers return a single status
return Array.isArray(status) ? status : [status];
});
// expose providers as stack metadata as it's too big for Lambda environment variables
// specifically integration testing got an error because lambda update request was >5kb
const stack = cdk.Stack.of(this);
const f = statusFunction.node.defaultChild;
f.addPropertyOverride('Environment.Variables.LOGICAL_ID', f.logicalId);
f.addPropertyOverride('Environment.Variables.STACK_NAME', stack.stackName);
f.addMetadata('providers', providers);
statusFunction.addToRolePolicy(new aws_cdk_lib_1.aws_iam.PolicyStatement({
actions: ['cloudformation:DescribeStackResource'],
resources: [stack.stackId],
}));
this.secrets.webhook.grantRead(statusFunction);
this.secrets.github.grantRead(statusFunction);
this.secrets.githubPrivateKey.grantRead(statusFunction);
this.secrets.setup.grantRead(statusFunction);
this.orchestrator.grantRead(statusFunction);
new cdk.CfnOutput(this, 'status command', {
value: `aws --region ${stack.region} lambda invoke --function-name ${statusFunction.functionName} status.json`,
});
const access = this.props?.statusAccess ?? access_1.LambdaAccess.noAccess();
const url = access.bind(this, 'status access', statusFunction);
if (url !== '') {
new cdk.CfnOutput(this, 'status url', {
value: url,
});
}
}
setupFunction() {
const setupFunction = new setup_function_1.SetupFunction(this, 'setup', {
description: 'Setup GitHub Actions integration with self-hosted runners',
environment: {
SETUP_SECRET_ARN: this.secrets.setup.secretArn,
WEBHOOK_SECRET_ARN: this.secrets.webhook.secretArn,
GITHUB_SECRET_ARN: this.secrets.github.secretArn,
GITHUB_PRIVATE_KEY_SECRET_ARN: this.secrets.githubPrivateKey.secretArn,
WEBHOOK_URL: this.webhook.url,
...this.extraLambdaEnv,
},
timeout: cdk.Duration.minutes(3),
logGroup: (0, utils_1.singletonLogGroup)(this, utils_1.SingletonLogType.SETUP),
loggingFormat: aws_cdk_lib_1.aws_lambda.LoggingFormat.JSON,
...this.extraLambdaProps,
});
// this.secrets.webhook.grantRead(setupFunction);
this.secrets.webhook.grantWrite(setupFunction);
this.secrets.github.grantRead(setupFunction);
this.secrets.github.grantWrite(setupFunction);
// this.secrets.githubPrivateKey.grantRead(setupFunction);
this.secrets.githubPrivateKey.grantWrite(setupFunction);
this.secrets.setup.grantRead(setupFunction);
this.secrets.setup.grantWrite(setupFunction);
const access = this.props?.setupAccess ?? access_1.LambdaAccess.lambdaUrl();
return access.bind(this, 'setup access', setupFunction);
}
checkIntersectingLabels() {
// this "algorithm" is very inefficient, but good enough for the tiny datasets we expect
for (const p1 of this.providers) {
for (const p2 of this.providers) {
if (p1 == p2) {
continue;
}
if (p1.labels.every(l => p2.labels.includes(l))) {
if (p2.labels.every(l => p1.labels.includes(l))) {
throw new Error(`Both ${p1.node.path} and ${p2.node.path} use the same labels [${p1.labels.join(', ')}]`);
}
aws_cdk_lib_1.Annotations.of(p1).addWarning(`Labels [${p1.labels.join(', ')}] intersect with another provider (${p2.node.path} -- [${p2.labels.join(', ')}]). If a workflow specifies the labels [${p1.labels.join(', ')}], it is not guaranteed which provider will be used. It is recommended you do not use intersecting labels`);
}
}
}
}
idleReaper() {
return new idle_runner_repear_function_1.IdleRunnerRepearFunction(this, 'Idle Reaper', {
description: 'Stop idle GitHub runners to avoid paying for runners when the job was already canceled',
environment: {
GITHUB_SECRET_ARN: this.secrets.github.secretArn,
GITHUB_PRIVATE_KEY_SECRET_ARN: this.secrets.githubPrivateKey.secretArn,
...this.extraLambdaEnv,
},
logGroup: (0, utils_1.singletonLogGroup)(this, utils_1.SingletonLogType.ORCHESTRATOR),
loggingFormat: aws_cdk_lib_1.aws_lambda.LoggingFormat.JSON,
timeout: cdk.Duration.minutes(5),
...this.extraLambdaProps,
});
}
idleReaperQueue(reaper) {
// see this comment to understand why it's a queue that's out of the step function
// https://github.com/CloudSnorkel/cdk-github-runners/pull/314#issuecomment-1528901192
const queue = new aws_cdk_lib_1.aws_sqs.Queue(this, 'Idle Reaper Queue', {
deliveryDelay: cdk.Duration.minutes(10),
visibilityTimeout: cdk.Duration.minutes(10),
});
reaper.addEventSource(new aws_cdk_lib_1.aws_lambda_event_sources.SqsEventSource(queue, {
reportBatchItemFailures: true,
maxBatchingWindow: cdk.Duration.minutes(1),
}));
this.secrets.github.grantRead(reaper);
this.secrets.githubPrivateKey.grantRead(reaper);
return queue;
}
lambdaSecurityGroups() {
if (!this.props?.vpc) {
if (this.props?.securityGroup) {
cdk.Annotations.of(this).addWarning('securityGroup is specified, but vpc is not. securityGroup will be ignored');
}
if (this.props?.securityGroups) {
cdk.Annotations.of(this).addWarning('securityGroups is specified, but vpc is not. securityGroups will be ignored');
}
return undefined;
}
if (this.props.securityGroups) {
if (this.props.securityGroup) {
cdk.Annotations.of(this).addWarning('Both securityGroup and securityGroups are specified. securityGroup will be ignored');
}
return this.props.securityGroups;
}
if (this.props.securityGroup) {
return [this.props.securityGroup];
}
return [new aws_cdk_lib_1.aws_ec2.SecurityGroup(this, 'Management Lambdas Security Group', { vpc: this.props.vpc })];
}
/**
* Extracts all unique IRunnerProvider instances from providers and composite providers (one level only).
* Uses a Set to ensure we don't process the same provider twice, even if it's used in multiple composites.
*
* @returns Set of unique IRunnerProvider instances
*/
extractUniqueSubProviders() {
const seen = new Set();
for (const provider of this.providers) {
// instanceof doesn't really work in CDK so use this hack instead
if ('logGroup' in provider) {
// Regular provider
seen.add(provider);
}
else {
// Composite provider - access the providers field
for (const subProvider of provider.providers) {
seen.add(subProvider);
}
}
}
return seen;
}
/**
* Creates a Lambda layer with certificates if extraCertificates is specified.
*/
createCertificateLayer(scope) {
if (!this.props?.extraCertificates) {
return;
}
const certificateFiles = (0, utils_1.discoverCertificateFiles)(this.props.extraCertificates);
// Concatenate all certificates into a single file for NODE_EXTRA_CA_CERTS
let combinedCertContent = '';
for (const certFile of certificateFiles) {
const certContent = fs.readFileSync(certFile, 'utf8');
combinedCertContent += certContent;
// Ensure proper PEM format with newline between certificates
if (!certContent.endsWith('\n')) {
combinedCertContent += '\n';
}
}
// Create a temporary directory, write the certificate file, create asset, then delete temp dir
const workdir = fs.mkdtempSync(path.join(os.tmpdir(), 'certificate-layer-'));
try {
const certPath = path.join(workdir, 'certs.pem');
fs.writeFileSync(certPath, combinedCertContent);
// Set environment variable and create layer
this.extraLambdaEnv.NODE_EXTRA_CA_CERTS = '/opt/certs.pem';
this.extraLambdaProps.layers.push(new aws_cdk_lib_1.aws_lambda.LayerVersion(scope, 'Certificate Layer', {
description: 'Layer containing GitHub Enterprise Server certificate(s) for cdk-github-runners',
code: aws_cdk_lib_1.aws_lambda.Code.fromAsset(workdir),
}));
}
finally {
// Calling `fromAsset()` has copied files to the assembly, so we can delete the temporary directory.
fs.rmSync(workdir, { recursive: true, force: true });
}
}
/**
* Metric for the number of GitHub Actions jobs completed. It has `ProviderLabels` and `Status` dimensions. The status can be one of "Succeeded", "SucceededWithIssues", "Failed", "Canceled", "Skipped", or "Abandoned".
*
* **WARNING:** this method creates a metric filter for each provider. Each metric has a status dimension with six possible values. These resources may incur cost.
*/
metricJobCompleted(props) {
if (!this.jobsCompletedMetricFiltersInitialized) {
// we can't use logs.FilterPattern.spaceDelimited() because it has no support for ||
// status list taken from https://github.com/actions/runner/blob/be9632302ceef50bfb36ea998cea9c94c75e5d4d/src/Sdk/DTWebApi/WebApi/TaskResult.cs
// we need "..." for Lambda that prefixes some extra data to log lines
const pattern = aws_cdk_lib_1.aws_logs.FilterPattern.literal('[..., marker = "CDKGHA", job = "JOB", done = "DONE", labels, status = "Succeeded" || status = "SucceededWithIssues" || status = "Failed" || status = "Canceled" || status = "Skipped" || status = "Abandoned"]');
// Extract all unique sub-providers from regular and composite providers
// Build a set first to avoid filtering the same log twice
for (const p of this.extractUniqueSubProviders()) {
const metricFilter = p.logGroup.addMetricFilter(`${p.logGroup.node.id} filter`, {
metricNamespace: 'GitHubRunners',
metricName: 'JobCompleted',
filterPattern: pattern,
metricValue: '1',
// can't with dimensions -- defaultValue: 0,
dimensions: {
ProviderLabels: '$labels',
Status: '$status',
},
});
if (metricFilter.node.defaultChild instanceof aws_cdk_lib_1.aws_logs.CfnMetricFilter) {
metricFilter.node.defaultChild.addPropertyOverride('MetricTransformations.0.Unit', 'Count');
}
else {
aws_cdk_lib_1.Annotations.of(metricFilter).addWarning('Unable to set metric filter Unit to Count');
}
}
this.jobsCompletedMetricFiltersInitialized = true;
}
return new aws_cdk_lib_1.aws_cloudwatch.Metric({
namespace: 'GitHubRunners',
metricName: 'JobsCompleted',
unit: aws_cdk_lib_1.aws_cloudwatch.Unit.COUNT,
statistic: aws_cdk_lib_1.aws_cloudwatch.Stats.SUM,
...props,
}).attachTo(this);
}
/**
* Metric for successful executions.
*
* A successful execution doesn't always mean a runner was started. It can be successful even without any label matches.
*
* A successful runner doesn't mean the job it executed was successful. For that, see {@link metricJobCompleted}.
*/
metricSucceeded(props) {
return this.orchestrator.metricSucceeded(props);
}
/**
* Metric for failed runner executions.
*
* A failed runner usually means the runner failed to start and so a job was never executed. It doesn't necessarily mean the job was executed and failed. For that, see {@link metricJobCompleted}.
*/
metricFailed(props) {
return this.orchestrator.metricFailed(props);
}
/**
* Metric for the interval, in milliseconds, between the time the execution starts and the time it closes. This time may be longer than the time the runner took.
*/
metricTime(props) {
return this.orchestrator.metricTime(props);
}
/**
* Creates a topic for notifications when a runner image build fails.
*
* Runner images are rebuilt every week by default. This provides the latest GitHub Runner version and software updates.
*
* If you want to be sure you are using the latest runner version, you can use this topic to be notified when a build fails.
*/
failedImageBuildsTopic() {
const topic = new aws_cdk_lib_1.aws_sns.Topic(this, 'Failed Runner Image Builds');
const stack = cdk.Stack.of(this);
cdk.Aspects.of(stack).add(new providers_1.CodeBuildImageBuilderFailedBuildNotifier(topic));
cdk.Aspects.of(stack).add(new providers_1.AwsImageBuilderFailedBuildNotifier(providers_1.AwsImageBuilderFailedBuildNotifier.createFilteringTopic(this, topic)));
return topic;
}
/**
* Creates CloudWatch Logs Insights saved queries that can be used to debug issues with the runners.
*
* * "Webhook errors" helps diagnose configuration issues with GitHub integration
* * "Ignored webhook" helps understand why runners aren't started
* * "Ignored jobs based on labels" helps debug label matching issues
* * "Webhook started runners" helps understand which runners were started
*/
createLogsInsightsQueries() {
new aws_cdk_lib_1.aws_logs.QueryDefinition(this, 'Webhook errors', {
queryDefinitionName: 'GitHub Runners/Webhook errors',
logGroups: [this.webhook.handler.logGroup],
queryString: new aws_cdk_lib_1.aws_logs.QueryString({
filterStatements: [
`strcontains(@logStream, "${this.webhook.handler.functionName}")`,
'level = "ERROR"',
],
sort: '@timestamp desc',
limit: 100,
}),
});
new aws_cdk_lib_1.aws_logs.QueryDefinition(this, 'Orchestration errors', {
queryDefinitionName: 'GitHub Runners/Orchestration errors',
logGroups: [(0, utils_1.singletonLogGroup)(this, utils_1.SingletonLogType.ORCHESTRATOR)],
queryString: new aws_cdk_lib_1.aws_logs.QueryString({
filterStatements: [
'level = "ERROR"',
],
sort: '@timestamp desc',
limit: 100,
}),
});
new aws_cdk_lib_1.aws_logs.QueryDefinition(this, 'Runner image build errors', {
queryDefinitionName: 'GitHub Runners/Runner image build errors',
logGroups: [(0, utils_1.singletonLogGroup)(this, utils_1.SingletonLogType.RUNNER_IMAGE_BUILD)],
queryString: new aws_cdk_lib_1.aws_logs.QueryString({
filterStatements: [
'strcontains(message, "error") or strcontains(message, "ERROR") or strcontains(message, "Error") or level = "ERROR"',
],
sort: '@timestamp desc',
limit: 100,
}),
});
new aws_cdk_lib_1.aws_logs.QueryDefinition(this, 'Ignored webhooks', {
queryDefinitionName: 'GitHub Runners/Ignored webhooks',
logGroups: [this.webhook.handler.logGroup],
queryString: new aws_cdk_lib_1.aws_logs.QueryString({
fields: ['@timestamp', 'message.notice'],
filterStatements: [
`strcontains(@logStream, "${this.webhook.handler.functionName}")`,
'strcontains(message.notice, "Ignoring")',
],
sort: '@timestamp desc',
limit: 100,
}),
});
new aws_cdk_lib_1.aws_logs.QueryDefinition(this, 'Ignored jobs based on labels', {
queryDefinitionName: 'GitHub Runners/Ignored jobs based on labels',
logGroups: [this.webhook.handler.logGroup],
queryString: new aws_cdk_lib_1.aws_logs.QueryString({
fields: ['@timestamp', 'message.notice'],
filterStatements: [
`strcontains(@logStream, "${this.webhook.handler.functionName}")`,
'strcontains(message.notice, "Ignoring labels")',
],
sort: '@timestamp desc',
limit: 100,
}),
});
new aws_cdk_lib_1.aws_logs.QueryDefinition(this, 'Webhook started runners', {
queryDefinitionName: 'GitHub Runners/Webhook started runners',
logGroups: [this.webhook.handler.logGroup],
queryString: new aws_cdk_lib_1.aws_logs.QueryString({
fields: ['@timestamp', 'message.sfnInput.jobUrl', 'message.sfnInput.jobLabels', 'message.sfnInput.labels', 'message.sfnInput.provider'],
filterStatements: [
`strcontains(@logStream, "${this.webhook.handler.functionName}")`,
'message.sfnInput.jobUrl like /http.*/',
],
sort: '@timestamp desc',
limit: 100,
}),
});
new aws_cdk_lib_1.aws_logs.QueryDefinition(this, 'Webhook redeliveries', {
queryDefinitionName: 'GitHub Runners/Webhook redeliveries',
logGroups: [this.redeliverer.handler.logGroup],
queryString: new aws_cdk_lib_1.aws_logs.QueryString({
fields: ['@timestamp', 'message.notice', 'message.deliveryId', 'message.guid'],
filterStatements: [
'isPresent(message.deliveryId)',
],
sort: '@timestamp desc',
limit: 100,
}),
});
}
}
exports.GitHubRunners = GitHubRunners;
_a = JSII_RTTI_SYMBOL_1;
GitHubRunners[_a] = { fqn: "@cloudsnorkel/cdk-github-runners.GitHubRunners", version: "0.14.21" };
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicnVubmVyLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vc3JjL3J1bm5lci50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiOzs7OztBQUFBLHlCQUF5QjtBQUN6Qix5QkFBeUI7QUFDekIsNkJBQTZCO0FBQzdCLG1DQUFtQztBQUNuQyw2Q0FZcUI7QUFDckIsMkNBQXVDO0FBQ3ZDLHFDQUF3QztBQUN4QyxtRkFBNkU7QUFDN0UsK0VBQXlFO0FBQ3pFLDJDQVNxQjtBQUNyQix1Q0FBb0M7QUFDcEMscURBQWlEO0FBQ2pELHVEQUFtRDtBQUNuRCx5RUFBb0U7QUFDcEUsbUNBQXdGO0FBQ3hGLHVDQUFpRDtBQUNqRCw2REFBK0Q7QUEyTS9EOzs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7R0F1Q0c7QUFDSCxNQUFhLGFBQWMsU0FBUSxzQkFBUztJQTJCMUMsWUFBWSxLQUFnQixFQUFFLEVBQVUsRUFBVyxLQUEwQjtRQUMzRSxLQUFLLENBQUMsS0FBSyxFQUFFLEVBQUUsQ0FBQyxDQUFDO1FBRGdDLFVBQUssR0FBTCxLQUFLLENBQXFCO1FBTDVELG1CQUFjLEdBQTRCLEVBQUUsQ0FBQztRQUd0RCwwQ0FBcUMsR0FBRyxLQUFLLENBQUM7UUFLcEQsSUFBSSxDQUFDLE9BQU8sR0FBRyxJQUFJLGlCQUFPLENBQUMsSUFBSSxFQUFFLFNBQVMsQ0FBQyxDQUFDO1FBRTVDLElBQUksQ0FBQyxnQkFBZ0IsR0FBRztZQUN0QixHQUFHLEVBQUUsSUFBSSxDQUFDLEtBQUssRUFBRSxHQUFHO1lBQ3BCLFVBQVUsRUFBRSxJQUFJLENBQUMsS0FBSyxFQUFFLFVBQVU7WUFDbEMsaUJBQWlCLEVBQUUsSUFBSSxDQUFDLEtBQUssRUFBRSxpQkFBaUI7WUFDaEQsY0FBYyxFQUFFLElBQUksQ0FBQyxvQkFBb0IsRUFBRTtZQUMzQyxNQUFNLEVBQUUsRUFBRTtTQUNYLENBQUM7UUFDRixJQUFJLENBQUMsV0FBVyxHQUFHLElBQUkscUJBQUcsQ0FBQyxXQUFXLENBQUMsRUFBRSxjQUFjLEVBQUUsSUFBSSxDQUFDLGdCQUFnQixDQUFDLGNBQWMsRUFBRSxDQUFDLENBQUM7UUFFakcsSUFBSSxDQUFDLHNCQUFzQixDQUFDLEtBQUssQ0FBQyxDQUFDO1FBRW5DLElBQUksSUFBSSxDQUFDLEtBQUssRUFBRSxTQUFTLEVBQUUsQ0FBQztZQUMxQixJQUFJLENBQUMsU0FBUyxHQUFHLElBQUksQ0FBQyxLQUFLLENBQUMsU0FBUyxDQUFDO1FBQ3hDLENBQUM7YUFBTSxDQUFDO1lBQ04sSUFBSSxDQUFDLFNBQVMsR0FBRztnQkFDZixJQUFJLG1DQUF1QixDQUFDLElBQUksRUFBRSxXQUFXLENBQUM7Z0JBQzlDLElBQUksZ0NBQW9CLENBQUMsSUFBSSxFQUFFLFFBQVEsQ0FBQztnQkFDeEMsSUFBSSxpQ0FBcUIsQ0FBQyxJQUFJLEVBQUUsU0FBUyxDQUFDO2FBQzNDLENBQUM7UUFDSixDQUFDO1FBRUQsSUFBSSxJQUFJLENBQUMsU0FBUyxDQUFDLE1BQU0sSUFBSSxDQUFDLEVBQUUsQ0FBQztZQUMvQixNQUFNLElBQUksS0FBSyxDQUFDLDBDQUEwQyxDQUFDLENBQUM7UUFDOUQsQ0FBQztRQUVELElBQUksQ0FBQyx1QkFBdUIsRUFBRSxDQUFDO1FBRS9CLElBQUksQ0FBQyxZQUFZLEdBQUcsSUFBSSxDQUFDLFlBQVksQ0FBQyxLQUFLLENBQUMsQ0FBQztRQUM3QyxJQUFJLENBQUMsT0FBTyxHQUFHLElBQUksOEJBQW9CLENBQUMsSUFBSSxFQUFFLGlCQUFpQixFQUFFO1lBQy9ELFlBQVksRUFBRSxJQUFJLENBQUMsWUFBWTtZQUMvQixPQUFPLEVBQUUsSUFBSSxDQUFDLE9BQU87WUFDckIsTUFBTSxFQUFFLElBQUksQ0FBQyxLQUFLLEVBQUUsYUFBYSxJQUFJLHFCQUFZLENBQUMsU0FBUyxFQUFFO1lBQzdELFNBQVMsRUFBRSxJQUFJLENBQUMsU0FBUyxDQUFDLE1BQU0sQ0FBMkIsQ0FBQyxHQUFHLEVBQUUsQ0FBQyxFQUFFLEVBQUU7Z0JBQ3BFLEdBQUcsQ0FBQyxDQUFDLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxHQUFHLENBQUMsQ0FBQyxNQUFNLENBQUM7Z0JBQzVCLE9BQU8sR0FBRyxDQUFDO1lBQ2IsQ0FBQyxFQUFFLEVBQUUsQ0FBQztZQUNOLHNCQUFzQixFQUFFLElBQUksQ0FBQyxLQUFLLEVBQUUsc0JBQXNCLElBQUksSUFBSTtZQUNsRSxnQkFBZ0IsRUFBRSxJQUFJLENBQUMsS0FBSyxFQUFFLGdCQUFnQjtZQUM5QyxnQkFBZ0IsRUFBRSxJQUFJLENBQUMsZ0JBQWdCO1lBQ3ZDLGNBQWMsRUFBRSxJQUFJLENBQUMsY0FBYztTQUNwQyxDQUFDLENBQUM7UUFDSCxJQUFJLENBQUMsV0FBVyxHQUFHLElBQUksNENBQXVCLENBQUMsSUFBSSxFQUFFLG9CQUFvQixFQUFFO1lBQ3pFLE9BQU8sRUFBRSxJQUFJLENBQUMsT0FBTztZQUNyQixnQkFBZ0IsRUFBRSxJQUFJLENBQUMsZ0JBQWdCO1lBQ3ZDLGNBQWMsRUFBRSxJQUFJLENBQUMsY0FBYztTQUNwQyxDQUFDLENBQUM7UUFFSCxJQUFJLENBQUMsUUFBUSxHQUFHLElBQUksQ0FBQyxhQUFhLEVBQUUsQ0FBQztRQUNyQyxJQUFJLENBQUMsY0FBYyxFQUFFLENBQUM7SUFDeEIsQ0FBQztJQUVPLFlBQVksQ0FBQyxLQUEwQjtRQUM3QyxNQUFNLGtCQUFrQixHQUFHLElBQUkscUNBQW1CLENBQUMsWUFBWSxDQUM3RCxJQUFJLEVBQ0osa0JBQWtCLEVBQ2xCO1lBQ0UsY0FBYyxFQUFFLElBQUksQ0FBQyxjQUFjLEVBQUU7WUFDckMsbUJBQW1CLEVBQUUsSUFBSTtZQUN6QixVQUFVLEVBQUUsVUFBVTtTQUN2QixDQUNGLENBQUM7UUFFRixJQUFJLDBCQUEwQixHQUFHLElBQUksQ0FBQyxrQkFBa0IsRUFBRSxDQUFDO1FBQzNELE1BQU0sc0JBQXNCLEdBQUcsSUFBSSxxQ0FBbUIsQ0FBQyxZQUFZLENBQ2pFLElBQUksRUFDSixzQkFBc0IsRUFDdEI7WUFDRSxjQUFjLEVBQUUsMEJBQTBCO1lBQzFDLG1CQUFtQixFQUFFLElBQUk7WUFDekIsVUFBVSxFQUFFLFVBQVU7WUFDdEIsT0FBTyxFQUFFLCtCQUFhLENBQUMsU0FBUyxDQUFDLFVBQVUsQ0FBQztnQkFDMUMsVUFBVSxFQUFFLCtCQUFhLENBQUMsUUFBUSxDQUFDLFFBQVEsQ0FBQyxtQkFBbUIsQ0FBQztnQkFDaEUsS0FBSyxFQUFFLCtCQUFhLENBQUMsUUFBUSxDQUFDLFFBQVEsQ0FBQyxTQUFTLENBQUM7Z0JBQ2pELElBQUksRUFBRSwrQkFBYSxDQUFDLFFBQVEsQ0FBQyxRQUFRLENBQUMsUUFBUSxDQUFDO2dCQUMvQyxjQUFjLEVBQUUsK0JBQWEsQ0FBQyxRQUFRLENBQUMsUUFBUSxDQUFDLGtCQUFrQixDQUFDO2dCQUNuRSxLQUFLLEVBQUUsK0JBQWEsQ0FBQyxRQUFRLENBQUMsUUFBUSxDQUFDLFNBQVMsQ0FBQzthQUNsRCxDQUFDO1NBQ0gsQ0FDRixDQUFDO1FBQ0Ysc0JBQXNCLENBQUMsUUFBUSxDQUFDO1lBQzlCLE1BQU0sRUFBRTtnQkFDTixZQUFZO2FBQ2I7WUFDRCxRQUFRLEVBQUUsR0FBRyxDQUFDLFFBQVEsQ0FBQyxPQUFPLENBQUMsQ0FBQyxDQUFDO1lBQ2pDLFdBQVcsRUFBRSxDQUFDO1lBQ2QsV0FBVyxFQUFFLEVBQUU7U0FDaEIsQ0FBQyxDQUFDO1FBRUgsTUFBTSxVQUFVLEdBQUcsSUFBSSxDQUFDLFVBQVUsRUFBRSxDQUFDO1FBQ3JDLE1BQU0sbUJBQW1CLEdBQUcsSUFBSSxxQ0FBbUIsQ0FBQyxjQUFjLENBQUMsSUFBSSxFQUFFLG1CQUFtQixFQUFFO1lBQzVGLEtBQUssRUFBRSxJQUFJLENBQUMsZUFBZSxDQUFDLFVBQVUsQ0FBQztZQUN2QyxXQUFXLEVBQUUsK0JBQWEsQ0FBQyxTQUFTLENBQUMsVUFBVSxDQUFDO2dCQUM5QyxZQUFZLEVBQUUsK0JBQWEsQ0FBQyxRQUFRLENBQUMsUUFBUSxDQUFDLGlCQUFpQixDQUFDO2dCQUNoRSxVQUFVLEVBQUUsK0JBQWEsQ0FBQyxRQUFRLENBQUMsUUFBUSxDQUFDLG1CQUFtQixDQUFDO2dCQUNoRSxLQUFLLEVBQUUsK0JBQWEsQ0FBQyxRQUFRLENBQUMsUUFBUSxDQUFDLFNBQVMsQ0FBQztnQkFDakQsSUFBSSxFQUFFLCtCQUFhLENBQUMsUUFBUSxDQUFDLFFBQVEsQ0FBQyxRQUFRLENBQUM7Z0JBQy9DLGNBQWMsRUFBRSwrQkFBYSxDQUFDLFFBQVEsQ0FBQyxRQUFRLENBQUMsa0JBQWtCLENBQUM7Z0JBQ25FLGNBQWMsRUFBRSxDQUFDLEtBQUssRUFBRSxXQUFXLElBQUksR0FBRyxDQUFDLFFBQVEsQ0FBQyxPQUFPLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxTQUFTLEVBQUU7YUFDNUUsQ0FBQztZQUNGLFVBQVUsRUFBRSwrQkFBYSxDQUFDLFFBQVEsQ0FBQyxPQUFPO1NBQzNDLENBQUMsQ0FBQztRQUVILE1BQU0sZUFBZSxHQUFHLElBQUksK0JBQWEsQ0FBQyxNQUFNLENBQUMsSUFBSSxFQUFFLGlCQUFpQixDQUFDLENBQUM7UUFDMUUsS0FBSyxNQUFNLFFBQVEsSUFBSSxJQUFJLENBQUMsU0FBUyxFQUFFLENBQUM7WUFDdEMsTUFBTSxZQUFZLEdBQUcsUUFBUSxDQUFDLG1CQUFtQixDQUMvQztnQkFDRSxlQUFlLEVBQUUsK0JBQWEsQ0FBQyxRQUFRLENBQUMsUUFBUSxDQUFDLGdCQUFnQixDQUFDO2dCQUNsRSxjQUFjLEVBQUUsK0JBQWEsQ0FBQyxRQUFRLENBQUMsUUFBUSxDQUFDLG1CQUFtQixDQUFDO2dCQUNwRSxnQkFBZ0IsRUFBRSwrQkFBYSxDQUFDLFFBQVEsQ0FBQyxRQUFRLENBQUMsaUJBQWlCLENBQUM7Z0JBQ3BFLFNBQVMsRUFBRSwrQkFBYSxDQUFDLFFBQVEsQ0FBQyxRQUFRLENBQUMsU0FBUyxDQUFDO2dCQUNyRCxRQUFRLEVBQUUsK0JBQWEsQ0FBQyxRQUFRLENBQUMsUUFBUSxDQUFDLFFBQVEsQ0FBQztnQkFDbkQsZUFBZSxFQUFFLCtCQUFhLENBQUMsUUFBUSxDQUFDLFFBQVEsQ0FBQywwQkFBMEIsQ0FBQztnQkFDNUUsVUFBVSxFQUFFLCtCQUFhLENBQUMsUUFBUSxDQUFDLFFBQVEsQ0FBQyxVQUFVLENBQUM7YUFDeEQsQ0FDRixDQUFDO1lBQ0YsZUFBZSxDQUFDLElBQUksQ0FDbEIsK0JBQWEsQ0FBQyxTQUFTLENBQUMsR0FBRyxDQUN6QiwrQkFBYSxDQUFDLFNBQVMsQ0FBQyxZQUFZLENBQUMsWUFBWSxFQUFFLFFBQVEsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLENBQ3ZFLEVBQ0QsWUFBWSxFQUNaO2dCQUNFLE9BQU8sRUFBRSxXQUFXLFFBQVEsQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxFQUFFO2FBQ2pELENBQ0YsQ0FBQztRQUNKLENBQUM7UUFFRCxlQUFlLENBQUMsU0FBUyxDQUFDLElBQUksK0JBQWEsQ0FBQyxPQUFPLENBQUMsSUFBSSxFQUFFLGVBQWUsQ0FBQyxDQUFDLENBQUM7UUFFNUUsTUFBTSxZQUFZLEdBQUcsSUFBSSwrQkFBYSxDQUFDLFFBQVEsQ0FBQyxJQUFJLEVBQUUsZUFBZSxDQUFDLENBQUMsTUFBTSxDQUMzRSxJQUFJLCtCQUFhLENBQUMsUUFBUSxDQUFDLElBQUksRUFBRSxlQUFlLENBQUMsQ0FBQyxNQUFNO1FBQ3RELDhGQUE4RjtRQUM5RixrQkFBa0IsQ0FBQyxJQUFJLENBQUMsZUFBZSxDQUFDLENBQ3pDLENBQUMsUUFBUTtRQUNSLDhHQUE4RztRQUM5RyxzQkFBc0IsRUFDdEI7WUFDRSxVQUFVLEVBQUUsU0FBUztTQUN0QixDQUNGLENBQ0YsQ0FBQztRQUVGLElBQUksS0FBSyxFQUFFLFlBQVksRUFBRSxLQUFLLElBQUksSUFBSSxFQUFFLENBQUM7WUFDdkMsTUFBTSxRQUFRLEdBQUcsS0FBSyxFQUFFLFlBQVksRUFBRSxRQUFRLElBQUksR0FBRyxDQUFDLFFBQVEsQ0FBQyxPQUFPLENBQUMsQ0FBQyxDQUFDLENBQUM7WUFDMUUsTUFBTSxXQUFXLEdBQUcsS0FBSyxFQUFFLFlBQVksRUFBRSxXQUFXLElBQUksRUFBRSxDQUFDO1lBQzNELE1BQU0sV0FBVyxHQUFHLEtBQUssRUFBRSxZQUFZLEVBQUUsV0FBVyxJQUFJLEdBQUcsQ0FBQztZQUU1RCxNQUFNLFlBQVksR0FBRyxRQUFRLENBQUMsU0FBUyxFQUFFLEdBQUcsV0FBVyxJQUFJLFdBQVcsR0FBRyxDQUFDLFdBQVcsR0FBRyxDQUFDLENBQUMsQ0FBQztZQUMzRixJQUFJLFlBQVksSUFBSSxHQUFHLENBQUMsUUFBUSxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUMsQ0FBQyxTQUFTLEVBQUUsRUFBRSxDQUFDO2dCQUNyRCxrSUFBa0k7Z0JBQ2xJLHdOQUF3TjtnQkFDeE4seUJBQVcsQ0FBQyxFQUFFLENBQUMsSUFBSSxDQUFDLENBQUMsVUFBVSxDQUFDLDhDQUE4QyxJQUFJLENBQUMsS0FBSyxDQUFDLFlBQVksR0FBRyxFQUFFLEdBQUcsRUFBRSxDQUFDLDJGQUEyRixDQUFDLENBQUM7WUFDL00sQ0FBQztZQUVELFlBQVksQ0FBQyxRQUFRLENBQUM7Z0JBQ3BCLFFBQVE7Z0JBQ1IsV0FBVztnQkFDWCxXQUFXO2dCQUNYLHlCQUF5QjtnQkFDekIsNEdBQTRHO2FBQzdHLENBQUMsQ0FBQztRQUNMLENBQUM7UUFFRCxJQUFJLFVBQXdELENBQUM7UUFDN0QsSUFBSSxJQUFJLENBQUMsS0FBSyxFQUFFLFVBQVUsRUFBRSxDQUFDO1lBQzNCLElBQUksQ0FBQyxvQkFBb0IsR0FBRyxJQUFJLHNCQUFJLENBQUMsUUFBUSxDQUFDLElBQUksRUFBRSxNQUFNLEVBQUU7Z0JBQzFELFlBQVksRUFBRSxLQUFLLEVBQUUsVUFBVSxFQUFFLFlBQVk7Z0JBQzdDLFNBQVMsRUFBRSxLQUFLLEVBQUUsVUFBVSxFQUFFLFlBQVksSUFBSSxzQkFBSSxDQUFDLGFBQWEsQ0FBQyxTQUFTO2dCQUMxRSxhQUFhLEVBQUUsR0FBRyxDQUFDLGFBQWEsQ0FBQyxPQUFPO2FBQ3pDLENBQUMsQ0FBQztZQUVILFVBQVUsR0FBRztnQkFDWCxXQUFXLEVBQUUsSUFBSSxDQUFDLG9CQUFvQjtnQkFDdEMsb0JBQW9CLEVBQUUsS0FBSyxFQUFFLFVBQVUsRUFBRSxvQkFBb0IsSUFBSSxJQUFJO2dCQUNyRSxLQUFLLEVBQUUsS0FBSyxFQUFFLFVBQVUsRUFBRSxLQUFLLElBQUksK0JBQWEsQ0FBQyxRQUFRLENBQUMsR0FBRzthQUM5RCxDQUFDO1FBQ0osQ0FBQztRQUVELE1BQU0sWUFBWSxHQUFHLElBQUksK0JBQWEsQ0FBQyxZQUFZLENBQ2pELElBQUksRUFDSixxQkFBcUIsRUFDckI7WUFDRSxjQUFjLEVBQUUsK0JBQWEsQ0FBQyxjQUFjLENBQUMsYUFBYSxDQUFDLG1CQUFtQixDQUFDLElBQUksQ0FBQyxZQUFZLENBQUMsQ0FBQztZQUNsRyxJQUFJLEVBQUUsVUFBVTtTQUNqQixDQUNGLENBQUM7UUFFRixZQUFZLENBQUMsU0FBUyxDQUFDLFVBQVUsQ0FBQyxDQUFDO1FBQ25DLFlBQVksQ0FBQyxjQUFjLENBQUMsVUFBVSxFQUFFLHNCQUFzQixDQUFDLENBQUM7UUFDaEUsS0FBSyxNQUFNLFFBQVEsSUFBSSxJQUFJLENBQUMsU0FBUyxFQUFFLENBQUM7WUFDdEMsUUFBUSxDQUFDLGlCQUFpQixDQUFDLFlBQVksQ0FBQyxDQUFDO1FBQzNDLENBQUM7UUFFRCxPQUFPLFlBQVksQ0FBQztJQUN0QixDQUFDO0lBRU8sY0FBYztRQUNwQixNQUFNLElBQUksR0FBRyxJQUFJLGlEQUFzQixDQUNyQyxJQUFJLEVBQ0osaUJBQWlCLEVBQ2pCO1lBQ0UsV0FBVyxFQUFFLG9FQUFvRTtZQUNqRixXQUFXLEVBQUU7Z0JBQ1gsaUJBQWlCLEVBQUUsSUFBSSxDQUFDLE9BQU8sQ0FBQyxNQUFNLENBQUMsU0FBUztnQkFDaEQsNkJBQTZCLEVBQUUsSUFBSSxDQUFDLE9BQU8sQ0FBQyxnQkFBZ0IsQ0FBQyxTQUFTO2dCQUN0RSxHQUFHLElBQUksQ0FBQyxjQUFjO2FBQ3ZCO1lBQ0QsT0FBTyxFQUFFLEdBQUcsQ0FBQyxRQUFRLENBQUMsT0FBTyxDQUFDLEVBQUUsQ0FBQztZQUNqQyxRQUFRLEVBQUUsSUFBQSx5QkFBaUIsRUFBQyxJQUFJLEVBQUUsd0JBQWdCLENBQUMsWUFBWSxDQUFDO1lBQ2hFLGFBQWEsRUFBRSx3QkFBTSxDQUFDLGFBQWEsQ0FBQyxJQUFJO1lBQ3hDLEdBQUcsSUFBSSxDQUFDLGdCQUFnQjtTQUN6QixDQUNGLENBQUM7UUFFRixJQUFJLENBQUMsT0FBTyxDQUFDLE1BQU0sQ0FBQyxTQUFTLENBQUMsSUFBSSxDQUFDLENBQUM7UUFDcEMsSUFBSSxDQUFDLE9BQU8sQ0FBQyxnQkFBZ0IsQ0FBQyxTQUFTLENBQUMsSUFBSSxDQUFDLENBQUM7UUFFOUMsT0FBTyxJQUFJLENBQUM7SUFDZCxDQUFDO0lBRU8sa0JBQWtCO1FBQ3hCLE1BQU0sSUFBSSxHQUFHLElBQUksMERBQTBCLENBQ3pDLElBQUksRUFDSixlQUFlLEVBQ2Y7WUFDRSxXQUFXLEVBQUUsOENBQThDO1lBQzNELFdBQVcsRUFBRTtnQkFDWCxpQkFBaUIsRUFBRSxJQUFJLENBQUMsT0FBTyxDQUFDLE1BQU0sQ0FBQyxTQUFTO2dCQUNoRCw2QkFBNkIsRUFBRSxJQUFJLENBQUMsT0FBTyxDQUFDLGdCQUFnQixDQUFDLFNBQVM7Z0JBQ3RFLEdBQUcsSUFBSSxDQUFDLGNBQWM7YUFDdkI7WUFDRCxPQUFPLEVBQUUsR0FBRyxDQUFDLFFBQVEsQ0FBQyxPQUFPLENBQUMsRUFBRSxDQUFDO1lBQ2pDLFFBQVEsRUFBRSxJQUFBLHlCQUFpQixFQUFDLElBQUksRUFBRSx3QkFBZ0IsQ0FBQyxZQUFZLENBQUM7WUFDaEUsYUFBYSxFQUFFLHdCQUFNLENBQUMsYUFBYSxDQUFDLElBQUk7WUFDeEMsR0FBRyxJQUFJLENBQUMsZ0JBQWdCO1NBQ3pCLENBQ0YsQ0FBQztRQUVGLElBQUksQ0FBQyxPQUFPLENBQUMsTUFBTSxDQUFDLFNBQVMsQ0FBQyxJQUFJLENBQUMsQ0FBQztRQUNwQyxJQUFJLENBQUMsT0FBTyxDQUFDLGdCQUFnQixDQUFDLFNBQVMsQ0FBQyxJQUFJLENBQUMsQ0FBQztRQUU5QyxPQUFPLElBQUksQ0FBQztJQUNkLENBQUM7SUFFTyxjQUFjO1FBQ3BCLE1BQU0sY0FBYyxHQUFHLElBQUksZ0NBQWMsQ0FDdkMsSUFBSSxFQUNKLFFBQVEsRUFDUjtZQUNFLFdBQVcsRUFBRSxtRUFBbUU7WUFDaEYsV0FBVyxFQUFFO2dCQUNYLGtCQUFrQixFQUFFLElBQUksQ0FBQyxPQUFPLENBQUMsT0FBTyxDQUFDLFNBQVM7Z0JBQ2xELGlCQUFpQixFQUFFLElBQUksQ0FBQyxPQUFPLENBQUMsTUFBTSxDQUFDLFNBQVM7Z0JBQ2hELDZCQUE2QixFQUFFLElBQUksQ0FBQyxPQUFPLENBQUMsZ0JBQWdCLENBQUMsU0FBUztnQkFDdEUsZ0JBQWdCLEVBQUUsSUFBSSxDQUFDLE9BQU8sQ0FBQyxLQUFLLENBQUMsU0FBUztnQkFDOUMsV0FBVyxFQUFFLElBQUksQ0FBQyxPQUFPLENBQUMsR0FBRztnQkFDN0IsbUJBQW1CLEVBQUUsSUFBSSxDQUFDLE9BQU8sQ0FBQyxPQUFPLENBQUMsYUFBYSxDQUFDLFdBQVc7Z0JBQ25FLGlCQUFpQixFQUFFLElBQUksQ0FBQyxZQUFZLENBQUMsZUFBZTtnQkFDcEQsdUJBQXVCLEVBQUUsSUFBSSxDQUFDLG9CQUFvQixFQUFFLFlBQVksSUFBSSxFQUFFO2dCQUN0RSxrQkFBa0IsRUFBRSxJQUFJLENBQUMsUUFBUTtnQkFDakMsR0FBRyxJQUFJLENBQUMsY0FBYzthQUN2QjtZQUNELE9BQU8sRUFBRSxHQUFHLENBQUMsUUFBUSxDQUFDLE9BQU8sQ0FBQyxDQUFDLENBQUM7WUFDaEMsUUFBUSxFQUFFLElBQUEseUJBQWlCLEVBQUMsSUFBSSxFQUFFLHdCQUFnQixDQUFDLEtBQUssQ0FBQztZQUN6RCxhQUFhLEVBQUUsd0JBQU0sQ0FBQyxhQUFhLENBQUMsSUFBSTtZQUN4QyxHQUFHLElBQUksQ0FBQyxnQkFBZ0I7U0FDekIsQ0FDRixDQUFDO1FBRUYsTUFBTSxTQUFTLEdBQUcsSUFBSSxDQUFDLFNBQVMsQ0FBQyxPQUFPLENBQUMsUUFBUSxDQUFDLEVBQUU7WUFDbEQsTUFBTSxNQUFNLEdBQUcsUUFBUSxDQUFDLE1BQU0sQ0FBQyxjQUFjLENBQUMsQ0FBQztZQUMvQyxnRkFBZ0Y7WUFDaEYsT0FBTyxLQUFLLENBQUMsT0FBTyxDQUFDLE1BQU0sQ0FBQyxDQUFDLENBQUMsQ0FBQyxNQUFNLENBQUMsQ0FBQyxDQUFDLENBQUMsTUFBTSxDQUFDLENBQUM7UUFDbkQsQ0FBQyxDQUFDLENBQUM7UUFFSCxzRkFBc0Y7UUFDdEYsdUZBQXVGO1FBQ3ZGLE1BQU0sS0FBSyxHQUFHLEdBQUcsQ0FBQyxLQUFLLENBQUMsRUFBRSxDQUFDLElBQUksQ0FBQyxDQUFDO1FBQ2pDLE1BQU0sQ0FBQyxHQUFJLGNBQWMsQ0FBQyxJQUFJLENBQUMsWUFBbUMsQ0FBQztRQUNuRSxDQUFDLENBQUMsbUJBQW1CLENBQUMsa0NBQWtDLEVBQUUsQ0FBQyxDQUFDLFNBQVMsQ0FBQyxDQUFDO1FBQ3ZFLENBQUMsQ0FBQyxtQkFBbUIsQ0FBQyxrQ0FBa0MsRUFBRSxLQUFLLENBQUMsU0FBUyxDQUFDLENBQUM7UUFDM0UsQ0FBQyxDQUFDLFdBQVcsQ0FBQyxXQUFXLEVBQUUsU0FBUyxDQUFDLENBQUM7UUFDdEMsY0FBYyxDQUFDLGVBQWUsQ0FBQyxJQUFJLHFCQUFHLENBQUMsZUFBZSxDQUFDO1lBQ3JELE9BQU8sRUFBRSxDQUFDLHNDQUFzQyxDQUFDO1lBQ2pELFNBQVMsRUFBRSxDQUFDLEtBQUssQ0FBQyxPQUFPLENBQUM7U0FDM0IsQ0FBQyxDQUFDLENBQUM7UUFFSixJQUFJLENBQUMsT0FBTyxDQUFDLE9BQU8sQ0FBQyxTQUFTLENBQUMsY0FBYyxDQUFDLENBQUM7UUFDL0MsSUFBSSxDQUFDLE9BQU8sQ0FBQyxNQUFNLENBQUMsU0FBUyxDQUFDLGNBQWMsQ0FBQyxDQUFDO1FBQzlDLElBQUksQ0FBQyxPQUFPLENBQUMsZ0JBQWdCLENBQUMsU0FBUyxDQUFDLGNBQWMsQ0FBQyxDQUFDO1FBQ3hELElBQUksQ0FBQyxPQUFPLENBQUMsS0FBSyxDQUFDLFNBQVMsQ0FBQyxjQUFjLENBQUMsQ0FBQztRQUM3QyxJQUFJLENBQUMsWUFBWSxDQUFDLFNBQVMsQ0FBQyxjQUFjLENBQUMsQ0FBQztRQUU1QyxJQUFJLEdBQUcsQ0FBQyxTQUFTLENBQ2YsSUFBSSxFQUNKLGdCQUFnQixFQUNoQjtZQUNFLEtBQUssRUFBRSxnQkFBZ0IsS0FBSyxDQUFDLE1BQU0sa0NBQWtDLGNBQWMsQ0FBQyxZQUFZLGNBQWM7U0FDL0csQ0FDRixDQUFDO1FBRUYsTUFBTSxNQUFNLEdBQUcsSUFBSSxDQUFDLEtBQUssRUFBRSxZQUFZLElBQUkscUJBQVksQ0FBQyxRQUFRLEVBQUUsQ0FBQztRQUNuRSxNQUFNLEdBQUcsR0FBRyxNQUFNLENBQUMsSUFBSSxDQUFDLElBQUksRUFBRSxlQUFlLEVBQUUsY0FBYyxDQUFDLENBQUM7UUFFL0QsSUFBSSxHQUFHLEtBQUssRUFBRSxFQUFFLENBQUM7WUFDZixJQUFJLEdBQUcsQ0FBQyxTQUFTLENBQ2YsSUFBSSxFQUNKLFlBQVksRUFDWjtnQkFDRSxLQUFLLEVBQUUsR0FBRzthQUNYLENBQ0YsQ0FBQztRQUNKLENBQUM7SUFDSCxDQUFDO0lBRU8sYUFBYTtRQUNuQixNQUFNLGFBQWEsR0FBRyxJQUFJLDhCQUFhLENBQ3JDLElBQUksRUFDSixPQUFPLEVBQ1A7WUFDRSxXQUFXLEVBQUUsMkRBQTJEO1lBQ3hFLFdBQVcsRUFBRTtnQkFDWCxnQkFBZ0IsRUFBRSxJQUFJLENBQUMsT0FBTyxDQUFDLEtBQUssQ0FBQyxTQUFTO2dCQUM5QyxrQkFBa0IsRUFBRSxJQUFJLENBQUMsT0FBTyxDQUFDLE9BQU8sQ0FBQyxTQUFTO2dCQUNsRCxpQkFBaUIsRUFBRSxJQUFJLENBQUMsT0FBTyxDQUFDLE1BQU0sQ0FBQyxTQUFTO2dCQUNoRCw2QkFBNkIsRUFBRSxJQUFJLENBQUMsT0FBTyxDQUFDLGdCQUFnQixDQUFDLFNBQVM7Z0JBQ3RFLFdBQVcsRUFBRSxJQUFJLENBQUMsT0FBTyxDQUFDLEdBQUc7Z0JBQzdCLEdBQUcsSUFBSSxDQUFDLGNBQWM7YUFDdkI7WUFDRCxPQUFPLEVBQUUsR0FBRyxDQUFDLFFBQVEsQ0FBQyxPQUFPLENBQUMsQ0FBQyxDQUFDO1lBQ2hDLFFBQVEsRUFBRSxJQUFBLHlCQUFpQixFQUFDLElBQUksRUFBRSx3QkFBZ0IsQ0FBQyxLQUFLLENBQUM7WUFDekQsYUFBYSxFQUFFLHdCQUFNLENBQUMsYUFBYSxDQUFDLElBQUk7WUFDeEMsR0FBRyxJQUFJLENBQUMsZ0JBQWdCO1NBQ3pCLENBQ0YsQ0FBQztRQUVGLGlEQUFpRDtRQUNqRCxJQUFJLENBQUMsT0FBTyxDQUFDLE9BQU8sQ0FBQyxVQUFVLENBQUMsYUFBYSxDQUFDLENBQUM7UUFDL0MsSUFBSSxDQUFDLE9BQU8sQ0FBQyxNQUFNLENBQUMsU0FBUyxDQUFDLGFBQWEsQ0FBQyxDQUFDO1FBQzdDLElBQUksQ0FBQyxPQUFPLENBQUMsTUFBTSxDQUFDLFVBQVUsQ0FBQyxhQUFhLENBQUMsQ0FBQztRQUM5QywwREFBMEQ7UUFDMUQsSUFBSSxDQUFDLE9BQU8sQ0FBQyxnQkFBZ0IsQ0FBQyxVQUFVLENBQUMsYUFBYSxDQUFDLENBQUM7UUFDeEQsSUFBSSxDQUFDLE9BQU8sQ0FBQyxLQUFLLENBQUMsU0FBUyxDQUFDLGFBQWEsQ0FBQyxDQUFDO1FBQzVDLElBQUksQ0FBQyxPQUFPLENBQUMsS0FBSyxDQUFDLFVBQVUsQ0FBQyxhQUFhLENBQUMsQ0FBQztRQUU3QyxNQUFNLE1BQU0sR0FBRyxJQUFJLENBQUMsS0FBSyxFQUFFLFdBQVcsSUFBSSxxQkFBWSxDQUFDLFNBQVMsRUFBRSxDQUFDO1FBQ25FLE9BQU8sTUFBTSxDQUFDLElBQUksQ0FBQyxJQUFJLEVBQUUsY0FBYyxFQUFFLGFBQWEsQ0FBQyxDQUFDO0lBQzFELENBQUM7SUFFTyx1QkFBdUI7UUFDN0Isd0ZBQXdGO1FBQ3hGLEtBQUssTUFBTSxFQUFFLElBQUksSUFBSSxDQUFDLFNBQVMsRUFBRSxDQUFDO1lBQ2hDLEtBQUssTUFBTSxFQUFFLElBQUksSUFBSSxDQUFDLFNBQVMsRUFBRSxDQUFDO2dCQUNoQyxJQUFJLEVBQUUsSUFBSSxFQUFFLEVBQUUsQ0FBQztvQkFDYixTQUFTO2dCQUNYLENBQUM7Z0JBQ0QsSUFBSSxFQUFFLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDLEVBQUUsQ0FBQyxNQUFNLENBQUMsUUFBUSxDQUFDLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQztvQkFDaEQsSUFBSSxFQUFFLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDLEVBQUUsQ0FBQyxNQUFNLENBQUMsUUFBUSxDQUFDLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQzt3QkFDaEQsTUFBTSxJQUFJLEtBQUssQ0FBQyxRQUFRLEVBQUUsQ0FBQyxJQUFJLENBQUMsSUFBSSxRQUFRLEVBQUUsQ0FBQyxJQUFJLENBQUMsSUFBSSx5QkFBeUIsRUFBRSxDQUFDLE1BQU0sQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLEdBQUcsQ0FBQyxDQUFDO29CQUM1RyxDQUFDO29CQUNELHlCQUFXLENBQUMsRUFBRSxDQUFDLEVBQUUsQ0FBQyxDQUFDLFVBQVUsQ0FBQyxXQUFXLEVBQUUsQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxzQ0FBc0MsRUFBRSxDQUFDLElBQUksQ0FBQyxJQUFJLFFBQVEsRUFBRSxDQUFDLE1BQU0sQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLDJDQUEyQyxFQUFFLENBQUMsTUFBTSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsMkdBQTJHLENBQUMsQ0FBQztnQkFDelQsQ0FBQztZQUNILENBQUM7UUFDSCxDQUFDO0lBQ0gsQ0FBQztJQUVPLFVBQVU7UUFDaEIsT0FBTyxJQUFJLHNEQUF3QixDQUFDLElBQUksRUFBRSxhQUFhLEVBQUU7WUFDdkQsV0FBVyxFQUFFLHdGQUF3RjtZQUNyRyxXQUFXLEVBQUU