@cloudsnorkel/cdk-github-runners
Version:
CDK construct to create GitHub Actions self-hosted runners. Creates ephemeral runners on demand. Easy to deploy and highly customizable.
768 lines • 137 kB
JavaScript
"use strict";
var _a;
Object.defineProperty(exports, "__esModule", { value: true });
exports.GitHubRunners = void 0;
const JSII_RTTI_SYMBOL_1 = Symbol.for("jsii.rtti");
const fs = require("fs");
const os = require("os");
const path = require("path");
const cdk = require("aws-cdk-lib");
const aws_cdk_lib_1 = require("aws-cdk-lib");
const constructs_1 = require("constructs");
const access_1 = require("./access");
const delete_failed_runner_function_1 = require("./delete-failed-runner-function");
const idle_runner_repear_function_1 = require("./idle-runner-repear-function");
const providers_1 = require("./providers");
const secrets_1 = require("./secrets");
const setup_function_1 = require("./setup-function");
const status_function_1 = require("./status-function");
const token_retriever_function_1 = require("./token-retriever-function");
const utils_1 = require("./utils");
const warm_runner_manager_function_1 = require("./warm-runner-manager-function");
const webhook_1 = require("./webhook");
const webhook_redelivery_1 = require("./webhook-redelivery");
/**
* Create all the required infrastructure to provide self-hosted GitHub runners. It creates a webhook, secrets, and a step function to orchestrate all runs. Secrets are not automatically filled. See README.md for instructions on how to setup GitHub integration.
*
* By default, this will create a runner provider of each available type with the defaults. This is good enough for the initial setup stage when you just want to get GitHub integration working.
*
* ```typescript
* new GitHubRunners(this, 'runners');
* ```
*
* Usually you'd want to configure the runner providers so the runners can run in a certain VPC or have certain permissions.
*
* ```typescript
* const vpc = ec2.Vpc.fromLookup(this, 'vpc', { vpcId: 'vpc-1234567' });
* const runnerSg = new ec2.SecurityGroup(this, 'runner security group', { vpc: vpc });
* const dbSg = ec2.SecurityGroup.fromSecurityGroupId(this, 'database security group', 'sg-1234567');
* const bucket = new s3.Bucket(this, 'runner bucket');
*
* // create a custom CodeBuild provider
* const myProvider = new CodeBuildRunnerProvider(
* this, 'codebuild runner',
* {
* labels: ['my-codebuild'],
* vpc: vpc,
* securityGroups: [runnerSg],
* },
* );
* // grant some permissions to the provider
* bucket.grantReadWrite(myProvider);
* dbSg.connections.allowFrom(runnerSg, ec2.Port.tcp(3306), 'allow runners to connect to MySQL database');
*
* // create the runner infrastructure
* new GitHubRunners(
* this,
* 'runners',
* {
* providers: [myProvider],
* }
* );
* ```
*/
class GitHubRunners extends constructs_1.Construct {
constructor(scope, id, props) {
super(scope, id);
this.props = props;
this.extraLambdaEnv = {};
this.jobsCompletedMetricFiltersInitialized = false;
this.warmConfigHashes = [];
this.deleteFailedRunnerIndex = 0;
this.secrets = new secrets_1.Secrets(this, 'Secrets');
this.extraLambdaProps = {
vpc: this.props?.vpc,
vpcSubnets: this.props?.vpcSubnets,
allowPublicSubnet: this.props?.allowPublicSubnet,
securityGroups: this.lambdaSecurityGroups(),
layers: [],
};
this.connections = new aws_cdk_lib_1.aws_ec2.Connections({ securityGroups: this.extraLambdaProps.securityGroups });
this.createCertificateLayer(scope);
if (this.props?.providers) {
this.providers = this.props.providers;
}
else {
this.providers = [
new providers_1.CodeBuildRunnerProvider(this, 'CodeBuild'),
new providers_1.LambdaRunnerProvider(this, 'Lambda'),
new providers_1.FargateRunnerProvider(this, 'Fargate'),
];
}
if (this.providers.length == 0) {
aws_cdk_lib_1.Annotations.of(this).addError('At least one runner provider is required');
}
this.checkIntersectingLabels();
this.orchestrator = this.stateMachine(props);
this.webhook = new webhook_1.GithubWebhookHandler(this, 'Webhook Handler', {
orchestrator: this.orchestrator,
secrets: this.secrets,
access: this.props?.webhookAccess ?? access_1.LambdaAccess.lambdaUrl(),
providers: this.providers.reduce((acc, p) => {
acc[p.node.path] = p.labels;
return acc;
}, {}),
requireSelfHostedLabel: this.props?.requireSelfHostedLabel ?? true,
providerSelector: this.props?.providerSelector,
extraLambdaProps: this.extraLambdaProps,
extraLambdaEnv: this.extraLambdaEnv,
idleTimeoutSeconds: this.props?.idleTimeout?.toSeconds(),
});
this.redeliverer = new webhook_redelivery_1.GithubWebhookRedelivery(this, 'Webhook Redelivery', {
secrets: this.secrets,
extraLambdaProps: this.extraLambdaProps,
extraLambdaEnv: this.extraLambdaEnv,
});
this.setupUrl = this.setupFunction();
this.statusFunction();
}
stateMachine(props) {
const tokenRetrieverTask = new aws_cdk_lib_1.aws_stepfunctions_tasks.LambdaInvoke(this, 'Get Runner Token', {
lambdaFunction: this.tokenRetriever(),
payloadResponseOnly: true,
resultPath: '$.runner',
});
const idleReaper = this.idleReaper();
const defaultIdleSeconds = (props?.idleTimeout ?? cdk.Duration.minutes(5)).toSeconds();
const queueIdleReaperTask = new aws_cdk_lib_1.aws_stepfunctions_tasks.SqsSendMessage(this, 'Queue Idle Reaper', {
queue: this.idleReaperQueue(idleReaper),
queryLanguage: aws_cdk_lib_1.aws_stepfunctions.QueryLanguage.JSONATA,
messageBody: aws_cdk_lib_1.aws_stepfunctions.TaskInput.fromObject({
executionArn: '{% $states.context.Execution.Id %}',
runnerName: '{% $states.context.Execution.Name %}',
owner: '{% $states.input.owner %}',
repo: '{% $states.input.repo %}',
installationId: '{% $states.input.installationId %}',
maxIdleSeconds: `{% $exists($states.input.maxIdleSeconds) ? $states.input.maxIdleSeconds : ${defaultIdleSeconds} %}`,
}),
outputs: '{% $states.input %}', // discard
});
const providerConsts = (0, providers_1.mergeConstMaps)(...this.providers.map(p => p.stepFunctionConstants()));
const afterRunnerToken = Object.keys(providerConsts).length > 0
? tokenRetrieverTask.next(new aws_cdk_lib_1.aws_stepfunctions.Pass(this, 'Provider Constants', {
parameters: providerConsts,
resultPath: '$.consts',
}))
: tokenRetrieverTask;
const providerChooser = new aws_cdk_lib_1.aws_stepfunctions.Choice(this, 'Choose provider');
for (const provider of this.providers) {
const providerTask = provider.getStepFunctionTask({
runnerTokenPath: aws_cdk_lib_1.aws_stepfunctions.JsonPath.stringAt('$.runner.token'),
runnerNamePath: aws_cdk_lib_1.aws_stepfunctions.JsonPath.stringAt('$$.Execution.Name'),
githubDomainPath: aws_cdk_lib_1.aws_stepfunctions.JsonPath.stringAt('$.runner.domain'),
ownerPath: aws_cdk_lib_1.aws_stepfunctions.JsonPath.stringAt('$.owner'),
repoPath: aws_cdk_lib_1.aws_stepfunctions.JsonPath.stringAt('$.repo'),
registrationUrl: aws_cdk_lib_1.aws_stepfunctions.JsonPath.stringAt('$.runner.registrationUrl'),
labelsPath: aws_cdk_lib_1.aws_stepfunctions.JsonPath.stringAt('$.labels'),
addCatchAndCleanUp: (state, next) => this.addCatchAndCleanUp(state, next),
});
providerChooser.when(aws_cdk_lib_1.aws_stepfunctions.Condition.and(aws_cdk_lib_1.aws_stepfunctions.Condition.stringEquals('$.provider', provider.node.path)), providerTask, {
comment: `Labels: ${provider.labels.join(', ')}`,
});
}
providerChooser.otherwise(new aws_cdk_lib_1.aws_stepfunctions.Succeed(this, 'Unknown label'));
const errorHandler = new aws_cdk_lib_1.aws_stepfunctions.Parallel(this, 'Error Handler').branch(
// we get a token for every retry because the token can expire faster than the job can timeout
afterRunnerToken.next(providerChooser));
this.addCatchAndCleanUp(errorHandler);
const runProviders = new aws_cdk_lib_1.aws_stepfunctions.Parallel(this, 'Run Providers').branch(errorHandler);
if (props?.retryOptions?.retry ?? true) {
const interval = props?.retryOptions?.interval ?? cdk.Duration.minutes(1);
const maxAttempts = props?.retryOptions?.maxAttempts ?? 23;
const backoffRate = props?.retryOptions?.backoffRate ?? 1.3;
const totalSeconds = interval.toSeconds() * backoffRate ** maxAttempts / (backoffRate - 1);
if (totalSeconds >= cdk.Duration.days(1).toSeconds()) {
// https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/about-self-hosted-runners#usage-limits
// "Job queue time - Each job for self-hosted runners can be queued for a maximum of 24 hours. If a self-hosted runner does not start executing the job within this limit, the job is terminated and fails to complete."
aws_cdk_lib_1.Annotations.of(this).addWarning(`Total retry time is greater than 24 hours (${Math.floor(totalSeconds / 60 / 60)} hours). Jobs expire after 24 hours so it would be a waste of resources to retry further.`);
}
runProviders.addRetry({
interval,
maxAttempts,
backoffRate,
// we retry on everything
// deleted idle runners will also fail, but the reaper will stop this step function to avoid endless retries
});
}
let logOptions;
if (this.props?.logOptions) {
this.stateMachineLogGroup = new aws_cdk_lib_1.aws_logs.LogGroup(this, 'Logs', {
logGroupName: props?.logOptions?.logGroupName,
retention: props?.logOptions?.logRetention ?? aws_cdk_lib_1.aws_logs.RetentionDays.ONE_MONTH,
removalPolicy: cdk.RemovalPolicy.DESTROY,
});
logOptions = {
destination: this.stateMachineLogGroup,
includeExecutionData: props?.logOptions?.includeExecutionData ?? true,
level: props?.logOptions?.level ?? aws_cdk_lib_1.aws_stepfunctions.LogLevel.ALL,
};
}
const stateMachine = new aws_cdk_lib_1.aws_stepfunctions.StateMachine(this, 'Runner Orchestrator', {
definitionBody: aws_cdk_lib_1.aws_stepfunctions.DefinitionBody.fromChainable(queueIdleReaperTask.next(runProviders)),
logs: logOptions,
});
stateMachine.grantRead(idleReaper);
stateMachine.grantExecution(idleReaper, 'states:StopExecution');
for (const provider of this.providers) {
provider.grantStateMachine(stateMachine);
}
return stateMachine;
}
tokenRetriever() {
const func = new token_retriever_function_1.TokenRetrieverFunction(this, 'token-retriever', {
description: 'Get token from GitHub Actions used to start new self-hosted runner',
environment: {
GITHUB_SECRET_ARN: this.secrets.github.secretArn,
GITHUB_PRIVATE_KEY_SECRET_ARN: this.secrets.githubPrivateKey.secretArn,
...this.extraLambdaEnv,
},
timeout: cdk.Duration.seconds(30),
logGroup: (0, utils_1.singletonLogGroup)(this, utils_1.SingletonLogType.ORCHESTRATOR),
loggingFormat: aws_cdk_lib_1.aws_lambda.LoggingFormat.JSON,
...this.extraLambdaProps,
});
this.secrets.github.grantRead(func);
this.secrets.githubPrivateKey.grantRead(func);
return func;
}
deleteFailedRunner() {
const func = new delete_failed_runner_function_1.DeleteFailedRunnerFunction(this, 'delete-runner', {
description: 'Delete failed GitHub Actions runner on error',
environment: {
GITHUB_SECRET_ARN: this.secrets.github.secretArn,
GITHUB_PRIVATE_KEY_SECRET_ARN: this.secrets.githubPrivateKey.secretArn,
...this.extraLambdaEnv,
},
timeout: cdk.Duration.seconds(30),
logGroup: (0, utils_1.singletonLogGroup)(this, utils_1.SingletonLogType.ORCHESTRATOR),
loggingFormat: aws_cdk_lib_1.aws_lambda.LoggingFormat.JSON,
...this.extraLambdaProps,
});
this.secrets.github.grantRead(func);
this.secrets.githubPrivateKey.grantRead(func);
return func;
}
addCatchAndCleanUp(state, next) {
this.deleteFailedRunnerFunction ?? (this.deleteFailedRunnerFunction = this.deleteFailedRunner());
this.deleteFailedRunnerIndex++;
const task = new aws_cdk_lib_1.aws_stepfunctions_tasks.LambdaInvoke(this, `Delete Failed Runner ${this.deleteFailedRunnerIndex}`, {
stateName: `Delete Failed Runner ${this.deleteFailedRunnerIndex}`,
comment: 'Clean-up failed runner from GitHub Actions (if present)',
lambdaFunction: this.deleteFailedRunnerFunction,
payloadResponseOnly: true,
resultPath: '$.delete',
payload: aws_cdk_lib_1.aws_stepfunctions.TaskInput.fromObject({
runnerName: aws_cdk_lib_1.aws_stepfunctions.JsonPath.stringAt('$$.Execution.Name'),
owner: aws_cdk_lib_1.aws_stepfunctions.JsonPath.stringAt('$.owner'),
repo: aws_cdk_lib_1.aws_stepfunctions.JsonPath.stringAt('$.repo'),
installationId: aws_cdk_lib_1.aws_stepfunctions.JsonPath.numberAt('$.installationId'),
error: aws_cdk_lib_1.aws_stepfunctions.JsonPath.objectAt('$.error'),
}),
});
task.addRetry({
errors: ['RunnerBusy'],
interval: cdk.Duration.minutes(1),
backoffRate: 1,
maxAttempts: 60,
});
if (next) {
const nextStart = next.startState;
task.next(nextStart);
task.addCatch(nextStart, {
errors: [aws_cdk_lib_1.aws_stepfunctions.Errors.ALL],
resultPath: aws_cdk_lib_1.aws_stepfunctions.JsonPath.DISCARD,
});
}
state.addCatch(task, {
errors: [aws_cdk_lib_1.aws_stepfunctions.Errors.ALL],
resultPath: '$.error',
});
}
statusFunction() {
const statusFunction = new status_function_1.StatusFunction(this, 'status', {
description: 'Provide user with status about self-hosted GitHub Actions runners',
environment: {
WEBHOOK_SECRET_ARN: this.secrets.webhook.secretArn,
GITHUB_SECRET_ARN: this.secrets.github.secretArn,
GITHUB_PRIVATE_KEY_SECRET_ARN: this.secrets.githubPrivateKey.secretArn,
SETUP_SECRET_ARN: this.secrets.setup.secretArn,
WEBHOOK_URL: this.webhook.url,
WEBHOOK_HANDLER_ARN: this.webhook.handler.latestVersion.functionArn,
STEP_FUNCTION_ARN: this.orchestrator.stateMachineArn,
STEP_FUNCTION_LOG_GROUP: this.stateMachineLogGroup?.logGroupName ?? '',
SETUP_FUNCTION_URL: this.setupUrl,
...this.extraLambdaEnv,
},
timeout: cdk.Duration.minutes(3),
logGroup: (0, utils_1.singletonLogGroup)(this, utils_1.SingletonLogType.SETUP),
loggingFormat: aws_cdk_lib_1.aws_lambda.LoggingFormat.JSON,
...this.extraLambdaProps,
});
const providers = this.providers.flatMap(provider => {
const status = provider.status(statusFunction);
// Composite providers return an array, regular providers return a single status
return Array.isArray(status) ? status : [status];
});
// expose providers as stack metadata as it's too big for Lambda environment variables
// specifically integration testing got an error because lambda update request was >5kb
const stack = cdk.Stack.of(this);
const f = statusFunction.node.defaultChild;
f.addPropertyOverride('Environment.Variables.LOGICAL_ID', f.logicalId);
f.addPropertyOverride('Environment.Variables.STACK_NAME', stack.stackName);
f.addMetadata('providers', providers);
statusFunction.addToRolePolicy(new aws_cdk_lib_1.aws_iam.PolicyStatement({
actions: ['cloudformation:DescribeStackResource'],
resources: [stack.stackId],
}));
this.secrets.webhook.grantRead(statusFunction);
this.secrets.github.grantRead(statusFunction);
this.secrets.githubPrivateKey.grantRead(statusFunction);
this.secrets.setup.grantRead(statusFunction);
this.orchestrator.grantRead(statusFunction);
new cdk.CfnOutput(this, 'status command', {
value: `aws --region ${stack.region} lambda invoke --function-name ${statusFunction.functionName} status.json`,
});
const access = this.props?.statusAccess ?? access_1.LambdaAccess.noAccess();
const url = access.bind(this, 'status access', statusFunction);
if (url !== '') {
new cdk.CfnOutput(this, 'status url', {
value: url,
});
}
}
setupFunction() {
const setupFunction = new setup_function_1.SetupFunction(this, 'setup', {
description: 'Setup GitHub Actions integration with self-hosted runners',
environment: {
SETUP_SECRET_ARN: this.secrets.setup.secretArn,
WEBHOOK_SECRET_ARN: this.secrets.webhook.secretArn,
GITHUB_SECRET_ARN: this.secrets.github.secretArn,
GITHUB_PRIVATE_KEY_SECRET_ARN: this.secrets.githubPrivateKey.secretArn,
WEBHOOK_URL: this.webhook.url,
...this.extraLambdaEnv,
},
timeout: cdk.Duration.minutes(3),
logGroup: (0, utils_1.singletonLogGroup)(this, utils_1.SingletonLogType.SETUP),
loggingFormat: aws_cdk_lib_1.aws_lambda.LoggingFormat.JSON,
...this.extraLambdaProps,
});
// this.secrets.webhook.grantRead(setupFunction);
this.secrets.webhook.grantWrite(setupFunction);
this.secrets.github.grantRead(setupFunction);
this.secrets.github.grantWrite(setupFunction);
// this.secrets.githubPrivateKey.grantRead(setupFunction);
this.secrets.githubPrivateKey.grantWrite(setupFunction);
this.secrets.setup.grantRead(setupFunction);
this.secrets.setup.grantWrite(setupFunction);
const access = this.props?.setupAccess ?? access_1.LambdaAccess.lambdaUrl();
return access.bind(this, 'setup access', setupFunction);
}
checkIntersectingLabels() {
// this "algorithm" is very inefficient, but good enough for the tiny datasets we expect
for (const p1 of this.providers) {
for (const p2 of this.providers) {
if (p1 == p2) {
continue;
}
if (p1.labels.every(l => p2.labels.includes(l))) {
if (p2.labels.every(l => p1.labels.includes(l))) {
aws_cdk_lib_1.Annotations.of(this).addError(`Both ${p1.node.path} and ${p2.node.path} use the same labels [${p1.labels.join(', ')}]`);
return;
}
aws_cdk_lib_1.Annotations.of(p1).addWarning(`Labels [${p1.labels.join(', ')}] intersect with another provider (${p2.node.path} -- [${p2.labels.join(', ')}]). If a workflow specifies the labels [${p1.labels.join(', ')}], it is not guaranteed which provider will be used. It is recommended you do not use intersecting labels`);
}
}
}
}
idleReaper() {
return new idle_runner_repear_function_1.IdleRunnerRepearFunction(this, 'Idle Reaper', {
description: 'Stop idle GitHub runners to avoid paying for runners when the job was already canceled',
environment: {
GITHUB_SECRET_ARN: this.secrets.github.secretArn,
GITHUB_PRIVATE_KEY_SECRET_ARN: this.secrets.githubPrivateKey.secretArn,
...this.extraLambdaEnv,
},
logGroup: (0, utils_1.singletonLogGroup)(this, utils_1.SingletonLogType.ORCHESTRATOR),
loggingFormat: aws_cdk_lib_1.aws_lambda.LoggingFormat.JSON,
timeout: cdk.Duration.minutes(5),
...this.extraLambdaProps,
});
}
idleReaperQueue(reaper) {
// see this comment to understand why it's a queue that's out of the step function
// https://github.com/CloudSnorkel/cdk-github-runners/pull/314#issuecomment-1528901192
const queue = new aws_cdk_lib_1.aws_sqs.Queue(this, 'Idle Reaper Queue', {
deliveryDelay: cdk.Duration.minutes(10),
visibilityTimeout: cdk.Duration.minutes(10),
});
reaper.addEventSource(new aws_cdk_lib_1.aws_lambda_event_sources.SqsEventSource(queue, {
reportBatchItemFailures: true,
maxBatchingWindow: cdk.Duration.minutes(1),
batchSize: 10,
}));
this.secrets.github.grantRead(reaper);
this.secrets.githubPrivateKey.grantRead(reaper);
return queue;
}
lambdaSecurityGroups() {
if (!this.props?.vpc) {
if (this.props?.securityGroup) {
cdk.Annotations.of(this).addWarning('securityGroup is specified, but vpc is not. securityGroup will be ignored');
}
if (this.props?.securityGroups) {
cdk.Annotations.of(this).addWarning('securityGroups is specified, but vpc is not. securityGroups will be ignored');
}
return undefined;
}
if (this.props.securityGroups) {
if (this.props.securityGroup) {
cdk.Annotations.of(this).addWarning('Both securityGroup and securityGroups are specified. securityGroup will be ignored');
}
return this.props.securityGroups;
}
if (this.props.securityGroup) {
return [this.props.securityGroup];
}
return [new aws_cdk_lib_1.aws_ec2.SecurityGroup(this, 'Management Lambdas Security Group', { vpc: this.props.vpc })];
}
/**
* Extracts all unique IRunnerProvider instances from providers and composite providers (one level only).
* Uses a Set to ensure we don't process the same provider twice, even if it's used in multiple composites.
*
* @returns Set of unique IRunnerProvider instances
*/
extractUniqueSubProviders() {
const seen = new Set();
for (const provider of this.providers) {
// instanceof doesn't really work in CDK so use this hack instead
if ('logGroup' in provider) {
// Regular provider
seen.add(provider);
}
else {
// Composite provider - access the providers field
for (const subProvider of provider.providers) {
seen.add(subProvider);
}
}
}
return seen;
}
/**
* Creates a Lambda layer with certificates if extraCertificates is specified.
*/
createCertificateLayer(scope) {
if (!this.props?.extraCertificates) {
return;
}
const certificateFiles = (0, utils_1.discoverCertificateFiles)(this.props.extraCertificates);
// Concatenate all certificates into a single file for NODE_EXTRA_CA_CERTS
let combinedCertContent = '';
for (const certFile of certificateFiles) {
const certContent = fs.readFileSync(certFile, 'utf8');
combinedCertContent += certContent;
// Ensure proper PEM format with newline between certificates
if (!certContent.endsWith('\n')) {
combinedCertContent += '\n';
}
}
// Create a temporary directory, write the certificate file, create asset, then delete temp dir
const workdir = fs.mkdtempSync(path.join(os.tmpdir(), 'certificate-layer-'));
try {
const certPath = path.join(workdir, 'certs.pem');
fs.writeFileSync(certPath, combinedCertContent);
// Set environment variable and create layer
this.extraLambdaEnv.NODE_EXTRA_CA_CERTS = '/opt/certs.pem';
this.extraLambdaProps.layers.push(new aws_cdk_lib_1.aws_lambda.LayerVersion(scope, 'Certificate Layer', {
description: 'Layer containing GitHub Enterprise Server certificate(s) for cdk-github-runners',
code: aws_cdk_lib_1.aws_lambda.Code.fromAsset(workdir),
}));
}
finally {
// Calling `fromAsset()` has copied files to the assembly, so we can delete the temporary directory.
fs.rmSync(workdir, { recursive: true, force: true });
}
}
/**
* Metric for the number of GitHub Actions jobs completed. It has `ProviderLabels` and `Status` dimensions. The status can be one of "Succeeded", "SucceededWithIssues", "Failed", "Canceled", "Skipped", or "Abandoned".
*
* **WARNING:** this method creates a metric filter for each provider. Each metric has a status dimension with six possible values. These resources may incur cost.
*/
metricJobCompleted(props) {
if (!this.jobsCompletedMetricFiltersInitialized) {
// we can't use logs.FilterPattern.spaceDelimited() because it has no support for ||
// status list taken from https://github.com/actions/runner/blob/be9632302ceef50bfb36ea998cea9c94c75e5d4d/src/Sdk/DTWebApi/WebApi/TaskResult.cs
// we need "..." for Lambda that prefixes some extra data to log lines
const pattern = aws_cdk_lib_1.aws_logs.FilterPattern.literal('[..., marker = "CDKGHA", job = "JOB", done = "DONE", labels, status = "Succeeded" || status = "SucceededWithIssues" || status = "Failed" || status = "Canceled" || status = "Skipped" || status = "Abandoned"]');
// Extract all unique sub-providers from regular and composite providers
// Build a set first to avoid filtering the same log twice
for (const p of this.extractUniqueSubProviders()) {
const metricFilter = p.logGroup.addMetricFilter(`${p.logGroup.node.id} filter`, {
metricNamespace: 'GitHubRunners',
metricName: 'JobCompleted',
filterPattern: pattern,
metricValue: '1',
// can't with dimensions -- defaultValue: 0,
dimensions: {
ProviderLabels: '$labels',
Status: '$status',
},
});
if (metricFilter.node.defaultChild instanceof aws_cdk_lib_1.aws_logs.CfnMetricFilter) {
metricFilter.node.defaultChild.addPropertyOverride('MetricTransformations.0.Unit', 'Count');
}
else {
aws_cdk_lib_1.Annotations.of(metricFilter).addWarning('Unable to set metric filter Unit to Count');
}
}
this.jobsCompletedMetricFiltersInitialized = true;
}
return new aws_cdk_lib_1.aws_cloudwatch.Metric({
namespace: 'GitHubRunners',
metricName: 'JobsCompleted',
unit: aws_cdk_lib_1.aws_cloudwatch.Unit.COUNT,
statistic: aws_cdk_lib_1.aws_cloudwatch.Stats.SUM,
...props,
}).attachTo(this);
}
/**
* Metric for successful executions.
*
* A successful execution doesn't always mean a runner was started. It can be successful even without any label matches.
*
* A successful runner doesn't mean the job it executed was successful. For that, see {@link metricJobCompleted}.
*/
metricSucceeded(props) {
return this.orchestrator.metricSucceeded(props);
}
/**
* Metric for failed runner executions.
*
* A failed runner usually means the runner failed to start and so a job was never executed. It doesn't necessarily mean the job was executed and failed. For that, see {@link metricJobCompleted}.
*/
metricFailed(props) {
return this.orchestrator.metricFailed(props);
}
/**
* Metric for the interval, in milliseconds, between the time the execution starts and the time it closes. This time may be longer than the time the runner took.
*/
metricTime(props) {
return this.orchestrator.metricTime(props);
}
/**
* Creates a topic for notifications when a runner image build fails.
*
* Runner images are rebuilt every week by default. This provides the latest GitHub Runner version and software updates.
*
* If you want to be sure you are using the latest runner version, you can use this topic to be notified when a build fails.
*
* When the image builder is defined in a separate stack (e.g. in a split-stacks setup), pass that stack or construct
* as the optional scope so the topic and failure-notification aspects are created in the same stack as the image
* builder. Otherwise the aspects may not find the image builder resources.
*
* @param scope Optional scope (e.g. the image builder stack) where the topic and aspects will be created. Defaults to this construct.
*/
failedImageBuildsTopic(scope) {
scope ?? (scope = this);
const topic = new aws_cdk_lib_1.aws_sns.Topic(scope, 'Failed Runner Image Builds');
const stack = cdk.Stack.of(scope);
cdk.Aspects.of(stack).add(new providers_1.CodeBuildImageBuilderFailedBuildNotifier(topic));
cdk.Aspects.of(stack).add(new providers_1.AwsImageBuilderFailedBuildNotifier(providers_1.AwsImageBuilderFailedBuildNotifier.createFilteringTopic(scope, topic)));
return topic;
}
/**
* Creates CloudWatch Logs Insights saved queries that can be used to debug issues with the runners.
*
* * "Webhook errors" helps diagnose configuration issues with GitHub integration
* * "Ignored webhook" helps understand why runners aren't started
* * "Ignored jobs based on labels" helps debug label matching issues
* * "Webhook started runners" helps understand which runners were started
* * "Warm runner status" and "Warm runner errors" (when warm runners are configured)
*
* @param prefix Prefix for the query definitions. Defaults to "GitHub Runners".
*/
createLogsInsightsQueries(prefix = 'GitHub Runners') {
new aws_cdk_lib_1.aws_logs.QueryDefinition(this, 'Webhook errors', {
queryDefinitionName: `${prefix}/Webhook errors`,
logGroups: [this.webhook.handler.logGroup],
queryString: new aws_cdk_lib_1.aws_logs.QueryString({
filterStatements: [
`strcontains(@logStream, "${this.webhook.handler.functionName}")`,
'level = "ERROR"',
],
sort: '@timestamp desc',
limit: 100,
}),
});
new aws_cdk_lib_1.aws_logs.QueryDefinition(this, 'Orchestration errors', {
queryDefinitionName: `${prefix}/Orchestration errors`,
logGroups: [(0, utils_1.singletonLogGroup)(this, utils_1.SingletonLogType.ORCHESTRATOR)],
queryString: new aws_cdk_lib_1.aws_logs.QueryString({
filterStatements: [
'level = "ERROR"',
],
sort: '@timestamp desc',
limit: 100,
}),
});
new aws_cdk_lib_1.aws_logs.QueryDefinition(this, 'Runner image build errors', {
queryDefinitionName: `${prefix}/Runner image build errors`,
logGroups: [(0, utils_1.singletonLogGroup)(this, utils_1.SingletonLogType.RUNNER_IMAGE_BUILD)],
queryString: new aws_cdk_lib_1.aws_logs.QueryString({
filterStatements: [
'strcontains(message, "error") or strcontains(message, "ERROR") or strcontains(message, "Error") or level = "ERROR"',
],
sort: '@timestamp desc',
limit: 100,
}),
});
new aws_cdk_lib_1.aws_logs.QueryDefinition(this, 'Ignored webhooks', {
queryDefinitionName: `${prefix}/Ignored webhooks`,
logGroups: [this.webhook.handler.logGroup],
queryString: new aws_cdk_lib_1.aws_logs.QueryString({
fields: ['@timestamp', 'message.notice'],
filterStatements: [
`strcontains(@logStream, "${this.webhook.handler.functionName}")`,
'strcontains(message.notice, "Ignoring")',
],
sort: '@timestamp desc',
limit: 100,
}),
});
new aws_cdk_lib_1.aws_logs.QueryDefinition(this, 'Ignored jobs based on labels', {
queryDefinitionName: `${prefix}/Ignored jobs based on labels`,
logGroups: [this.webhook.handler.logGroup],
queryString: new aws_cdk_lib_1.aws_logs.QueryString({
fields: ['@timestamp', 'message.notice'],
filterStatements: [
`strcontains(@logStream, "${this.webhook.handler.functionName}")`,
'strcontains(message.notice, "Ignoring labels")',
],
sort: '@timestamp desc',
limit: 100,
}),
});
new aws_cdk_lib_1.aws_logs.QueryDefinition(this, 'Webhook started runners', {
queryDefinitionName: `${prefix}/Webhook started runners`,
logGroups: [this.webhook.handler.logGroup],
queryString: new aws_cdk_lib_1.aws_logs.QueryString({
fields: ['@timestamp', 'message.sfnInput.jobUrl', 'message.sfnInput.jobLabels', 'message.sfnInput.labels', 'message.sfnInput.provider'],
filterStatements: [
`strcontains(@logStream, "${this.webhook.handler.functionName}")`,
'message.sfnInput.jobUrl like /http.*/',
],
sort: '@timestamp desc',
limit: 100,
}),
});
new aws_cdk_lib_1.aws_logs.QueryDefinition(this, 'Webhook redeliveries', {
queryDefinitionName: `${prefix}/Webhook redeliveries`,
logGroups: [this.redeliverer.handler.logGroup],
queryString: new aws_cdk_lib_1.aws_logs.QueryString({
fields: ['@timestamp', 'message.notice', 'message.deliveryId', 'message.guid'],
filterStatements: [
'isPresent(message.deliveryId)',
],
sort: '@timestamp desc',
limit: 100,
}),
});
new aws_cdk_lib_1.aws_logs.QueryDefinition(this, 'Warm runner status', {
queryDefinitionName: `${prefix}/Warm runner status`,
logGroups: [(0, utils_1.singletonLogGroup)(this, utils_1.SingletonLogType.ORCHESTRATOR)],
queryString: new aws_cdk_lib_1.aws_logs.QueryString({
fields: ['@timestamp', 'message.notice', 'message.input.runnerName', 'message.input.providerPath', 'message.started', 'message.stillRunning', 'message.runnerBusy'],
filterStatements: [
cdk.Lazy.string({
produce: () => {
if (this.warmRunnerManager) {
return `strcontains(@logStream, "${this.warmRunnerManager.functionName}")`;
}
else {
return 'WARM RUNNERS NOT ENABLED';
}
},
}),
],
sort: '@timestamp desc',
limit: 200,
}),
});
new aws_cdk_lib_1.aws_logs.QueryDefinition(this, 'Warm runner errors', {
queryDefinitionName: `${prefix}/Warm runner errors`,
logGroups: [(0, utils_1.singletonLogGroup)(this, utils_1.SingletonLogType.ORCHESTRATOR)],
queryString: new aws_cdk_lib_1.aws_logs.QueryString({
fields: ['@timestamp', 'message.notice', 'message.input.runnerName', 'message.error'],
filterStatements: [
cdk.Lazy.string({
produce: () => {
if (this.warmRunnerManager) {
return `strcontains(@logStream, "${this.warmRunnerManager.functionName}")`;
}
else {
return 'WARM RUNNERS NOT ENABLED';
}
},
}),
'level = "ERROR"',
],
sort: '@timestamp desc',
limit: 100,
}),
});
}
/**
* Register a warm runner config hash. All registered hashes are passed to the
* manager Lambda via WARM_CONFIG_HASHES env var so keepers can detect stale configs.
*
* @internal
*/
_registerWarmConfigHash(hash) {
this.warmConfigHashes.push(hash);
}
/**
* Lazily create shared warm runner infrastructure (Lambda, SQS queue).
* Returns the manager Lambda and queue for use as EventBridge targets.
*
* @internal
*/
_ensureWarmRunnerInfra() {
if (this.warmRunnerManager && this.warmRunnerQueue) {
return { lambda: this.warmRunnerManager, queue: this.warmRunnerQueue };
}
this.warmRunnerQueue = new aws_cdk_lib_1.aws_sqs.Queue(this, 'Warm Runner Queue', {
visibilityTimeout: cdk.Duration.minutes(1),
});
this.warmRunnerManager = new warm_runner_manager_function_1.WarmRunnerManagerFunction(this, 'Warm Runner Manager', {
description: 'Manage warm GitHub runners: fill on invoke, keep alive via SQS',
environment: {
GITHUB_SECRET_ARN: this.secrets.github.secretArn,
GITHUB_PRIVATE_KEY_SECRET_ARN: this.secrets.githubPrivateKey.secretArn,
STEP_FUNCTION_ARN: this.orchestrator.stateMachineArn,
WARM_RUNNER_QUEUE_URL: this.warmRunnerQueue.queueUrl,
WARM_CONFIG_HASHES: cdk.Lazy.string({ produce: () => this.warmConfigHashes.join(',') }),
...this.extraLambdaEnv,
},
timeout: cdk.Duration.seconds(50),
logGroup: (0, utils_1.singletonLogGroup)(this, utils_1.SingletonLogType.ORCHESTRATOR),
loggingFormat: aws_cdk_lib_1.aws_lambda.LoggingFormat.JSON,
...this.extraLambdaProps,
});
this.secrets.github.grantRead(this.warmRunnerManager);
this.secrets.githubPrivateKey.grantRead(this.warmRunnerManager);
this.orchestrator.grantRead(this.warmRunnerManager);
this.orchestrator.grantStartExecution(this.warmRunnerManager);
this.orchestrator.grantExecution(this.warmRunnerManager, 'states:StopExecution');
this.warmRunnerManager.addEventSource(new aws_cdk_lib_1.aws_lambda_event_sources.SqsEventSource(this.warmRunnerQueue, {
reportBatchItemFailures: true,
maxBatchingWindow: cdk.Duration.seconds(10),
batchSize: 10,
}));
this.warmRunnerQueue.grantSendMessages(this.warmRunnerManager);
return { lambda: this.warmRunnerManager, queue: this.warmRunnerQueue };
}
}
exports.GitHubRunners = GitHubRunners;
_a = JSII_RTTI_SYMBOL_1;
GitHubRunners[_a] = { fqn: "@cloudsnorkel/cdk-github-runners.GitHubRunners", version: "0.15.1" };
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicnVubmVyLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vc3JjL3J1bm5lci50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiOzs7OztBQUFBLHlCQUF5QjtBQUN6Qix5QkFBeUI7QUFDekIsNkJBQTZCO0FBQzdCLG1DQUFtQztBQUNuQyw2Q0FZcUI7QUFDckIsMkNBQXVDO0FBQ3ZDLHFDQUF3QztBQUN4QyxtRkFBNkU7QUFDN0UsK0VBQXlFO0FBQ3pFLDJDQVVxQjtBQUNyQix1Q0FBb0M7QUFDcEMscURBQWlEO0FBQ2pELHVEQUFtRDtBQUNuRCx5RUFBb0U7QUFDcEUsbUNBQXdGO0FBQ3hGLGlGQUEyRTtBQUMzRSx1Q0FBaUQ7QUFDakQsNkRBQStEO0FBNk0vRDs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7O0dBdUNHO0FBQ0gsTUFBYSxhQUFjLFNBQVEsc0JBQVM7SUFnQzFDLFlBQVksS0FBZ0IsRUFBRSxFQUFVLEVBQVcsS0FBMEI7UUFDM0UsS0FBSyxDQUFDLEtBQUssRUFBRSxFQUFFLENBQUMsQ0FBQztRQURnQyxVQUFLLEdBQUwsS0FBSyxDQUFxQjtRQVY1RCxtQkFBYyxHQUE0QixFQUFFLENBQUM7UUFHdEQsMENBQXFDLEdBQUcsS0FBSyxDQUFDO1FBRzlDLHFCQUFnQixHQUFhLEVBQUUsQ0FBQztRQUNoQyw0QkFBdUIsR0FBRyxDQUFDLENBQUM7UUFNbEMsSUFBSSxDQUFDLE9BQU8sR0FBRyxJQUFJLGlCQUFPLENBQUMsSUFBSSxFQUFFLFNBQVMsQ0FBQyxDQUFDO1FBRTVDLElBQUksQ0FBQyxnQkFBZ0IsR0FBRztZQUN0QixHQUFHLEVBQUUsSUFBSSxDQUFDLEtBQUssRUFBRSxHQUFHO1lBQ3BCLFVBQVUsRUFBRSxJQUFJLENBQUMsS0FBSyxFQUFFLFVBQVU7WUFDbEMsaUJBQWlCLEVBQUUsSUFBSSxDQUFDLEtBQUssRUFBRSxpQkFBaUI7WUFDaEQsY0FBYyxFQUFFLElBQUksQ0FBQyxvQkFBb0IsRUFBRTtZQUMzQyxNQUFNLEVBQUUsRUFBRTtTQUNYLENBQUM7UUFDRixJQUFJLENBQUMsV0FBVyxHQUFHLElBQUkscUJBQUcsQ0FBQyxXQUFXLENBQUMsRUFBRSxjQUFjLEVBQUUsSUFBSSxDQUFDLGdCQUFnQixDQUFDLGNBQWMsRUFBRSxDQUFDLENBQUM7UUFFakcsSUFBSSxDQUFDLHNCQUFzQixDQUFDLEtBQUssQ0FBQyxDQUFDO1FBRW5DLElBQUksSUFBSSxDQUFDLEtBQUssRUFBRSxTQUFTLEVBQUUsQ0FBQztZQUMxQixJQUFJLENBQUMsU0FBUyxHQUFHLElBQUksQ0FBQyxLQUFLLENBQUMsU0FBUyxDQUFDO1FBQ3hDLENBQUM7YUFBTSxDQUFDO1lBQ04sSUFBSSxDQUFDLFNBQVMsR0FBRztnQkFDZixJQUFJLG1DQUF1QixDQUFDLElBQUksRUFBRSxXQUFXLENBQUM7Z0JBQzlDLElBQUksZ0NBQW9CLENBQUMsSUFBSSxFQUFFLFFBQVEsQ0FBQztnQkFDeEMsSUFBSSxpQ0FBcUIsQ0FBQyxJQUFJLEVBQUUsU0FBUyxDQUFDO2FBQzNDLENBQUM7UUFDSixDQUFDO1FBRUQsSUFBSSxJQUFJLENBQUMsU0FBUyxDQUFDLE1BQU0sSUFBSSxDQUFDLEVBQUUsQ0FBQztZQUMvQix5QkFBVyxDQUFDLEVBQUUsQ0FBQyxJQUFJLENBQUMsQ0FBQyxRQUFRLENBQUMsMENBQTBDLENBQUMsQ0FBQztRQUM1RSxDQUFDO1FBRUQsSUFBSSxDQUFDLHVCQUF1QixFQUFFLENBQUM7UUFFL0IsSUFBSSxDQUFDLFlBQVksR0FBRyxJQUFJLENBQUMsWUFBWSxDQUFDLEtBQUssQ0FBQyxDQUFDO1FBQzdDLElBQUksQ0FBQyxPQUFPLEdBQUcsSUFBSSw4QkFBb0IsQ0FBQyxJQUFJLEVBQUUsaUJBQWlCLEVBQUU7WUFDL0QsWUFBWSxFQUFFLElBQUksQ0FBQyxZQUFZO1lBQy9CLE9BQU8sRUFBRSxJQUFJLENBQUMsT0FBTztZQUNyQixNQUFNLEVBQUUsSUFBSSxDQUFDLEtBQUssRUFBRSxhQUFhLElBQUkscUJBQVksQ0FBQyxTQUFTLEVBQUU7WUFDN0QsU0FBUyxFQUFFLElBQUksQ0FBQyxTQUFTLENBQUMsTUFBTSxDQUEyQixDQUFDLEdBQUcsRUFBRSxDQUFDLEVBQUUsRUFBRTtnQkFDcEUsR0FBRyxDQUFDLENBQUMsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLEdBQUcsQ0FBQyxDQUFDLE1BQU0sQ0FBQztnQkFDNUIsT0FBTyxHQUFHLENBQUM7WUFDYixDQUFDLEVBQUUsRUFBRSxDQUFDO1lBQ04sc0JBQXNCLEVBQUUsSUFBSSxDQUFDLEtBQUssRUFBRSxzQkFBc0IsSUFBSSxJQUFJO1lBQ2xFLGdCQUFnQixFQUFFLElBQUksQ0FBQyxLQUFLLEVBQUUsZ0JBQWdCO1lBQzlDLGdCQUFnQixFQUFFLElBQUksQ0FBQyxnQkFBZ0I7WUFDdkMsY0FBYyxFQUFFLElBQUksQ0FBQyxjQUFjO1lBQ25DLGtCQUFrQixFQUFFLElBQUksQ0FBQyxLQUFLLEVBQUUsV0FBVyxFQUFFLFNBQVMsRUFBRTtTQUN6RCxDQUFDLENBQUM7UUFDSCxJQUFJLENBQUMsV0FBVyxHQUFHLElBQUksNENBQXVCLENBQUMsSUFBSSxFQUFFLG9CQUFvQixFQUFFO1lBQ3pFLE9BQU8sRUFBRSxJQUFJLENBQUMsT0FBTztZQUNyQixnQkFBZ0IsRUFBRSxJQUFJLENBQUMsZ0JBQWdCO1lBQ3ZDLGNBQWMsRUFBRSxJQUFJLENBQUMsY0FBYztTQUNwQyxDQUFDLENBQUM7UUFFSCxJQUFJLENBQUMsUUFBUSxHQUFHLElBQUksQ0FBQyxhQUFhLEVBQUUsQ0FBQztRQUNyQyxJQUFJLENBQUMsY0FBYyxFQUFFLENBQUM7SUFDeEIsQ0FBQztJQUVPLFlBQVksQ0FBQyxLQUEwQjtRQUM3QyxNQUFNLGtCQUFrQixHQUFHLElBQUkscUNBQW1CLENBQUMsWUFBWSxDQUM3RCxJQUFJLEVBQ0osa0JBQWtCLEVBQ2xCO1lBQ0UsY0FBYyxFQUFFLElBQUksQ0FBQyxjQUFjLEVBQUU7WUFDckMsbUJBQW1CLEVBQUUsSUFBSTtZQUN6QixVQUFVLEVBQUUsVUFBVTtTQUN2QixDQUNGLENBQUM7UUFFRixNQUFNLFVBQVUsR0FBRyxJQUFJLENBQUMsVUFBVSxFQUFFLENBQUM7UUFDckMsTUFBTSxrQkFBa0IsR0FBRyxDQUFDLEtBQUssRUFBRSxXQUFXLElBQUksR0FBRyxDQUFDLFFBQVEsQ0FBQyxPQUFPLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxTQUFTLEVBQUUsQ0FBQztRQUV2RixNQUFNLG1CQUFtQixHQUFHLElBQUkscUNBQW1CLENBQUMsY0FBYyxDQUFDLElBQUksRUFBRSxtQkFBbUIsRUFBRTtZQUM1RixLQUFLLEVBQUUsSUFBSSxDQUFDLGVBQWUsQ0FBQyxVQUFVLENBQUM7WUFDdkMsYUFBYSxFQUFFLCtCQUFhLENBQUMsYUFBYSxDQUFDLE9BQU87WUFDbEQsV0FBVyxFQUFFLCtCQUFhLENBQUMsU0FBUyxDQUFDLFVBQVUsQ0FBQztnQkFDOUMsWUFBWSxFQUFFLG9DQUFvQztnQkFDbEQsVUFBVSxFQUFFLHNDQUFzQztnQkFDbEQsS0FBSyxFQUFFLDJCQUEyQjtnQkFDbEMsSUFBSSxFQUFFLDBCQUEwQjtnQkFDaEMsY0FBYyxFQUFFLG9DQUFvQztnQkFDcEQsY0FBYyxFQUFFLDZFQUE2RSxrQkFBa0IsS0FBSzthQUNySCxDQUFDO1lBQ0YsT0FBTyxFQUFFLHFCQUFxQixFQUFFLFVBQVU7U0FDM0MsQ0FBQyxDQUFDO1FBRUgsTUFBTSxjQUFjLEdBQUcsSUFBQSwwQkFBYyxFQUFDLEdBQUcsSUFBSSxDQUFDLFNBQVMsQ0FBQyxHQUFHLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxDQUFDLENBQUMscUJBQXFCLEVBQUUsQ0FBQyxDQUFDLENBQUM7UUFDN0YsTUFBTSxnQkFBZ0IsR0FDcEIsTUFBTSxDQUFDLElBQUksQ0FBQyxjQUFjLENBQUMsQ0FBQyxNQUFNLEdBQUcsQ0FBQztZQUNwQyxDQUFDLENBQUMsa0JBQWtCLENBQUMsSUFBSSxDQUN2QixJQUFJLCtCQUFhLENBQUMsSUFBSSxDQUFDLElBQUksRUFBRSxvQkFBb0IsRUFBRTtnQkFDakQsVUFBVSxFQUFFLGNBQWM7Z0JBQzFCLFVBQVUsRUFBRSxVQUFVO2FBQ3ZCLENBQUMsQ0FDSDtZQUNELENBQUMsQ0FBQyxrQkFBa0IsQ0FBQztRQUV6QixNQUFNLGVBQWUsR0FBRyxJQUFJLCtCQUFhLENBQUMsTUFBTSxDQUFDLElBQUksRUFBRSxpQkFBaUIsQ0FBQyxDQUFDO1FBQzFFLEtBQUssTUFBTSxRQUFRLElBQUksSUFBSSxDQUFDLFNBQVMsRUFBRSxDQUFDO1lBQ3RDLE1BQU0sWUFBWSxHQUFHLFFBQVEsQ0FBQyxtQkFBbUIsQ0FDL0M7Z0JBQ0UsZUFBZSxFQUFFLCtCQUFhLENBQUMsUUFBUSxDQUFDLFFBQVEsQ0FBQyxnQkFBZ0IsQ0FBQztnQkFDbEUsY0FBYyxFQUFFLCtCQUFhLENBQUMsUUFBUSxDQUFDLFFBQVEsQ0FBQyxtQkFBbUIsQ0FBQztnQkFDcEUsZ0JBQWdCLEVBQUUsK0JBQWEsQ0FBQyxRQUFRLENBQUMsUUFBUSxDQUFDLGlCQUFpQixDQUFDO2dCQUNwRSxTQUFTLEVBQUUsK0JBQWEsQ0FBQyxRQUFRLENBQUMsUUFBUSxDQUFDLFNBQVMsQ0FBQztnQkFDckQsUUFBUSxFQUFFLCtCQUFhLENBQUMsUUFBUSxDQUFDLFFBQVEsQ0FBQyxRQUFRLENBQUM7Z0JBQ25ELGVBQWUsRUFBRSwrQkFBYSxDQUFDLFFBQVEsQ0FBQyxRQUFRLENBQUMsMEJBQTBCLENBQUM7Z0JBQzVFLFVBQVUsRUFBRSwrQkFBYSxDQUFDLFFBQVEsQ0FBQyxRQUFRLENBQUMsVUFBVSxDQUFDO2dCQUN2RCxrQkFBa0IsRUFBRSxDQUFDLEtBQUssRUFBRSxJQUFJLEVBQUUsRUFBRSxDQUFDLElBQUksQ0FBQyxrQkFBa0IsQ0FBQyxLQUFLLEVBQUUsSUFBSSxDQUFDO2FBQzFFLENBQ0YsQ0FBQztZQUNGLGVBQWUsQ0FBQyxJQUFJLENBQ2xCLCtCQUFhLENBQUMsU0FBUyxDQUFDLEdBQUcsQ0FDekIsK0JBQWEsQ0FBQyxTQUFTLENBQUMsWUFBWSxDQUFDLFlBQVksRUFBRSxRQUFRLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxDQUN2RSxFQUNELFlBQVksRUFDWjtnQkFDRSxPQUFPLEVBQUUsV0FBVyxRQUFRLENBQUMsTUFBTSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsRUFBRTthQUNqRCxDQUNGLENBQUM7UUFDSixDQUFDO1FBRUQsZUFBZSxDQUFDLFNBQVMsQ0FBQyxJQUFJLCtCQUFhLENBQUMsT0FBTyxDQUFDLElBQUksRUFBRSxlQUFlLENBQUMsQ0FBQyxDQUFDO1FBRTVFLE1BQU0sWUFBWSxHQUFHLElBQUksK0JBQWEsQ0FBQyxRQUFRLENBQUMsSUFBSSxFQUFFLGVBQWUsQ0FBQyxDQUFDLE1BQU07UUFDM0UsOEZBQThGO1FBQzlGLGdCQUFnQixDQUFDLElBQUksQ0FBQyxlQUFlLENBQUMsQ0FDdkMsQ0FBQztRQUNGLElBQUksQ0FBQyxrQkFBa0IsQ0FBQyxZQUFZLENBQUMsQ0FBQztRQUV0QyxNQUFNLFlBQVksR0FBRyxJQUFJLCtCQUFhLENBQUMsUUFBUSxDQUFDLElBQUksRUFBRSxlQUFlLENBQUMsQ0FBQyxNQUFNLENBQUMsWUFBWSxDQUFDLENBQUM7UUFFNUYsSUFBSSxLQUFLLEVBQUUsWUFBWSxFQUFFLEtBQUssSUFBSSxJQUFJLEVBQUUsQ0FBQztZQUN2QyxNQUFNLFFBQVEsR0FBRyxLQUFLLEVBQUUsWUFBWSxFQUFFLFFBQVEsSUFBSSxHQUFHLENBQUMsUUFBUSxDQUFDLE9BQU8sQ0FBQyxDQUFDLENBQUMsQ0FBQztZQUMxRSxNQUFNLFdBQVcsR0FBRyxLQUFLLEVBQUUsWUFBWSxFQUFFLFdBQVcsSUFBSSxFQUFFLENBQUM7WUFDM0QsTUFBTSxXQUFXLEdBQUcsS0FBSyxFQUFFLFlBQVksRUFBRSxXQUFXLElBQUksR0FBRyxDQUFDO1lBRTVELE1BQU0sWUFBWSxHQUFHLFFBQVEsQ0FBQyxTQUFTLEVBQUUsR0FBRyxXQUFXLElBQUksV0FBVyxHQUFHLENBQUMsV0FBVyxHQUFHLENBQUMsQ0FBQyxDQUFDO1lBQzNGLElBQUksWUFBWSxJQUFJLEdBQUcsQ0FBQyxRQUFRLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQyxDQUFDLFNBQVMsRUFBRSxFQUFFLENBQUM7Z0JBQ3JELGtJQUFrSTtnQkFDbEksd05BQXdOO2dCQUN4Tix5QkFBVyxDQUFDLEVBQUUsQ0FBQyxJQUFJLENBQUMsQ0FBQyxVQUFVLENBQUMsOENBQThDLElBQUksQ0FBQyxLQUFLLENBQUMsWUFBWSxHQUFHLEVBQUUsR0FBRyxFQUFFLENBQUMsMkZBQTJGLENBQUMsQ0FBQztZQUMvTSxDQUFDO1lBRUQsWUFBWSxDQUFDLFFBQVEsQ0FBQztnQkFDcEIsUUFBUTtnQkFDUixXQUFXO2dCQUNYLFdBQVc7Z0JBQ1gseUJBQXlCO2dCQUN6Qiw0R0FBNEc7YUFDN0csQ0FBQyxDQUFDO1FBQ0wsQ0FBQztRQUVELElBQUksVUFBd0QsQ0FBQztRQUM3RCxJQUFJLElBQUksQ0FBQyxLQUFLLEVBQUUsVUFBVSxFQUFFLENBQUM7WUFDM0IsSUFBSSxDQUFDLG9CQUFvQixHQUFHLElBQUksc0JBQUksQ0FBQyxRQUFRLENBQUMsSUFBSSxFQUFFLE1BQU0sRUFBRTtnQkFDMUQsWUFBWSxFQUFFLEtBQUssRUFBRSxVQUFVLEVBQUUsWUFBWTtnQkFDN0MsU0FBUyxFQUFFLEtBQUssRUFBRSxVQUFVLEVBQUUsWUFBWSxJQUFJLHNCQUFJLENBQUMsYUFBYSxDQUFDLFNBQVM7Z0JBQzFFLGFBQWEsRUFBRSxHQUFHLENBQUMsYUFBYSxDQUFDLE9BQU87YUFDekMsQ0FBQyxDQUFDO1lBRUgsVUFBVSxHQUFHO2dCQUNYLFdBQVcsRUFBRSxJQUFJLENBQUMsb0JBQW9CO2dCQUN0QyxvQkFBb0IsRUFBRSxLQUFLLEVBQUUsVUFBVSxFQUFFLG9CQUFvQixJQUFJLElBQUk7Z0JBQ3JFLEtBQUssRUFBRSxLQUFLLEVBQUUsVUFBVSxFQUFFLEtBQUssSUFBSSwrQkFBYSxDQUFDLFFBQVEsQ0FBQyxHQUFHO2FBQzlELENBQUM7UUFDSixDQUFDO1FBRUQsTUFBTSxZQUFZLEdBQUcsSUFBSSwrQkFBYSxDQUFDLFlBQVksQ0FDakQsSUFBSSxFQUNKLHFCQUFxQixFQUNyQjtZQUNFLGNBQWMsRUFBRSwrQkFBYSxDQUFDLGNBQWMsQ0FBQyxhQUFhLENBQUMsbUJBQW1CLENBQUMsSUFBSSxDQUFDLFlBQVksQ0FBQyxDQUFDO1lBQ2xHLElBQUksRUFBRSxVQUFVO1NBQ2pCLENBQ0YsQ0FBQztRQUVGLFlBQVksQ0FBQyxTQUFTLENBQUMsVUFBVSxDQUFDLENBQUM7UUFDbkMsWUFBWSxDQUFDLGNBQWMsQ0FBQyxVQUFVLEVBQUUsc0JBQXNCLENBQUMsQ0FBQztRQUNoRSxLQUFLLE1BQU0sUUFBUSxJQUFJLElBQUksQ0FBQyxTQUFTLEVBQUUsQ0FBQztZQUN0QyxRQUFRLENBQUMsaUJBQWlCLENBQUMsWUFBWSxDQUFDLENBQUM7UUFDM0MsQ0FBQztRQUVELE9BQU8sWUFBWSxDQUFDO0lBQ3RCLENBQUM7SUFFTyxjQUFjO1FBQ3BCLE1BQU0sSUFBSSxHQUFHLElBQUksaURBQXNCLENBQ3JDLElBQUksRUFDSixpQkFBaUIsRUFDakI7WUFDRSxXQUFXLEVBQUUsb0VBQW9FO1lBQ2pGLFdBQVcsRUFBRTtnQkFDWCxpQkFBaUIsRUFBRSxJQUFJLENBQUMsT0FBTyxDQUFDLE1BQU0sQ0FBQyxTQUFTO2dCQUNoRCw2QkFBNkIsRUFBRSxJQUFJLENBQUMsT0FBTyxDQUFDLGdCQUFnQixDQUFDLFNBQVM7Z0JBQ3RFLEdBQUcsSUFBSSxDQUFDLGNBQWM7YUFDdkI7WUFDRCxPQUFPLEVBQUUsR0FBRyxDQUFDLFFBQVEsQ0FBQyxPQUFPLENBQUMsRUFBRSxDQUFDO1lBQ2pDLFFBQVEsRUFBRSxJQUFBLHlCQUFpQixFQUFDLElBQUksRUFBRSx3QkFBZ0IsQ0FBQyxZQUFZLENBQUM7WUFDaEUsYUFBYSxFQUFFLHdCQUFNLENBQUMsYUFBYSxDQUFDLElBQUk7WUFDeEMsR0FBRyxJQUFJLENBQUMsZ0JBQWdCO1NBQ3pCLENBQ0YsQ0FBQztRQUVGLElBQUksQ0FBQyxPQUFPLENBQUMsTUFBTSxDQUFDLFNBQVMsQ0FBQyxJQUFJLENBQUMsQ0FBQztRQUNwQyxJQUFJLENBQUMsT0FBTyxDQUFDLGdCQUFnQixDQUFDLFNBQVMsQ0FBQyxJQUFJLEN