faastjs
Version:
Serverless batch computing made simple.
909 lines • 150 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.AwsImpl = exports.costSnapshot = exports.requestAwsPrices = exports.awsPrice = exports.createResponseQueueImpl = exports.awsPacker = exports.getAccountId = exports.collectGarbage = exports.clearLastGc = exports.cleanup = exports.deleteResources = exports.deleteRole = exports.initialize = exports.logUrl = exports.createLayer = exports.ensureRole = exports.ensureRoleRaw = exports.createAwsApis = exports.quietly = exports.carefully = exports.AwsMetrics = exports.defaults = exports.defaultGcWorker = void 0;
const tslib_1 = require("tslib");
const abort_controller_1 = require("@aws-sdk/abort-controller");
const client_cloudwatch_logs_1 = require("@aws-sdk/client-cloudwatch-logs");
const client_iam_1 = require("@aws-sdk/client-iam");
const client_lambda_1 = require("@aws-sdk/client-lambda");
const client_pricing_1 = require("@aws-sdk/client-pricing");
const client_s3_1 = require("@aws-sdk/client-s3");
const client_sns_1 = require("@aws-sdk/client-sns");
const client_sqs_1 = require("@aws-sdk/client-sqs");
const client_sts_1 = require("@aws-sdk/client-sts");
const crypto_1 = require("crypto");
const fs_extra_1 = require("fs-extra");
const util_1 = require("util");
const cache_1 = require("../cache");
const cost_1 = require("../cost");
const error_1 = require("../error");
const faast_1 = require("../faast");
const log_1 = require("../log");
const packer_1 = require("../packer");
const provider_1 = require("../provider");
const serialize_1 = require("../serialize");
const shared_1 = require("../shared");
const throttle_1 = require("../throttle");
const awsNpm = tslib_1.__importStar(require("./aws-npm"));
const aws_queue_1 = require("./aws-queue");
const aws_shared_1 = require("./aws-shared");
const awsTrampoline = tslib_1.__importStar(require("./aws-trampoline"));
const webpack_merge_1 = tslib_1.__importDefault(require("webpack-merge"));
exports.defaultGcWorker = (0, throttle_1.throttle)({ concurrency: 5, rate: 5, burst: 2 }, async (work, services) => {
switch (work.type) {
case "SetLogRetention":
if (await carefully(services.cloudwatch.putRetentionPolicy({
logGroupName: work.logGroupName,
retentionInDays: work.retentionInDays || 1
}))) {
log_1.log.gc(`Added retention policy %O`, work);
}
break;
case "DeleteResources":
await deleteResources(work.resources, services, log_1.log.gc);
break;
case "DeleteLayerVersion":
if (await carefully(services.lambda.deleteLayerVersion({
LayerName: work.LayerName,
VersionNumber: work.VersionNumber
}))) {
log_1.log.gc(`deleted layer %O`, work);
}
break;
}
});
exports.defaults = {
...provider_1.commonDefaults,
region: "us-west-2",
RoleName: "faast-cached-lambda-role",
memorySize: 1728,
awsLambdaOptions: {},
awsClientFactory: {},
_gcWorker: exports.defaultGcWorker
};
class AwsMetrics {
constructor() {
this.outboundBytes = 0;
this.sns64kRequests = 0;
this.sqs64kRequests = 0;
}
}
exports.AwsMetrics = AwsMetrics;
async function carefully(arg) {
try {
return await arg;
}
catch (err) {
log_1.log.warn(err);
return;
}
}
exports.carefully = carefully;
async function quietly(arg) {
try {
return await arg;
}
catch (err) {
return;
}
}
exports.quietly = quietly;
exports.createAwsApis = (0, throttle_1.throttle)({ concurrency: 1 }, async (region, awsClientFactory = {}) => {
const logger = log_1.log.awssdk.enabled
? { warn: log_1.log.awssdk, debug: log_1.log.awssdk, info: log_1.log.awssdk, error: log_1.log.awssdk }
: undefined;
const common = { maxAttempts: 6, region, logger };
const services = {
iam: awsClientFactory?.createIAM?.() ?? new client_iam_1.IAM(common),
lambda: awsClientFactory?.createLambda?.() ?? new client_lambda_1.Lambda(common),
// Special Lambda instance with configuration optimized for
// invocations.
lambda2: awsClientFactory?.createLambdaForInvocations?.() ??
new client_lambda_1.Lambda({
...common,
// Retries are handled by faast.js, not the sdk.
maxAttempts: 0
// The default 120s timeout is too short, especially for https
// mode.,
}),
cloudwatch: awsClientFactory?.createCloudWatchLogs?.() ?? new client_cloudwatch_logs_1.CloudWatchLogs(common),
sqs: awsClientFactory.createSQS?.() ?? new client_sqs_1.SQS(common),
sns: awsClientFactory.createSNS?.() ?? new client_sns_1.SNS(common),
pricing: awsClientFactory.createPricing?.() ?? new client_pricing_1.Pricing(common),
sts: awsClientFactory.createSts?.() ?? new client_sts_1.STS(common),
s3: awsClientFactory.createS3?.() ?? new client_s3_1.S3(common)
};
return services;
});
async function ensureRoleRaw(RoleName, services, createRole) {
const { iam } = services;
log_1.log.info(`Checking for cached lambda role`);
try {
const response = await iam.getRole({ RoleName });
if (!response.Role) {
throw new Error();
}
}
catch (err) {
if (!createRole) {
throw new error_1.FaastError(err instanceof Error ? err : {}, `could not find role "${RoleName}"`);
}
}
log_1.log.info(`Creating default role "${RoleName}" for faast trampoline function`);
const AssumeRolePolicyDocument = JSON.stringify({
Version: "2012-10-17",
Statement: [
{
Principal: { Service: "lambda.amazonaws.com" },
Action: "sts:AssumeRole",
Effect: "Allow"
}
]
});
const roleParams = {
AssumeRolePolicyDocument,
RoleName,
Description: "role for lambda functions created by faast",
MaxSessionDuration: 3600
};
log_1.log.info(`Calling createRole`);
const PolicyArn = "arn:aws:iam::aws:policy/AdministratorAccess";
try {
const roleResponse = await iam.createRole(roleParams);
log_1.log.info(`Attaching administrator role policy`);
await iam.attachRolePolicy({ RoleName, PolicyArn });
return roleResponse.Role;
}
catch (err) {
if (err instanceof client_iam_1.EntityAlreadyExistsException) {
await (0, shared_1.sleep)(5000);
const roleResponse = await iam.getRole({ RoleName });
await iam.attachRolePolicy({ RoleName, PolicyArn });
return roleResponse.Role;
}
throw new error_1.FaastError(err ?? {}, `failed to create role "${RoleName}"`);
}
}
exports.ensureRoleRaw = ensureRoleRaw;
exports.ensureRole = (0, throttle_1.throttle)({ concurrency: 1, rate: 2, memoize: false, retry: 12 }, ensureRoleRaw);
const ResponseQueueId = awsTrampoline.INVOCATION_TEST_QUEUE;
const emptyFcall = { callId: "0", modulePath: "", name: "", args: "", ResponseQueueId };
async function createLayer(lambda, packageJson, useDependencyCaching, FunctionName, region, retentionInDays, awsLambdaOptions) {
if (!packageJson) {
return;
}
log_1.log.info(`Building node_modules`);
const packageJsonContents = typeof packageJson === "string"
? (await (0, fs_extra_1.readFile)(packageJson)).toString()
: JSON.stringify(packageJson);
let LayerName;
if (useDependencyCaching) {
const hasher = (0, crypto_1.createHash)("sha256");
hasher.update(packageJsonContents);
hasher.update(JSON.stringify(awsLambdaOptions.Architectures ?? ""));
const cacheKey = hasher.digest("hex");
LayerName = `faast-${cacheKey}`;
const layers = await quietly(lambda.listLayerVersions({ LayerName }));
if (layers?.LayerVersions?.length ?? 0 > 0) {
const [{ Version, LayerVersionArn, CreatedDate }] = layers?.LayerVersions ?? [];
if (!(0, shared_1.hasExpired)(CreatedDate, retentionInDays) && Version && LayerVersionArn) {
return { Version, LayerVersionArn, LayerName };
}
}
}
else {
LayerName = FunctionName;
}
try {
const faastModule = await (0, faast_1.faastAws)(awsNpm, {
region,
timeout: 300,
memorySize: 2048,
mode: "https",
gc: "off",
maxRetries: 0,
webpackOptions: {
externals: []
},
awsLambdaOptions
});
try {
const installArgs = {
packageJsonContents,
LayerName,
FunctionName,
region,
retentionInDays
};
const { installLog, layerInfo } = await faastModule.functions.npmInstall(installArgs);
log_1.log.info(installLog);
return layerInfo;
}
finally {
await faastModule.cleanup();
}
}
catch (err) {
console.log(err);
throw new error_1.FaastError(err instanceof Error ? err : {}, "failed to create lambda layer from packageJson");
}
}
exports.createLayer = createLayer;
function logUrl(state) {
const { region, FunctionName } = state.resources;
return (0, aws_shared_1.getLogUrl)(region, FunctionName);
}
exports.logUrl = logUrl;
exports.initialize = (0, throttle_1.throttle)({ concurrency: Infinity, rate: 2 }, async (fModule, nonce, options) => {
const { region, timeout, memorySize, env, concurrency, mode } = options;
if (concurrency > 100 && mode !== "queue") {
log_1.log.warn(`Consider using queue mode for higher levels of concurrency:`);
log_1.log.warn(`https://faastjs.org/docs/api/faastjs.commonoptions.mode`);
}
log_1.log.info(`Creating AWS APIs`);
const services = await (0, exports.createAwsApis)(region, options.awsClientFactory);
const metrics = new AwsMetrics();
const { lambda } = services;
const FunctionName = `faast-${nonce}`;
const { packageJson, useDependencyCaching, description } = options;
async function createFunctionRequest(Code, Role, responseQueueArn, layerInfo) {
const { Layers = [], ...rest } = options.awsLambdaOptions;
if (layerInfo) {
Layers.push(layerInfo.LayerVersionArn);
}
const request = {
FunctionName,
Role,
Runtime: "nodejs18.x",
Handler: "index.trampoline",
Code,
Description: "faast trampoline function",
Timeout: timeout,
MemorySize: memorySize,
Environment: { Variables: env },
Layers,
...rest
};
log_1.log.info(`createFunctionRequest: %O`, request);
let func;
try {
func = await lambda.createFunction(request);
await (0, client_lambda_1.waitUntilFunctionExists)({ client: lambda, maxWaitTime: 120 }, { FunctionName });
}
catch (err) {
if (err instanceof client_lambda_1.ResourceConflictException) {
func = (await lambda.getFunction({ FunctionName })).Configuration;
}
else {
throw new error_1.FaastError(err ?? {}, "Create function failure");
}
}
log_1.log.info(`Created function ${func.FunctionName}, FunctionArn: ${func.FunctionArn} [${description}]`);
log_1.log.minimal(`Created function ${func.FunctionName} [${description}]`);
try {
await (0, client_lambda_1.waitUntilFunctionActiveV2)({ client: lambda, maxWaitTime: 240 }, { FunctionName });
}
catch (err) {
throw new error_1.FaastError(err ?? {}, "Lambda function did not enter Active state");
}
log_1.log.info(`Function ${func.FunctionName} is Active`);
try {
const config = await (0, throttle_1.retryOp)((err, n) => n < 5 && err instanceof client_lambda_1.ResourceNotFoundException, () => lambda.putFunctionEventInvokeConfig({
FunctionName,
MaximumRetryAttempts: 0,
MaximumEventAgeInSeconds: 120,
DestinationConfig: {
OnFailure: { Destination: responseQueueArn }
}
}));
log_1.log.info(`Function event invocation config: %O`, config);
}
catch (err) {
throw new error_1.FaastError(err, "putFunctionEventInvokeConfig failure");
}
return func;
}
const { wrapperVerbose } = options.debugOptions;
async function createCodeBundle() {
const { timeout, childProcess, mode } = options;
const hasLambdaTimeoutBug = mode !== "queue" && timeout >= 180;
const childProcessTimeoutMs = hasLambdaTimeoutBug && childProcess ? (timeout - 5) * 1000 : 0;
const childProcessMemoryLimitMb = options.childProcessMemoryMb;
const wrapperOptions = {
wrapperVerbose,
childProcessTimeoutMs,
childProcessMemoryLimitMb
};
const bundle = awsPacker(fModule, options, wrapperOptions, FunctionName);
return { ZipFile: await (0, shared_1.streamToBuffer)((await bundle).archive) };
}
const { RoleName } = options;
const state = {
resources: {
FunctionName,
RoleName,
region,
logGroupName: (0, aws_shared_1.getLogGroupName)(FunctionName)
},
services,
metrics,
options
};
const { gc, retentionInDays, _gcWorker: gcWorker } = options;
if (gc === "auto" || gc === "force") {
log_1.log.gc(`Starting garbage collector`);
state.gcPromise = collectGarbage(gcWorker, services, region, retentionInDays, gc).catch(err => {
log_1.log.gc(`Garbage collection error: ${err}`);
return "skipped";
});
}
try {
log_1.log.info(`Creating lambda function`);
const rolePromise = (0, exports.ensureRole)(RoleName, services, RoleName === exports.defaults.RoleName);
const responseQueuePromise = createResponseQueueImpl(state, FunctionName);
const pricingPromise = (0, exports.requestAwsPrices)(services.pricing, region);
const codeBundlePromise = createCodeBundle();
// Ensure role exists before creating lambda layer, which also needs the role.
const role = await rolePromise;
const layerPromise = createLayer(services.lambda, packageJson, useDependencyCaching, FunctionName, region, retentionInDays, options.awsLambdaOptions);
const codeBundle = await codeBundlePromise;
const responseQueueArn = await responseQueuePromise;
const layer = await layerPromise;
if (layer) {
state.resources.layer = layer;
}
let lambdaFnArn;
const retryable = [
/role/,
/KMS Exception/,
/internal service error/,
/Layer version/
];
const shouldRetry = (err, n) => n < 5 && !!retryable.find(regex => err?.message?.match(regex));
await (0, throttle_1.retryOp)(shouldRetry, async () => {
try {
const lambdaFn = await createFunctionRequest(codeBundle, role.Arn, responseQueueArn, layer);
lambdaFnArn = lambdaFn.FunctionArn;
// If the role for the lambda function was created
// recently, test that the role works by invoking the
// function. If an exception occurs, the function is
// deleted and re-deployed. Empirically, this is the way
// to ensure successful lambda creation when an IAM role
// is recently created.
if (Date.now() - role.CreateDate.getTime() < 300 * 1000) {
const { metrics } = state;
const fn = FunctionName;
const never = new Promise(_ => { });
await (0, throttle_1.retryOp)(1, () => invokeHttps(lambda, fn, emptyFcall, metrics, never));
}
}
catch (err) {
/* c8 ignore next */ {
await lambda.deleteFunction({ FunctionName }).catch(_ => { });
throw new error_1.FaastError(err, `New lambda function ${FunctionName} failed invocation test`);
}
}
});
const { mode } = options;
if (mode === "queue") {
await createRequestQueueImpl(state, FunctionName, lambdaFnArn);
}
await pricingPromise;
log_1.log.info(`Lambda function initialization complete.`);
return state;
}
catch (err) {
try {
await cleanup(state, {
deleteResources: true,
deleteCaches: false,
gcTimeout: 30
});
}
catch { }
throw new error_1.FaastError({ cause: err, name: error_1.FaastErrorNames.ECREATE }, "failed to initialize cloud function");
}
});
async function invoke(state, call, cancel) {
const { metrics, services, resources, options } = state;
switch (options.mode) {
case "auto":
case "https":
const { lambda2 } = services;
const { FunctionName } = resources;
await invokeHttps(lambda2, FunctionName, call, metrics, cancel);
return;
case "queue":
const { sns } = services;
const { RequestTopicArn } = resources;
try {
await (0, aws_queue_1.publishFunctionCallMessage)(sns, RequestTopicArn, call, metrics);
}
catch (err) {
throw new error_1.FaastError(err, `invoke sns error ${(0, util_1.inspect)(call, undefined, 9)}`);
}
return;
}
}
function poll(state, cancel) {
return (0, aws_queue_1.receiveMessages)(state.services.sqs, state.resources.ResponseQueueUrl, state.metrics, cancel);
}
function responseQueueId(state) {
return state.resources.ResponseQueueUrl;
}
async function invokeHttps(lambda, FunctionName, message, metrics, cancel) {
const abortController = new abort_controller_1.AbortController();
const request = {
FunctionName,
Payload: (0, serialize_1.serializeToUint8Array)(message),
LogType: "None"
};
const awsRequest = lambda.invoke(request, { abortSignal: abortController.signal });
const rawResponse = await Promise.race([awsRequest, cancel]);
if (!rawResponse) {
log_1.log.info(`cancelling lambda invoke`);
abortController.abort();
return;
}
metrics.outboundBytes += rawResponse.Payload?.byteLength ?? 0;
if (rawResponse.LogResult) {
log_1.log.info(Buffer.from(rawResponse.LogResult, "base64").toString());
}
if (rawResponse.FunctionError) {
const msg = new TextDecoder().decode(rawResponse.Payload);
const error = (0, aws_queue_1.processAwsErrorMessage)(msg);
throw error;
}
}
async function deleteRole(RoleName, iam) {
const policies = await carefully(iam.listAttachedRolePolicies({ RoleName }));
const AttachedPolicies = policies?.AttachedPolicies ?? [];
await Promise.all(AttachedPolicies.map(p => p.PolicyArn).map(PolicyArn => carefully(iam.detachRolePolicy({ RoleName, PolicyArn }))));
const rolePolicyListResponse = await carefully(iam.listRolePolicies({ RoleName }));
const RolePolicies = rolePolicyListResponse?.PolicyNames ?? [];
await Promise.all(RolePolicies.map(PolicyName => carefully(iam.deleteRolePolicy({ RoleName, PolicyName }))));
await carefully(iam.deleteRole({ RoleName }));
}
exports.deleteRole = deleteRole;
async function deleteResources(resources, services, output = log_1.log.info) {
const { FunctionName, RoleName, region, RequestTopicArn, ResponseQueueUrl, ResponseQueueArn, SNSLambdaSubscriptionArn, logGroupName, layer, Bucket, ...rest } = resources;
const _exhaustiveCheck = {};
const { lambda, sqs, sns, iam, s3, cloudwatch } = services;
if (SNSLambdaSubscriptionArn) {
if (await quietly(sns.unsubscribe({ SubscriptionArn: SNSLambdaSubscriptionArn }))) {
output(`Deleted request queue subscription to lambda`);
}
}
if (RoleName) {
await deleteRole(RoleName, iam);
}
if (RequestTopicArn) {
if (await quietly(sns.deleteTopic({ TopicArn: RequestTopicArn }))) {
output(`Deleted request queue topic: ${RequestTopicArn}`);
}
}
if (ResponseQueueUrl) {
if (await quietly(sqs.deleteQueue({ QueueUrl: ResponseQueueUrl }))) {
output(`Deleted response queue: ${ResponseQueueUrl}`);
}
}
if (layer) {
if (await quietly(lambda.deleteLayerVersion({
LayerName: layer.LayerName,
VersionNumber: layer.Version
}))) {
output(`Deleted lambda layer: ${layer.LayerName}:${layer.Version}`);
}
}
if (Bucket) {
const objects = await quietly(s3.listObjectsV2({ Bucket, Prefix: "faast-" }));
if (objects) {
const keys = (objects.Contents || []).map(elem => ({ Key: elem.Key }));
if (await quietly(s3.deleteObjects({ Bucket, Delete: { Objects: keys } }))) {
output(`Deleted s3 objects: ${keys.length} objects in bucket ${Bucket}`);
}
}
if (await quietly(s3.deleteBucket({ Bucket }))) {
output(`Deleted s3 bucket: ${Bucket}`);
}
}
if (FunctionName) {
if (await quietly(lambda.deleteFunction({ FunctionName }))) {
output(`Deleted function: ${FunctionName}`);
}
}
if (logGroupName) {
if (await quietly(cloudwatch.deleteLogGroup({ logGroupName }))) {
output(`Deleted log group: ${logGroupName}`);
}
}
}
exports.deleteResources = deleteResources;
async function addLogRetentionPolicy(FunctionName, cloudwatch) {
const logGroupName = (0, aws_shared_1.getLogGroupName)(FunctionName);
const response = await quietly(cloudwatch.putRetentionPolicy({ logGroupName, retentionInDays: 1 }));
if (response !== undefined) {
log_1.log.info(`Added 1 day retention policy to log group ${logGroupName}`);
}
}
async function cleanup(state, options) {
log_1.log.info(`aws cleanup starting.`);
if (state.gcPromise) {
log_1.log.info(`Waiting for garbage collection...`);
await state.gcPromise;
log_1.log.info(`Garbage collection done.`);
}
if (options.deleteResources) {
log_1.log.info(`Cleaning up infrastructure for ${state.resources.FunctionName}...`);
await addLogRetentionPolicy(state.resources.FunctionName, state.services.cloudwatch);
// Don't delete cached role. It may be in use by other instances of
// faast. Don't delete logs. They are often useful. By default log
// stream retention will be 1 day, and gc will clean out the log group
// after the streams are expired. Don't delete a lambda layer that is
// used to cache packages.
const { logGroupName, RoleName, layer, ...rest } = state.resources;
await deleteResources(rest, state.services);
if (!state.options.useDependencyCaching || options.deleteCaches) {
await deleteResources({ layer }, state.services);
}
}
log_1.log.info(`aws cleanup done.`);
}
exports.cleanup = cleanup;
const logGroupNameRegexp = new RegExp(`^/aws/lambda/(faast-${shared_1.uuidv4Pattern})$`);
function functionNameFromLogGroup(logGroupName) {
const match = logGroupName.match(logGroupNameRegexp);
return match && match[1];
}
let lastGc;
function clearLastGc() {
lastGc = undefined;
}
exports.clearLastGc = clearLastGc;
async function collectGarbage(executor, services, region, retentionInDays, mode) {
if (executor === exports.defaultGcWorker) {
if (mode === "auto") {
if (lastGc && Date.now() <= lastGc + 3600 * 1000) {
return "skipped";
}
const gcEntry = await cache_1.caches.awsGc.get("gc");
if (gcEntry) {
try {
const lastGcPersistent = JSON.parse(gcEntry.toString());
if (lastGcPersistent &&
typeof lastGcPersistent === "number" &&
Date.now() <= lastGcPersistent + 3600 * 1000) {
lastGc = lastGcPersistent;
return "skipped";
}
}
catch (err) {
log_1.log.warn(err);
}
}
}
lastGc = Date.now();
cache_1.caches.awsGc.set("gc", lastGc.toString());
}
const promises = [];
function scheduleWork(work) {
if (executor === exports.defaultGcWorker) {
log_1.log.gc(`Scheduling work pushing promise: %O`, work);
}
promises.push(executor(work, services));
}
const functionsWithLogGroups = new Set();
const accountId = await getAccountId(services.sts);
for await (const { logGroups = [] } of (0, client_cloudwatch_logs_1.paginateDescribeLogGroups)({ client: services.cloudwatch }, { logGroupNamePrefix: "/aws/lambda/faast-" })) {
logGroups.forEach(g => {
const FunctionName = functionNameFromLogGroup(g.logGroupName);
functionsWithLogGroups.add(FunctionName);
});
log_1.log.gc(`Log groups size: ${logGroups.length}`);
garbageCollectLogGroups(logGroups, retentionInDays, region, accountId, scheduleWork);
}
for await (const { Functions = [] } of (0, client_lambda_1.paginateListFunctions)({ client: services.lambda }, {})) {
const fnPattern = new RegExp(`^faast-${shared_1.uuidv4Pattern}$`);
const funcs = (Functions || [])
.filter(fn => fn.FunctionName.match(fnPattern))
.filter(fn => !functionsWithLogGroups.has(fn.FunctionName))
.filter(fn => (0, shared_1.hasExpired)(fn.LastModified, retentionInDays))
.map(fn => fn.FunctionName);
deleteGarbageFunctions(region, accountId, funcs, scheduleWork);
}
// Collect Lambda Layers
for await (const { Layers = [] } of (0, client_lambda_1.paginateListLayers)({ client: services.lambda }, { CompatibleRuntime: "nodejs" })) {
for (const layer of Layers) {
if (layer.LayerName.match(/^faast-/)) {
for await (const { LayerVersions = [] } of (0, client_lambda_1.paginateListLayerVersions)({ client: services.lambda }, { LayerName: layer.LayerName, CompatibleRuntime: "nodejs" })) {
for (const layerVersion of LayerVersions) {
if ((0, shared_1.hasExpired)(layerVersion.CreatedDate, retentionInDays)) {
scheduleWork({
type: "DeleteLayerVersion",
LayerName: layer.LayerName,
VersionNumber: layerVersion.Version
});
}
}
}
}
}
}
log_1.log.gc(`Awaiting ${promises.length} scheduled work promises`);
await Promise.all(promises);
return "done";
}
exports.collectGarbage = collectGarbage;
async function getAccountId(sts) {
const response = await sts.getCallerIdentity({});
const { Account, Arn, UserId } = response;
log_1.log.info(`Account ID: %O`, { Account, Arn, UserId });
return response.Account;
}
exports.getAccountId = getAccountId;
function garbageCollectLogGroups(logGroups, retentionInDays, region, accountId, scheduleWork) {
const logGroupsMissingRetentionPolicy = logGroups.filter(g => g.retentionInDays === undefined);
log_1.log.gc(`Log groups missing retention: ${logGroupsMissingRetentionPolicy.length}`);
logGroupsMissingRetentionPolicy.forEach(g => {
scheduleWork({
type: "SetLogRetention",
logGroupName: g.logGroupName,
retentionInDays
});
});
const garbageFunctions = logGroups
.filter(g => (0, shared_1.hasExpired)(g.creationTime, retentionInDays))
.map(g => functionNameFromLogGroup(g.logGroupName))
.filter(shared_1.defined);
deleteGarbageFunctions(region, accountId, garbageFunctions, scheduleWork);
}
function deleteGarbageFunctions(region, accountId, garbageFunctions, scheduleWork) {
garbageFunctions.forEach(FunctionName => {
const resources = {
FunctionName,
region,
RoleName: "",
RequestTopicArn: getSNSTopicArn(region, accountId, FunctionName),
ResponseQueueUrl: getResponseQueueUrl(region, accountId, FunctionName),
logGroupName: (0, aws_shared_1.getLogGroupName)(FunctionName),
Bucket: FunctionName
};
scheduleWork({ type: "DeleteResources", resources });
});
}
async function awsPacker(functionModule, options, wrapperOptions, FunctionName) {
const webpackOptions = (0, webpack_merge_1.default)(options.webpackOptions ?? {}, {
externals: [new RegExp("^@aws-sdk/")]
});
return (0, packer_1.packer)(awsTrampoline, functionModule, { ...options, webpackOptions }, wrapperOptions, FunctionName);
}
exports.awsPacker = awsPacker;
function getSNSTopicName(FunctionName) {
return `${FunctionName}-Requests`;
}
function getSNSTopicArn(region, accountId, FunctionName) {
const TopicName = getSNSTopicName(FunctionName);
return `arn:aws:sns:${region}:${accountId}:${TopicName}`;
}
function getSQSName(FunctionName) {
return `${FunctionName}-Responses`;
}
function getResponseQueueUrl(region, accountId, FunctionName) {
const queueName = getSQSName(FunctionName);
return `https://sqs.${region}.amazonaws.com/${accountId}/${queueName}`;
}
function createRequestQueueImpl(state, FunctionName, FunctionArn) {
const { sns, lambda } = state.services;
const { resources } = state;
log_1.log.info(`Creating SNS request topic`);
const createTopicPromise = (0, aws_queue_1.createSNSTopic)(sns, getSNSTopicName(FunctionName));
const assignRequestTopicArnPromise = createTopicPromise.then(topic => (resources.RequestTopicArn = topic));
const addPermissionsPromise = createTopicPromise.then(topic => {
log_1.log.info(`Adding SNS invoke permissions to function`);
return addSnsInvokePermissionsToFunction(FunctionName, topic, lambda);
});
const subscribePromise = createTopicPromise.then(topic => {
log_1.log.info(`Subscribing SNS to invoke lambda function`);
return sns.subscribe({
TopicArn: topic,
Protocol: "lambda",
Endpoint: FunctionArn
});
});
const assignSNSResponsePromise = subscribePromise.then(snsResponse => (resources.SNSLambdaSubscriptionArn = snsResponse.SubscriptionArn));
return Promise.all([
createTopicPromise,
assignRequestTopicArnPromise,
addPermissionsPromise,
subscribePromise,
assignSNSResponsePromise
]);
}
async function createResponseQueueImpl(state, FunctionName) {
const { sqs } = state.services;
const { resources } = state;
log_1.log.info(`Creating SQS response queue`);
const { QueueUrl, QueueArn } = await (0, aws_queue_1.createSQSQueue)(getSQSName(FunctionName), 60, sqs);
resources.ResponseQueueUrl = QueueUrl;
resources.ResponseQueueArn = QueueArn;
log_1.log.info(`Created response queue`);
return QueueArn;
}
exports.createResponseQueueImpl = createResponseQueueImpl;
function addSnsInvokePermissionsToFunction(FunctionName, RequestTopicArn, lambda) {
return lambda
.addPermission({
FunctionName,
Action: "lambda:InvokeFunction",
Principal: "sns.amazonaws.com",
StatementId: `${FunctionName}-Invoke`,
SourceArn: RequestTopicArn
})
.catch(err => {
if (err instanceof client_lambda_1.ResourceConflictException) {
}
else {
throw err;
}
});
}
const locations = {
"us-east-1": "US East (N. Virginia)",
"us-east-2": "US East (Ohio)",
"us-west-1": "US West (N. California)",
"us-west-2": "US West (Oregon)",
"ca-central-1": "Canada (Central)",
"eu-central-1": "EU (Frankfurt)",
"eu-west-1": "EU (Ireland)",
"eu-west-2": "EU (London)",
"eu-west-3": "EU (Paris)",
"ap-northeast-1": "Asia Pacific (Tokyo)",
"ap-northeast-2": "Asia Pacific (Seoul)",
"ap-northeast-3": "Asia Pacific (Osaka-Local)",
"ap-southeast-1": "Asia Pacific (Singapore)",
"ap-southeast-2": "Asia Pacific (Sydney)",
"ap-south-1": "Asia Pacific (Mumbai)",
"sa-east-1": "South America (São Paulo)"
};
exports.awsPrice = (0, throttle_1.throttle)({ concurrency: 6, rate: 5, memoize: true, cache: cache_1.caches.awsPrices }, async (pricing, ServiceCode, filter) => {
try {
function first(obj) {
return obj[Object.keys(obj)[0]];
}
function extractPrice(obj) {
const prices = Object.keys(obj.priceDimensions).map(key => Number(obj.priceDimensions[key].pricePerUnit.USD));
return Math.max(...prices);
}
const priceResult = await pricing.getProducts({
ServiceCode,
Filters: Object.keys(filter).map(key => ({
Field: key,
Type: "TERM_MATCH",
Value: filter[key]
}))
});
if (priceResult.PriceList.length > 1) {
log_1.log.warn(`Price query returned more than one product '${ServiceCode}' ($O)`, filter);
priceResult.PriceList.forEach(p => log_1.log.warn(`%O`, p));
}
const pList = priceResult.PriceList[0];
const price = extractPrice(first(pList.terms.OnDemand));
return price;
}
catch (err) {
/* c8 ignore next */
{
if (err instanceof client_pricing_1.InternalErrorException) {
log_1.log.warn(`Could not get AWS pricing for '${ServiceCode}' (%O)`, filter);
log_1.log.warn(err);
}
throw new error_1.FaastError(err instanceof Error ? err : {}, `failed to get AWS pricing for "${ServiceCode}"`);
}
}
});
const requestAwsPrices = async (pricing, region) => {
const location = locations[region];
/* c8 ignore next */
return {
lambdaPerRequest: await (0, exports.awsPrice)(pricing, "AWSLambda", {
location,
group: "AWS-Lambda-Requests"
}).catch(_ => 0.0000002),
lambdaPerGbSecond: await (0, exports.awsPrice)(pricing, "AWSLambda", {
location,
group: "AWS-Lambda-Duration"
}).catch(_ => 0.00001667),
snsPer64kPublish: await (0, exports.awsPrice)(pricing, "AmazonSNS", {
location,
group: "SNS-Requests-Tier1"
}).catch(_ => 0.5 / 1e6),
sqsPer64kRequest: await (0, exports.awsPrice)(pricing, "AWSQueueService", {
location,
group: "SQS-APIRequest-Tier1",
queueType: "Standard"
}).catch(_ => 0.4 / 1e6),
dataOutPerGb: await (0, exports.awsPrice)(pricing, "AWSDataTransfer", {
fromLocation: location,
transferType: "AWS Outbound"
}).catch(_ => 0.09),
logsIngestedPerGb: await (0, exports.awsPrice)(pricing, "AmazonCloudWatch", {
location,
group: "Ingested Logs",
groupDescription: "Existing system, application, and custom log files"
}).catch(_ => 0.5)
};
};
exports.requestAwsPrices = requestAwsPrices;
async function costSnapshot(state, stats) {
const { region } = state.resources;
const prices = await (0, exports.requestAwsPrices)(state.services.pricing, region);
const costMetrics = [];
const { memorySize = exports.defaults.memorySize } = state.options;
const billedTimeStats = stats.estimatedBilledTime;
const seconds = (billedTimeStats.mean / 1000) * billedTimeStats.samples || 0;
const provisionedGb = memorySize / 1024;
const functionCallDuration = new cost_1.CostMetric({
name: "functionCallDuration",
pricing: prices.lambdaPerGbSecond * provisionedGb,
unit: "second",
measured: seconds,
comment: `https://aws.amazon.com/lambda/pricing (rate = ${prices.lambdaPerGbSecond.toFixed(8)}/(GB*second) * ${provisionedGb} GB = ${(prices.lambdaPerGbSecond * provisionedGb).toFixed(8)}/second)`
});
costMetrics.push(functionCallDuration);
const functionCallRequests = new cost_1.CostMetric({
name: "functionCallRequests",
pricing: prices.lambdaPerRequest,
measured: stats.invocations,
unit: "request",
comment: "https://aws.amazon.com/lambda/pricing"
});
costMetrics.push(functionCallRequests);
const { metrics } = state;
const outboundDataTransfer = new cost_1.CostMetric({
name: "outboundDataTransfer",
pricing: prices.dataOutPerGb,
measured: metrics.outboundBytes / 2 ** 30,
unit: "GB",
comment: "https://aws.amazon.com/ec2/pricing/on-demand/#Data_Transfer"
});
costMetrics.push(outboundDataTransfer);
const sqs = new cost_1.CostMetric({
name: "sqs",
pricing: prices.sqsPer64kRequest,
measured: metrics.sqs64kRequests,
unit: "request",
comment: "https://aws.amazon.com/sqs/pricing"
});
costMetrics.push(sqs);
const sns = new cost_1.CostMetric({
name: "sns",
pricing: prices.snsPer64kPublish,
measured: metrics.sns64kRequests,
unit: "request",
comment: "https://aws.amazon.com/sns/pricing"
});
costMetrics.push(sns);
const logIngestion = new cost_1.CostMetric({
name: "logIngestion",
pricing: prices.logsIngestedPerGb,
measured: 0,
unit: "GB",
comment: "https://aws.amazon.com/cloudwatch/pricing/ - Log ingestion costs not currently included.",
informationalOnly: true
});
costMetrics.push(logIngestion);
return new cost_1.CostSnapshot("aws", state.options, stats, costMetrics);
}
exports.costSnapshot = costSnapshot;
exports.AwsImpl = {
name: "aws",
initialize: exports.initialize,
defaults: exports.defaults,
cleanup,
costSnapshot,
logUrl,
invoke,
poll,
responseQueueId
};
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiYXdzLWZhYXN0LmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vc3JjL2F3cy9hd3MtZmFhc3QudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7OztBQUFBLGdFQUE0RDtBQUM1RCw0RUFJeUM7QUFDekMsb0RBSTZCO0FBQzdCLDBEQVlnQztBQUNoQyw0REFBMEU7QUFDMUUsa0RBQXdDO0FBQ3hDLG9EQUEwQztBQUMxQyxvREFBMEM7QUFDMUMsb0RBQTBDO0FBQzFDLG1DQUFvQztBQUNwQyx1Q0FBb0M7QUFFcEMsK0JBQStCO0FBQy9CLG9DQUFrQztBQUNsQyxrQ0FBbUQ7QUFDbkQsb0NBQXVEO0FBQ3ZELG9DQUFvQztBQUNwQyxnQ0FBNkI7QUFDN0Isc0NBQWlEO0FBQ2pELDBDQVFxQjtBQUNyQiw0Q0FBcUQ7QUFDckQsc0NBQXNGO0FBQ3RGLDBDQUFnRDtBQUVoRCwwREFBb0M7QUFFcEMsMkNBTXFCO0FBQ3JCLDZDQUEwRDtBQUMxRCx3RUFBa0Q7QUFDbEQsMEVBQWtDO0FBRXJCLFFBQUEsZUFBZSxHQUFHLElBQUEsbUJBQVEsRUFDbkMsRUFBRSxXQUFXLEVBQUUsQ0FBQyxFQUFFLElBQUksRUFBRSxDQUFDLEVBQUUsS0FBSyxFQUFFLENBQUMsRUFBRSxFQUNyQyxLQUFLLEVBQUUsSUFBZSxFQUFFLFFBQXFCLEVBQUUsRUFBRTtJQUM3QyxRQUFRLElBQUksQ0FBQyxJQUFJLEVBQUUsQ0FBQztRQUNoQixLQUFLLGlCQUFpQjtZQUNsQixJQUNJLE1BQU0sU0FBUyxDQUNYLFFBQVEsQ0FBQyxVQUFVLENBQUMsa0JBQWtCLENBQUM7Z0JBQ25DLFlBQVksRUFBRSxJQUFJLENBQUMsWUFBWTtnQkFDL0IsZUFBZSxFQUFFLElBQUksQ0FBQyxlQUFlLElBQUksQ0FBQzthQUM3QyxDQUFDLENBQ0wsRUFDSCxDQUFDO2dCQUNDLFNBQUcsQ0FBQyxFQUFFLENBQUMsMkJBQTJCLEVBQUUsSUFBSSxDQUFDLENBQUM7WUFDOUMsQ0FBQztZQUNELE1BQU07UUFDVixLQUFLLGlCQUFpQjtZQUNsQixNQUFNLGVBQWUsQ0FBQyxJQUFJLENBQUMsU0FBUyxFQUFFLFFBQVEsRUFBRSxTQUFHLENBQUMsRUFBRSxDQUFDLENBQUM7WUFDeEQsTUFBTTtRQUNWLEtBQUssb0JBQW9CO1lBQ3JCLElBQ0ksTUFBTSxTQUFTLENBQ1gsUUFBUSxDQUFDLE1BQU0sQ0FBQyxrQkFBa0IsQ0FBQztnQkFDL0IsU0FBUyxFQUFFLElBQUksQ0FBQyxTQUFTO2dCQUN6QixhQUFhLEVBQUUsSUFBSSxDQUFDLGFBQWE7YUFDcEMsQ0FBQyxDQUNMLEVBQ0gsQ0FBQztnQkFDQyxTQUFHLENBQUMsRUFBRSxDQUFDLGtCQUFrQixFQUFFLElBQUksQ0FBQyxDQUFDO1lBQ3JDLENBQUM7WUFDRCxNQUFNO0lBQ2QsQ0FBQztBQUNMLENBQUMsQ0FDSixDQUFDO0FBeUlTLFFBQUEsUUFBUSxHQUF5QjtJQUN4QyxHQUFHLHlCQUFjO0lBQ2pCLE1BQU0sRUFBRSxXQUFXO0lBQ25CLFFBQVEsRUFBRSwwQkFBMEI7SUFDcEMsVUFBVSxFQUFFLElBQUk7SUFDaEIsZ0JBQWdCLEVBQUUsRUFBRTtJQUNwQixnQkFBZ0IsRUFBRSxFQUFFO0lBQ3BCLFNBQVMsRUFBRSx1QkFBZTtDQUM3QixDQUFDO0FBV0YsTUFBYSxVQUFVO0lBQXZCO1FBQ0ksa0JBQWEsR0FBRyxDQUFDLENBQUM7UUFDbEIsbUJBQWMsR0FBRyxDQUFDLENBQUM7UUFDbkIsbUJBQWMsR0FBRyxDQUFDLENBQUM7SUFDdkIsQ0FBQztDQUFBO0FBSkQsZ0NBSUM7QUEyRE0sS0FBSyxVQUFVLFNBQVMsQ0FBSSxHQUFlO0lBQzlDLElBQUksQ0FBQztRQUNELE9BQU8sTUFBTSxHQUFHLENBQUM7SUFDckIsQ0FBQztJQUFDLE9BQU8sR0FBRyxFQUFFLENBQUM7UUFDWCxTQUFHLENBQUMsSUFBSSxDQUFDLEdBQUcsQ0FBQyxDQUFDO1FBQ2QsT0FBTztJQUNYLENBQUM7QUFDTCxDQUFDO0FBUEQsOEJBT0M7QUFFTSxLQUFLLFVBQVUsT0FBTyxDQUFJLEdBQWU7SUFDNUMsSUFBSSxDQUFDO1FBQ0QsT0FBTyxNQUFNLEdBQUcsQ0FBQztJQUNyQixDQUFDO0lBQUMsT0FBTyxHQUFHLEVBQUUsQ0FBQztRQUNYLE9BQU87SUFDWCxDQUFDO0FBQ0wsQ0FBQztBQU5ELDBCQU1DO0FBRVksUUFBQSxhQUFhLEdBQUcsSUFBQSxtQkFBUSxFQUNqQyxFQUFFLFdBQVcsRUFBRSxDQUFDLEVBQUUsRUFDbEIsS0FBSyxFQUFFLE1BQWlCLEVBQUUsbUJBQXFDLEVBQUUsRUFBRSxFQUFFO0lBQ2pFLE1BQU0sTUFBTSxHQUFHLFNBQUcsQ0FBQyxNQUFNLENBQUMsT0FBTztRQUM3QixDQUFDLENBQUMsRUFBRSxJQUFJLEVBQUUsU0FBRyxDQUFDLE1BQU0sRUFBRSxLQUFLLEVBQUUsU0FBRyxDQUFDLE1BQU0sRUFBRSxJQUFJLEVBQUUsU0FBRyxDQUFDLE1BQU0sRUFBRSxLQUFLLEVBQUUsU0FBRyxDQUFDLE1BQU0sRUFBRTtRQUM5RSxDQUFDLENBQUMsU0FBUyxDQUFDO0lBQ2hCLE1BQU0sTUFBTSxHQUFHLEVBQUUsV0FBVyxFQUFFLENBQUMsRUFBRSxNQUFNLEVBQUUsTUFBTSxFQUFFLENBQUM7SUFDbEQsTUFBTSxRQUFRLEdBQWdCO1FBQzFCLEdBQUcsRUFBRSxnQkFBZ0IsRUFBRSxTQUFTLEVBQUUsRUFBRSxJQUFJLElBQUksZ0JBQUcsQ0FBQyxNQUFNLENBQUM7UUFDdkQsTUFBTSxFQUFFLGdCQUFnQixFQUFFLFlBQVksRUFBRSxFQUFFLElBQUksSUFBSSxzQkFBTSxDQUFDLE1BQU0sQ0FBQztRQUNoRSwyREFBMkQ7UUFDM0QsZUFBZTtRQUNmLE9BQU8sRUFDSCxnQkFBZ0IsRUFBRSwwQkFBMEIsRUFBRSxFQUFFO1lBQ2hELElBQUksc0JBQU0sQ0FBQztnQkFDUCxHQUFHLE1BQU07Z0JBQ1QsZ0RBQWdEO2dCQUNoRCxXQUFXLEVBQUUsQ0FBQztnQkFDZCw4REFBOEQ7Z0JBQzlELFNBQVM7YUFDWixDQUFDO1FBQ04sVUFBVSxFQUNOLGdCQUFnQixFQUFFLG9CQUFvQixFQUFFLEVBQUUsSUFBSSxJQUFJLHVDQUFjLENBQUMsTUFBTSxDQUFDO1FBQzVFLEdBQUcsRUFBRSxnQkFBZ0IsQ0FBQyxTQUFTLEVBQUUsRUFBRSxJQUFJLElBQUksZ0JBQUcsQ0FBQyxNQUFNLENBQUM7UUFDdEQsR0FBRyxFQUFFLGdCQUFnQixDQUFDLFNBQVMsRUFBRSxFQUFFLElBQUksSUFBSSxnQkFBRyxDQUFDLE1BQU0sQ0FBQztRQUN0RCxPQUFPLEVBQUUsZ0JBQWdCLENBQUMsYUFBYSxFQUFFLEVBQUUsSUFBSSxJQUFJLHdCQUFPLENBQUMsTUFBTSxDQUFDO1FBQ2xFLEdBQUcsRUFBRSxnQkFBZ0IsQ0FBQyxTQUFTLEVBQUUsRUFBRSxJQUFJLElBQUksZ0JBQUcsQ0FBQyxNQUFNLENBQUM7UUFDdEQsRUFBRSxFQUFFLGdCQUFnQixDQUFDLFFBQVEsRUFBRSxFQUFFLElBQUksSUFBSSxjQUFFLENBQUMsTUFBTSxDQUFDO0tBQ3RELENBQUM7SUFDRixPQUFPLFFBQVEsQ0FBQztBQUNwQixDQUFDLENBQ0osQ0FBQztBQUVLLEtBQUssVUFBVSxhQUFhLENBQy9CLFFBQWdCLEVBQ2hCLFFBQXFCLEVBQ3JCLFVBQW1CO0lBRW5CLE1BQU0sRUFBRSxHQUFHLEVBQUUsR0FBRyxRQUFRLENBQUM7SUFDekIsU0FBRyxDQUFDLElBQUksQ0FBQyxpQ0FBaUMsQ0FBQyxDQUFDO0lBQzVDLElBQUksQ0FBQztRQUNELE1BQU0sUUFBUSxHQUFHLE1BQU0sR0FBRyxDQUFDLE9BQU8sQ0FBQyxFQUFFLFFBQVEsRUFBRSxDQUFDLENBQUM7UUFDakQsSUFBSSxDQUFDLFFBQVEsQ0FBQyxJQUFJLEVBQUUsQ0FBQztZQUNqQixNQUFNLElBQUksS0FBSyxFQUFFLENBQUM7UUFDdEIsQ0FBQztJQUNMLENBQUM7SUFBQyxPQUFPLEdBQUcsRUFBRSxDQUFDO1FBQ1gsSUFBSSxDQUFDLFVBQVUsRUFBRSxDQUFDO1lBQ2QsTUFBTSxJQUFJLGtCQUFVLENBQ2hCLEdBQUcsWUFBWSxLQUFLLENBQUMsQ0FBQyxDQUFDLEdBQUcsQ0FBQyxDQUFDLENBQUMsRUFBRSxFQUMvQix3QkFBd0IsUUFBUSxHQUFHLENBQ3RDLENBQUM7UUFDTixDQUFDO0lBQ0wsQ0FBQztJQUNELFNBQUcsQ0FBQyxJQUFJLENBQUMsMEJBQTBCLFFBQVEsaUNBQWlDLENBQUMsQ0FBQztJQUM5RSxNQUFNLHdCQUF3QixHQUFHLElBQUksQ0FBQyxTQUFTLENBQUM7UUFDNUMsT0FBTyxFQUFFLFlBQVk7UUFDckIsU0FBUyxFQUFFO1lBQ1A7Z0JBQ0ksU0FBUyxFQUFFLEVBQUUsT0FBTyxFQUFFLHNCQUFzQixFQUFFO2dCQUM5QyxNQUFNLEVBQUUsZ0JBQWdCO2dCQUN4QixNQUFNLEVBQUUsT0FBTzthQUNsQjtTQUNKO0tBQ0osQ0FBQyxDQUFDO0lBQ0gsTUFBTSxVQUFVLEdBQXNCO1FBQ2xDLHdCQUF3QjtRQUN4QixRQUFRO1FBQ1IsV0FBVyxFQUFFLDRDQUE0QztRQUN6RCxrQkFBa0IsRUFBRSxJQUFJO0tBQzNCLENBQUM7SUFDRixTQUFHLENBQUMsSUFBSSxDQUFDLG9CQUFvQixDQUFDLENBQUM7SUFDL0IsTUFBTSxTQUFTLEdBQUcsNkNBQTZDLENBQUM7SUFDaEUsSUFBSSxDQUFDO1FBQ0QsTUFBTSxZQUFZLEdBQUcsTUFBTSxHQUFHLENBQUMsVUFBVSxDQUFDLFVBQVUsQ0FBQyxDQUFDO1FBQ3RELFNBQUcsQ0FBQyxJQUFJLENBQUMscUNBQXFDLENBQUMsQ0FBQztRQUNoRCxNQUFNLEdBQUcsQ0FBQyxnQkFBZ0IsQ0FBQyxFQUFFLFFBQVEsRUFBRSxTQUFTLEVBQUUsQ0FBQyxDQUFDO1FBQ3BELE9BQU8sWUFBWSxDQUFDLElBQUssQ0FBQztJQUM5QixDQUFDO0lBQUMsT0FBTyxHQUFHLEVBQUUsQ0FBQztRQUNYLElBQUksR0FBRyxZQUFZLHlDQUE0QixFQUFFLENBQUM7WUFDOUMsTUFBTSxJQUFBLGNBQUssRUFBQyxJQUFJLENBQUMsQ0FBQztZQUNsQixNQUFNLFlBQVksR0FBRyxNQUFNLEdBQUcsQ0FBQyxPQUFPLENBQUMsRUFBRSxRQUFRLEVBQUUsQ0FBQyxDQUFDO1lBQ3JELE1BQU0sR0FBRyxDQUFDLGdCQUFnQixDQUFDLEVBQUUsUUFBUSxFQUFFLFNBQVMsRUFBRSxDQUFDLENBQUM7WUFDcEQsT0FBTyxZQUFZLENBQUMsSUFBSyxDQUFDO1FBQzlCLENBQUM7UUFDRCxNQUFNLElBQUksa0JBQVUsQ0FBQyxHQUFHLElBQUksRUFBRSxFQUFFLDBCQUEwQixRQUFRLEdBQUcsQ0FBQyxDQUFDO0lBQzNFLENBQUM7QUFDTCxDQUFDO0FBckRELHNDQXFEQztBQUVZLFFBQUEsVUFBVSxHQUFHLElBQUEsbUJBQVEsRUFDOUIsRUFBRSxXQUFXLEVBQUUsQ0FBQyxFQUFFLElBQUksRUFBRSxDQUFDLEVBQUUsT0FBTyxFQUFFLEtBQUssRUFBRSxLQUFLLEVBQUUsRUFBRSxFQUFFLEVBQ3RELGFBQWEsQ0FDaEIsQ0FBQztBQUVGLE1BQU0sZUFBZSxHQUFHLGFBQWEsQ0FBQyxxQkFBcUIsQ0FBQztBQUM1RCxNQUFNLFVBQVUsR0FBRyxFQUFFLE1BQU0sRUFBRSxHQUFHLEVBQUUsVUFBVSxFQUFFLEVBQUUsRUFBRSxJQUFJLEVBQUUsRUFBRSxFQUFFLElBQUksRUFBRSxFQUFFLEVBQUUsZUFBZSxFQUFFLENBQUM7QUFFakYsS0FBSyxVQUFVLFdBQVcsQ0FDN0IsTUFBYyxFQUNkLFdBQXdDLEVBQ3hDLG9CQUE2QixFQUM3QixZQUFvQixFQUNwQixNQUFpQixFQUNqQixlQUF1QixFQUN2QixnQkFBZ0Q7SUFFaEQsSUFBSSxDQUFDLFdBQVcsRUFBRSxDQUFDO1FBQ2YsT0FBTztJQUNYLENBQUM7SUFDRCxTQUFHLENBQUMsSUFBSSxDQUFDLHVCQUF1QixDQUFDLENBQUM7SUFFbEMsTUFBTSxtQkFBbUIsR0FDckIsT0FBTyxXQUFXLEtBQUssUUFBUTtRQUMzQixDQUFDLENBQUMsQ0FBQyxNQUFNLElBQUEsbUJBQVEsRUFBQyxXQUFXLENBQUMsQ0FBQyxDQUFDLFFBQVEsRUFBRTtRQUMxQyxDQUFDLENBQUMsSUFBSSxDQUFDLFNBQVMsQ0FBQyxXQUFXLENBQUMsQ0FBQztJQUV0QyxJQUFJLFNBQVMsQ0FBQztJQUNkLElBQUksb0JBQW9CLEVBQUUsQ0FBQztRQUN2QixNQUFNLE1BQU0sR0FBRyxJQUFBLG1CQUFVLEVBQUMsUUFBUSxDQUFDLENBQUM7UUFDcEMsTUFBTSxDQUFDLE1BQU0sQ0FBQyxtQkFBbUIsQ0FBQyxDQUFDO1FBQ25DLE1BQU0sQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLFNBQVMsQ0FBQyxnQkFBZ0IsQ0FBQyxhQUFhLElBQUksRUFBRSxDQUFDLENBQUMsQ0FBQztRQUNwRSxNQUFNLFFBQVEsR0FBRyxNQUFNLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyxDQUFDO1FBQ3RDLFNBQVMsR0FBRyxTQUFTLFFBQVEsRUFBRSxDQUFDO1FBQ2hDLE1BQU0sTUFBTSxHQUFHLE1BQU0sT0FBTyxDQUFDLE1BQU0sQ0FBQyxpQkFBaUIsQ0FBQyxFQUFFLFNBQVMsRUFBRSxDQUFDLENBQUMsQ0FBQztRQUN0RSxJQUFJLE1BQU0sRUFBRSxhQUFhLEVBQUUsTUFBTSxJQUFJLENBQUMsR0FBRyxDQUFDLEVBQUUsQ0FBQztZQUN6QyxNQUFNLENBQUMsRUFBRSxPQUFPLEVBQUUsZUFBZSxFQUFFLFdBQVcsRUFBRSxDQUFDLEdBQzdDLE1BQU0sRUFBRSxhQUFhLElBQUksRUFBRSxDQUFDO1lBQ2hDLElBQUksQ0FBQyxJQUFBLG1CQUFVLEVBQUMsV0FBVyxFQUFFLGVBQWUsQ0FBQyxJQUFJLE9BQU8sSUFBSSxlQUFlLEVBQUUsQ0FBQztnQkFDMUUsT0FBTyxFQUFFLE9BQU8sRUFBRSxlQUFlLEVBQUUsU0FBUyxFQUFFLENBQUM7WUFDbkQsQ0FBQztRQUNMLENBQUM7SUFDTCxDQUFDO1NBQU0sQ0FBQztRQUNKLFNBQVMsR0FBRyxZQUFZLENBQUM7SUFDN0IsQ0FBQztJQUVELElBQUksQ0FBQztRQUNELE1BQU0sV0FBVyxHQUFHLE1BQU0sSUFBQSxnQkFBUSxFQUFDLE1BQU0sRUFBRTtZQUN2QyxNQUFNO1lBQ04sT0FBTyxFQUFFLEdBQUc7WUFDWixVQUFVLEVBQUUsSUFBSTtZQUNoQixJQUFJLEVBQUUsT0FBTztZQUNiLEVBQUUsRUFBRSxLQUFLO1lBQ1QsVUFBVSxFQUFFLENBQUM7WUFDYixjQUFjLEVBQUU7Z0JBQ1osU0FBUyxFQUFFLEVBQUU7YUFDaEI7WUFDRCxnQkFBZ0I7U0FDbkIsQ0FBQyxDQUFDO1FBQ0gsSUFBSSxDQUFDO1lBQ0QsTUFBTSxXQUFXLEdBQTBCO2dCQUN2QyxtQkFBbUI7Z0JBQ25CLFNBQVM7Z0JBQ1QsWUFBWTtnQkFDWixNQUFNO2dCQUNOLGVBQWU7YUFDbEIsQ0FBQztZQUNGLE1BQU0sRUFBRSxVQUFVLEVBQUUsU0FBUyxFQUFFLEdBQUcsTUFBTSxXQUFXLENBQUMsU0FBUyxDQUFDLFVBQVUsQ0FDcEUsV0FBVyxDQUNkLENBQUM7WUFDRixTQUFHLENBQUMsSUFBSSxDQUFDLFVBQVUsQ0FBQyxDQUFDO1lBQ3JCLE9BQU8sU0FBUyxDQUFDO1FBQ3JCLENBQUM7Z0JBQVMsQ0FBQztZQUNQLE1BQU0sV0FBVyxDQUFDLE9BQU8sRUFBRSxDQUFDO1FBQ2hDLENBQUM7SUFDTCxDQUFDO0lBQUMsT0FBTyxHQUFRLEVBQUUsQ0FBQztRQUNoQixPQUFPLENBQUMsR0FBRyxDQUFDLEdBQUcsQ0FBQyxDQUFDO1FBQ2pCLE1BQU0sSUFBSSxrQkFBVSxDQUNoQixHQUFHLFlBQVksS0FBSyxDQUFDLENBQUMsQ0FBQyxHQUFHLENBQUMsQ0FBQyxDQUFDLEVBQUUsRUFDL0IsZ0RBQWdELENBQ25ELENBQUM7SUFDTixDQUFDO0FBQ0wsQ0FBQztBQTFFRCxrQ0EwRUM7QUFFRCxTQUFnQixNQUFNLENBQUMsS0FBZTtJQUNsQyxNQUFNLEVBQUUsTUFBTSxFQUFFLFlBQVksRUFBRSxHQUFHLEtBQUssQ0FBQyxTQUFTLENBQUM7SUFDakQsT0FBTyxJQUFBLHNCQUFTLEVBQUMsTUFBTSxFQUFFLFlBQVksQ0FBQyxDQUFDO0FBQzNDLENBQUM7QUFIRCx3QkFHQztBQUVZLFFBQUEsVUFBVSxHQUFHLElBQUEsbUJBQVEsRUFDOUIsRUFBRSxXQUFXLEVBQUUsUUFBUSxFQUFFLElBQUksRUFBRSxDQUFDLEVBQUUsRUFDbEMsS0FBSyxFQUFFLE9BQWUsRUFBRSxLQUFXLEVBQUUsT0FBNkIsRUFBRSxFQUFFO0lBQ2xFLE1BQU0sRUFBRSxNQUFNLEVBQUUsT0FBTyxFQUFFLFVBQVUsRUFBRSxHQUFHLEVBQUUsV0FBVyxFQUFFLElBQUksRUFBRSxHQUFHLE9BQU8sQ0FBQztJQUN4RSxJQUFJLFdBQVcsR0FBRyxHQUFHLElBQUksSUFBSSxLQUFLLE9BQU8sRUFBRSxDQUFDO1FBQ3hDLFNBQUcsQ0FBQyxJQUFJLENBQUMsNkRBQTZELENBQUMsQ0FBQztRQUN4RSxTQUFHLENBQUMsSUFBSSxDQUFDLHlEQUF5RCxDQUFDLENBQUM7SUFDeEUsQ0FBQztJQUNELFNBQUcsQ0FBQyxJQUFJLENBQUMsbUJBQW1CLENBQUMsQ0FBQztJQUM5QixNQUFNLFFBQVEsR0FBRyxNQUFNLElBQUEscUJBQWEsRUFBQyxNQUFNLEVBQUUsT0FBTyxDQUFDLGdCQUFnQixDQUFDLENBQUM7SUFDdkUsTUFBTSxPQUFPLEdBQUcsSUFBSSxVQUFVLEVBQUUsQ0FBQztJQUNqQyxNQUFNLEVBQUUsTUFBTSxFQUFFLEdBQUcsUUFBUSxDQUFDO0lBQzVCLE1BQU0sWUFBWSxHQUFHLFNBQVMsS0FBSyxFQUFFLENBQUM7SUFDdEMsTUFBTSxFQUFFLFdBQVcsRUFBRSxvQkFBb0IsRUFBRSxXQUFXLEVBQUUsR0FBRyxPQUFPLENBQUM7SUFFbkUsS0FBSyxVQUFVLHFCQUFxQixDQUNoQyxJQUFrQixFQUNsQixJQUFZLEVBQ1osZ0JBQXdCLEVBQ3hCLFNBQXdCO1FBRXhCLE1BQU0sRUFBRSxNQUFNLEdBQUcsRUFBRSxFQUFFLEdBQUcsSUFBSSxFQUFFLEdBQUcsT0FBTyxDQUFDLGdCQUFnQixDQUFDO1FBQzFELElBQUksU0FBUyxFQUFFLENBQUM7WUFDWixNQUFNLENBQUMsSUFBSSxDQUFDLFNBQVMsQ0FBQyxlQUFlLENBQUMsQ0FBQztRQUMzQyxDQUFDO1FBQ0QsTUFBTSxPQUFPLEdBQTBCO1lBQ25DLFlBQVk7WUFDWixJQUFJO1lBQ0osT0FBTyxFQUFFLFlBQVk7WUFDckIsT0FBTyxFQUFFLGtCQUFrQjtZQUMzQixJQUFJO1lBQ0osV0FBVyxFQUFFLDJCQUEyQjtZQUN4QyxPQUFPLEVBQUUsT0FBTztZQUNoQixVQUFVLEVBQUUsVUFBVTtZQUN0QixXQUFXLEVBQUUsRUFBRSxTQUFTLEVBQUUsR0FBRyxFQUFFO1lBQy9CLE1BQU07WUFDTixHQUFHLElBQUk7U0FDVixDQUFDO1FBQ0YsU0FBRyxDQUFDLElBQUksQ0FBQywyQkFBMkIsRUFBRSxPQUFPLENBQUMsQ0FBQztRQUMvQyxJQUFJLElBQUksQ0FBQztRQUNULElBQUksQ0FBQztZQUNELElBQUksR0FBRyxNQUFNLE1BQ