faastjs
Version:
Serverless batch computing made simple.
940 lines • 149 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.AwsImpl = exports.costSnapshot = exports.requestAwsPrices = exports.awsPrice = exports.createResponseQueueImpl = exports.awsPacker = exports.getAccountId = exports.collectGarbage = exports.clearLastGc = exports.cleanup = exports.deleteResources = exports.deleteRole = exports.initialize = exports.logUrl = exports.createLayer = exports.ensureRole = exports.ensureRoleRaw = exports.createAwsApis = exports.quietly = exports.carefully = exports.AwsMetrics = exports.defaults = exports.defaultGcWorker = void 0;
const aws_sdk_1 = require("aws-sdk");
const crypto_1 = require("crypto");
const fs_extra_1 = require("fs-extra");
const https = require("https");
const util_1 = require("util");
const webpack_merge_1 = require("webpack-merge");
const cache_1 = require("../cache");
const cost_1 = require("../cost");
const error_1 = require("../error");
const faast_1 = require("../faast");
const log_1 = require("../log");
const packer_1 = require("../packer");
const provider_1 = require("../provider");
const serialize_1 = require("../serialize");
const shared_1 = require("../shared");
const throttle_1 = require("../throttle");
const awsNpm = require("./aws-npm");
const aws_queue_1 = require("./aws-queue");
const aws_shared_1 = require("./aws-shared");
const awsTrampoline = require("./aws-trampoline");
exports.defaultGcWorker = (0, throttle_1.throttle)({ concurrency: 5, rate: 5, burst: 2 }, async (work, services) => {
switch (work.type) {
case "SetLogRetention":
if (await carefully(services.cloudwatch.putRetentionPolicy({
logGroupName: work.logGroupName,
retentionInDays: work.retentionInDays || 1
}))) {
log_1.log.gc(`Added retention policy %O`, work);
}
break;
case "DeleteResources":
await deleteResources(work.resources, services, log_1.log.gc);
break;
case "DeleteLayerVersion":
if (await carefully(services.lambda.deleteLayerVersion({
LayerName: work.LayerName,
VersionNumber: work.VersionNumber
}))) {
log_1.log.gc(`deleted layer %O`, work);
}
break;
}
});
exports.defaults = {
...provider_1.commonDefaults,
region: "us-west-2",
RoleName: "faast-cached-lambda-role",
memorySize: 1728,
awsLambdaOptions: {},
awsConfig: {},
_gcWorker: exports.defaultGcWorker
};
class AwsMetrics {
constructor() {
this.outboundBytes = 0;
this.sns64kRequests = 0;
this.sqs64kRequests = 0;
}
}
exports.AwsMetrics = AwsMetrics;
async function carefully(arg) {
try {
return await arg.promise();
}
catch (err) {
log_1.log.warn(err);
return;
}
}
exports.carefully = carefully;
async function quietly(arg) {
try {
return await arg.promise();
}
catch (err) {
return;
}
}
exports.quietly = quietly;
exports.createAwsApis = (0, throttle_1.throttle)({ concurrency: 1 }, async (region, awsConfig = {}) => {
const logger = log_1.log.awssdk.enabled ? { log: log_1.log.awssdk } : undefined;
const common = {
maxRetries: 6,
correctClockSkew: true,
logger,
...awsConfig,
region
};
const agent = new https.Agent({ keepAlive: true, maxSockets: 1000, timeout: 0 });
const services = {
iam: new aws_sdk_1.IAM({ apiVersion: "2010-05-08", ...common }),
lambda: new aws_sdk_1.Lambda({ apiVersion: "2015-03-31", ...common }),
// Special Lambda instance with configuration optimized for
// invocations.
lambda2: new aws_sdk_1.Lambda({
apiVersion: "2015-03-31",
...common,
// Retries are handled by faast.js, not the sdk.
maxRetries: 0,
// The default 120s timeout is too short, especially for https
// mode.
httpOptions: { timeout: 0, agent }
}),
cloudwatch: new aws_sdk_1.CloudWatchLogs({ apiVersion: "2014-03-28", ...common }),
sqs: new aws_sdk_1.SQS({ apiVersion: "2012-11-05", ...common }),
sns: new aws_sdk_1.SNS({ apiVersion: "2010-03-31", ...common }),
pricing: new aws_sdk_1.Pricing({ region: "us-east-1", ...common }),
sts: new aws_sdk_1.STS({ apiVersion: "2011-06-15", ...common }),
s3: new aws_sdk_1.S3({ apiVersion: "2006-03-01", ...common })
};
return services;
});
async function ensureRoleRaw(RoleName, services, createRole) {
const { iam } = services;
log_1.log.info(`Checking for cached lambda role`);
try {
const response = await iam.getRole({ RoleName }).promise();
return response.Role;
}
catch (err) {
if (!createRole) {
throw new error_1.FaastError(err, `could not find role "${RoleName}"`);
}
}
log_1.log.info(`Creating default role "${RoleName}" for faast trampoline function`);
const AssumeRolePolicyDocument = JSON.stringify({
Version: "2012-10-17",
Statement: [
{
Principal: { Service: "lambda.amazonaws.com" },
Action: "sts:AssumeRole",
Effect: "Allow"
}
]
});
const roleParams = {
AssumeRolePolicyDocument,
RoleName,
Description: "role for lambda functions created by faast",
MaxSessionDuration: 3600
};
log_1.log.info(`Calling createRole`);
const PolicyArn = "arn:aws:iam::aws:policy/AdministratorAccess";
try {
const roleResponse = await iam.createRole(roleParams).promise();
log_1.log.info(`Attaching administrator role policy`);
await iam.attachRolePolicy({ RoleName, PolicyArn }).promise();
return roleResponse.Role;
}
catch (err) {
if (err.code === "EntityAlreadyExists") {
await (0, shared_1.sleep)(5000);
const roleResponse = await iam.getRole({ RoleName }).promise();
await iam.attachRolePolicy({ RoleName, PolicyArn }).promise();
return roleResponse.Role;
}
throw new error_1.FaastError(err, `failed to create role "${RoleName}"`);
}
}
exports.ensureRoleRaw = ensureRoleRaw;
exports.ensureRole = (0, throttle_1.throttle)({ concurrency: 1, rate: 2, memoize: true, retry: 12 }, ensureRoleRaw);
const ResponseQueueId = awsTrampoline.INVOCATION_TEST_QUEUE;
const emptyFcall = { callId: "0", modulePath: "", name: "", args: "", ResponseQueueId };
async function createLayer(lambda, packageJson, useDependencyCaching, FunctionName, region, retentionInDays, awsLambdaOptions) {
if (!packageJson) {
return;
}
log_1.log.info(`Building node_modules`);
const packageJsonContents = typeof packageJson === "string"
? (await (0, fs_extra_1.readFile)(packageJson)).toString()
: JSON.stringify(packageJson);
let LayerName;
if (useDependencyCaching) {
const hasher = (0, crypto_1.createHash)("sha256");
hasher.update(packageJsonContents);
hasher.update(JSON.stringify(awsLambdaOptions.Architectures ?? ""));
const cacheKey = hasher.digest("hex");
LayerName = `faast-${cacheKey}`;
const layers = await quietly(lambda.listLayerVersions({ LayerName }));
if (layers?.LayerVersions?.length ?? 0 > 0) {
const [{ Version, LayerVersionArn, CreatedDate }] = layers?.LayerVersions ?? [];
if (!(0, shared_1.hasExpired)(CreatedDate, retentionInDays) && Version && LayerVersionArn) {
return { Version, LayerVersionArn, LayerName };
}
}
}
else {
LayerName = FunctionName;
}
try {
const faastModule = await (0, faast_1.faastAws)(awsNpm, {
region,
timeout: 300,
memorySize: 2048,
mode: "https",
gc: "off",
maxRetries: 0,
webpackOptions: {
externals: []
},
awsLambdaOptions
});
try {
const installArgs = {
packageJsonContents,
LayerName,
FunctionName,
region,
retentionInDays
};
const { installLog, layerInfo } = await faastModule.functions.npmInstall(installArgs);
log_1.log.info(installLog);
return layerInfo;
}
finally {
await faastModule.cleanup();
}
}
catch (err) {
throw new error_1.FaastError(err, "failed to create lambda layer from packageJson");
}
}
exports.createLayer = createLayer;
function logUrl(state) {
const { region, FunctionName } = state.resources;
return (0, aws_shared_1.getLogUrl)(region, FunctionName);
}
exports.logUrl = logUrl;
exports.initialize = (0, throttle_1.throttle)({ concurrency: Infinity, rate: 2 }, async (fModule, nonce, options) => {
const { region, timeout, memorySize, env, concurrency, mode } = options;
if (concurrency > 100 && mode !== "queue") {
log_1.log.warn(`Consider using queue mode for higher levels of concurrency:`);
log_1.log.warn(`https://faastjs.org/docs/api/faastjs.commonoptions.mode`);
}
log_1.log.info(`Creating AWS APIs`);
const services = await (0, exports.createAwsApis)(region, options.awsConfig);
const { lambda } = services;
const FunctionName = `faast-${nonce}`;
const { packageJson, useDependencyCaching, description } = options;
async function createFunctionRequest(Code, Role, responseQueueArn, layerInfo) {
const { Layers = [], ...rest } = options.awsLambdaOptions;
if (layerInfo) {
Layers.push(layerInfo.LayerVersionArn);
}
const request = {
FunctionName,
Role,
Runtime: "nodejs14.x",
Handler: "index.trampoline",
Code,
Description: "faast trampoline function",
Timeout: timeout,
MemorySize: memorySize,
Environment: { Variables: env },
Layers,
...rest
};
log_1.log.info(`createFunctionRequest: %O`, request);
let func;
try {
func = await lambda.createFunction(request).promise();
await (0, throttle_1.retryOp)(2, () => lambda.waitFor("functionActive", { FunctionName }).promise());
}
catch (err) {
if (err?.message?.match(/Function already exist/)) {
func = (await lambda.getFunction({ FunctionName }).promise())
.Configuration;
}
else {
throw new error_1.FaastError(err, "Create function failure");
}
}
log_1.log.info(`Created function ${func.FunctionName}, FunctionArn: ${func.FunctionArn} [${description}]`);
log_1.log.minimal(`Created function ${func.FunctionName} [${description}]`);
try {
const config = await (0, throttle_1.retryOp)((err, n) => n < 5 && err?.message?.match(/destination ARN.*is invalid/), () => lambda
.putFunctionEventInvokeConfig({
FunctionName,
MaximumRetryAttempts: 0,
MaximumEventAgeInSeconds: 21600,
DestinationConfig: {
OnFailure: { Destination: responseQueueArn }
}
})
.promise());
log_1.log.info(`Function event invocation config: %O`, config);
}
catch (err) {
throw new error_1.FaastError(err, "putFunctionEventInvokeConfig failure");
}
return func;
}
const { wrapperVerbose } = options.debugOptions;
async function createCodeBundle() {
const { timeout, childProcess, mode } = options;
const hasLambdaTimeoutBug = mode !== "queue" && timeout >= 180;
const childProcessTimeoutMs = hasLambdaTimeoutBug && childProcess ? (timeout - 5) * 1000 : 0;
const childProcessMemoryLimitMb = options.childProcessMemoryMb;
const wrapperOptions = {
wrapperVerbose,
childProcessTimeoutMs,
childProcessMemoryLimitMb
};
const bundle = awsPacker(fModule, options, wrapperOptions, FunctionName);
return { ZipFile: await (0, shared_1.streamToBuffer)((await bundle).archive) };
}
const { RoleName } = options;
const state = {
resources: {
FunctionName,
RoleName,
region,
logGroupName: (0, aws_shared_1.getLogGroupName)(FunctionName)
},
services,
metrics: new AwsMetrics(),
options
};
const { gc, retentionInDays, _gcWorker: gcWorker } = options;
if (gc === "auto" || gc === "force") {
log_1.log.gc(`Starting garbage collector`);
state.gcPromise = collectGarbage(gcWorker, services, region, retentionInDays, gc).catch(err => {
log_1.log.gc(`Garbage collection error: ${err}`);
return "skipped";
});
}
try {
log_1.log.info(`Creating lambda function`);
const rolePromise = (0, exports.ensureRole)(RoleName, services, RoleName === exports.defaults.RoleName);
const responseQueuePromise = createResponseQueueImpl(state, FunctionName);
const pricingPromise = (0, exports.requestAwsPrices)(services.pricing, region);
const codeBundlePromise = createCodeBundle();
// Ensure role exists before creating lambda layer, which also needs the role.
const role = await rolePromise;
const layerPromise = createLayer(services.lambda, packageJson, useDependencyCaching, FunctionName, region, retentionInDays, options.awsLambdaOptions);
const codeBundle = await codeBundlePromise;
const responseQueueArn = await responseQueuePromise;
const layer = await layerPromise;
if (layer) {
state.resources.layer = layer;
}
let lambdaFnArn;
const retryable = [
/role/,
/KMS Exception/,
/internal service error/,
/Layer version/
];
const shouldRetry = (err, n) => n < 5 && !!retryable.find(regex => err?.message?.match(regex));
await (0, throttle_1.retryOp)(shouldRetry, async () => {
try {
const lambdaFn = await createFunctionRequest(codeBundle, role.Arn, responseQueueArn, layer);
lambdaFnArn = lambdaFn.FunctionArn;
// If the role for the lambda function was created
// recently, test that the role works by invoking the
// function. If an exception occurs, the function is
// deleted and re-deployed. Empirically, this is the way
// to ensure successful lambda creation when an IAM role
// is recently created.
if (Date.now() - role.CreateDate.getTime() < 300 * 1000) {
const { metrics } = state;
const fn = FunctionName;
const never = new Promise(_ => { });
await (0, throttle_1.retryOp)(1, () => invokeHttps(lambda, fn, emptyFcall, metrics, never));
}
}
catch (err) {
/* istanbul ignore next */ {
await lambda
.deleteFunction({ FunctionName })
.promise()
.catch(_ => { });
throw new error_1.FaastError(err, `New lambda function ${FunctionName} failed invocation test`);
}
}
});
const { mode } = options;
if (mode === "queue") {
await createRequestQueueImpl(state, FunctionName, lambdaFnArn);
}
await pricingPromise;
log_1.log.info(`Lambda function initialization complete.`);
return state;
}
catch (err) {
try {
await cleanup(state, {
deleteResources: true,
deleteCaches: false,
gcTimeout: 30
});
}
catch { }
throw new error_1.FaastError({ cause: err, name: error_1.FaastErrorNames.ECREATE }, "failed to initialize cloud function");
}
});
async function invoke(state, call, cancel) {
const { metrics, services, resources, options } = state;
switch (options.mode) {
case "auto":
case "https":
const { lambda2 } = services;
const { FunctionName } = resources;
await invokeHttps(lambda2, FunctionName, call, metrics, cancel);
return;
case "queue":
const { sns } = services;
const { RequestTopicArn } = resources;
try {
await (0, aws_queue_1.publishFunctionCallMessage)(sns, RequestTopicArn, call, metrics);
}
catch (err) {
throw new error_1.FaastError(err, `invoke sns error ${(0, util_1.inspect)(call, undefined, 9)}`);
}
return;
}
}
function poll(state, cancel) {
return (0, aws_queue_1.receiveMessages)(state.services.sqs, state.resources.ResponseQueueUrl, state.metrics, cancel);
}
function responseQueueId(state) {
return state.resources.ResponseQueueUrl;
}
async function invokeHttps(lambda, FunctionName, message, metrics, cancel) {
const request = {
FunctionName,
Payload: (0, serialize_1.serialize)(message),
LogType: "None"
};
const awsRequest = lambda.invoke(request);
const rawResponse = await Promise.race([awsRequest.promise(), cancel]);
if (!rawResponse) {
log_1.log.info(`cancelling lambda invoke`);
awsRequest.abort();
return;
}
metrics.outboundBytes += (0, shared_1.computeHttpResponseBytes)(rawResponse.$response.httpResponse.headers);
if (rawResponse.LogResult) {
log_1.log.info(Buffer.from(rawResponse.LogResult, "base64").toString());
}
if (rawResponse.FunctionError) {
const error = (0, aws_queue_1.processAwsErrorMessage)(rawResponse.Payload);
throw error;
}
}
async function deleteRole(RoleName, iam) {
const policies = await carefully(iam.listAttachedRolePolicies({ RoleName }));
const AttachedPolicies = policies?.AttachedPolicies ?? [];
await Promise.all(AttachedPolicies.map(p => p.PolicyArn).map(PolicyArn => carefully(iam.detachRolePolicy({ RoleName, PolicyArn }))));
const rolePolicyListResponse = await carefully(iam.listRolePolicies({ RoleName }));
const RolePolicies = rolePolicyListResponse?.PolicyNames ?? [];
await Promise.all(RolePolicies.map(PolicyName => carefully(iam.deleteRolePolicy({ RoleName, PolicyName }))));
await carefully(iam.deleteRole({ RoleName }));
}
exports.deleteRole = deleteRole;
async function deleteResources(resources, services, output = log_1.log.info) {
const { FunctionName, RoleName, region, RequestTopicArn, ResponseQueueUrl, ResponseQueueArn, SNSLambdaSubscriptionArn, logGroupName, layer, Bucket, ...rest } = resources;
const _exhaustiveCheck = {};
const { lambda, sqs, sns, iam, s3, cloudwatch } = services;
if (SNSLambdaSubscriptionArn) {
if (await quietly(sns.unsubscribe({ SubscriptionArn: SNSLambdaSubscriptionArn }))) {
output(`Deleted request queue subscription to lambda`);
}
}
if (RoleName) {
await deleteRole(RoleName, iam);
}
if (RequestTopicArn) {
if (await quietly(sns.deleteTopic({ TopicArn: RequestTopicArn }))) {
output(`Deleted request queue topic: ${RequestTopicArn}`);
}
}
if (ResponseQueueUrl) {
if (await quietly(sqs.deleteQueue({ QueueUrl: ResponseQueueUrl }))) {
output(`Deleted response queue: ${ResponseQueueUrl}`);
}
}
if (layer) {
if (await quietly(lambda.deleteLayerVersion({
LayerName: layer.LayerName,
VersionNumber: layer.Version
}))) {
output(`Deleted lambda layer: ${layer.LayerName}:${layer.Version}`);
}
}
if (Bucket) {
const objects = await quietly(s3.listObjectsV2({ Bucket, Prefix: "faast-" }));
if (objects) {
const keys = (objects.Contents || []).map(elem => ({ Key: elem.Key }));
if (await quietly(s3.deleteObjects({ Bucket, Delete: { Objects: keys } }))) {
output(`Deleted s3 objects: ${keys.length} objects in bucket ${Bucket}`);
}
}
if (await quietly(s3.deleteBucket({ Bucket }))) {
output(`Deleted s3 bucket: ${Bucket}`);
}
}
if (FunctionName) {
if (await quietly(lambda.deleteFunction({ FunctionName }))) {
output(`Deleted function: ${FunctionName}`);
}
}
if (logGroupName) {
if (await quietly(cloudwatch.deleteLogGroup({ logGroupName }))) {
output(`Deleted log group: ${logGroupName}`);
}
}
}
exports.deleteResources = deleteResources;
async function addLogRetentionPolicy(FunctionName, cloudwatch) {
const logGroupName = (0, aws_shared_1.getLogGroupName)(FunctionName);
const response = await quietly(cloudwatch.putRetentionPolicy({ logGroupName, retentionInDays: 1 }));
if (response !== undefined) {
log_1.log.info(`Added 1 day retention policy to log group ${logGroupName}`);
}
}
async function cleanup(state, options) {
log_1.log.info(`aws cleanup starting.`);
if (state.gcPromise) {
log_1.log.info(`Waiting for garbage collection...`);
await state.gcPromise;
log_1.log.info(`Garbage collection done.`);
}
if (options.deleteResources) {
log_1.log.info(`Cleaning up infrastructure for ${state.resources.FunctionName}...`);
await addLogRetentionPolicy(state.resources.FunctionName, state.services.cloudwatch);
// Don't delete cached role. It may be in use by other instances of
// faast. Don't delete logs. They are often useful. By default log
// stream retention will be 1 day, and gc will clean out the log group
// after the streams are expired. Don't delete a lambda layer that is
// used to cache packages.
const { logGroupName, RoleName, layer, ...rest } = state.resources;
await deleteResources(rest, state.services);
if (!state.options.useDependencyCaching || options.deleteCaches) {
await deleteResources({ layer }, state.services);
}
}
log_1.log.info(`aws cleanup done.`);
}
exports.cleanup = cleanup;
const logGroupNameRegexp = new RegExp(`^/aws/lambda/(faast-${shared_1.uuidv4Pattern})$`);
function functionNameFromLogGroup(logGroupName) {
const match = logGroupName.match(logGroupNameRegexp);
return match && match[1];
}
let lastGc;
function clearLastGc() {
lastGc = undefined;
}
exports.clearLastGc = clearLastGc;
function forEachPage(description, request, process) {
const throttlePaging = (0, throttle_1.throttle)({ concurrency: 1, rate: 1 }, async () => { });
return new Promise((resolve, reject) => {
request.eachPage((err, page, done) => {
if (err) {
log_1.log.warn(`GC: Error when listing ${description}: ${err}`);
reject(err);
return false;
}
if (page === null) {
resolve();
}
else {
process(page).then(() => throttlePaging().then(done));
}
return true;
});
});
}
async function collectGarbage(executor, services, region, retentionInDays, mode) {
if (executor === exports.defaultGcWorker) {
if (mode === "auto") {
if (lastGc && Date.now() <= lastGc + 3600 * 1000) {
return "skipped";
}
const gcEntry = await cache_1.caches.awsGc.get("gc");
if (gcEntry) {
try {
const lastGcPersistent = JSON.parse(gcEntry.toString());
if (lastGcPersistent &&
typeof lastGcPersistent === "number" &&
Date.now() <= lastGcPersistent + 3600 * 1000) {
lastGc = lastGcPersistent;
return "skipped";
}
}
catch (err) {
log_1.log.warn(err);
}
}
}
lastGc = Date.now();
cache_1.caches.awsGc.set("gc", lastGc.toString());
}
const promises = [];
function scheduleWork(work) {
if (executor === exports.defaultGcWorker) {
log_1.log.gc(`Scheduling work pushing promise: %O`, work);
}
promises.push(executor(work, services));
}
const functionsWithLogGroups = new Set();
const logGroupRequest = services.cloudwatch.describeLogGroups({
logGroupNamePrefix: "/aws/lambda/faast-"
});
const accountId = await getAccountId(services.sts);
await forEachPage("log groups", logGroupRequest, async ({ logGroups = [] }) => {
logGroups.forEach(g => {
const FunctionName = functionNameFromLogGroup(g.logGroupName);
functionsWithLogGroups.add(FunctionName);
});
log_1.log.gc(`Log groups size: ${logGroups.length}`);
garbageCollectLogGroups(logGroups, retentionInDays, region, accountId, scheduleWork);
});
const listFunctionsRequest = services.lambda.listFunctions();
await forEachPage("lambda functions", listFunctionsRequest, async ({ Functions = [] }) => {
const fnPattern = new RegExp(`^faast-${shared_1.uuidv4Pattern}$`);
const funcs = (Functions || [])
.filter(fn => fn.FunctionName.match(fnPattern))
.filter(fn => !functionsWithLogGroups.has(fn.FunctionName))
.filter(fn => (0, shared_1.hasExpired)(fn.LastModified, retentionInDays))
.map(fn => fn.FunctionName);
deleteGarbageFunctions(region, accountId, funcs, scheduleWork);
});
// Collect Lambda Layers
const layersRequest = services.lambda.listLayers({
CompatibleRuntime: "nodejs"
});
await forEachPage("Lambda Layers", layersRequest, async ({ Layers = [] }) => {
for (const layer of Layers) {
if (layer.LayerName.match(/^faast-/)) {
const layerVersionRequest = services.lambda.listLayerVersions({
LayerName: layer.LayerName,
CompatibleRuntime: "nodejs"
});
await forEachPage("Lambda Layer Versions", layerVersionRequest, async ({ LayerVersions = [] }) => {
LayerVersions.forEach(layerVersion => {
if ((0, shared_1.hasExpired)(layerVersion.CreatedDate, retentionInDays)) {
scheduleWork({
type: "DeleteLayerVersion",
LayerName: layer.LayerName,
VersionNumber: layerVersion.Version
});
}
});
});
}
}
});
log_1.log.gc(`Awaiting ${promises.length} scheduled work promises`);
await Promise.all(promises);
return "done";
}
exports.collectGarbage = collectGarbage;
async function getAccountId(sts) {
const response = await sts.getCallerIdentity().promise();
const { Account, Arn, UserId } = response;
log_1.log.info(`Account ID: %O`, { Account, Arn, UserId });
return response.Account;
}
exports.getAccountId = getAccountId;
function garbageCollectLogGroups(logGroups, retentionInDays, region, accountId, scheduleWork) {
const logGroupsMissingRetentionPolicy = logGroups.filter(g => g.retentionInDays === undefined);
log_1.log.gc(`Log groups missing retention: ${logGroupsMissingRetentionPolicy.length}`);
logGroupsMissingRetentionPolicy.forEach(g => {
scheduleWork({
type: "SetLogRetention",
logGroupName: g.logGroupName,
retentionInDays
});
});
const garbageFunctions = logGroups
.filter(g => (0, shared_1.hasExpired)(g.creationTime, retentionInDays))
.map(g => functionNameFromLogGroup(g.logGroupName))
.filter(shared_1.defined);
deleteGarbageFunctions(region, accountId, garbageFunctions, scheduleWork);
}
function deleteGarbageFunctions(region, accountId, garbageFunctions, scheduleWork) {
garbageFunctions.forEach(FunctionName => {
const resources = {
FunctionName,
region,
RoleName: "",
RequestTopicArn: getSNSTopicArn(region, accountId, FunctionName),
ResponseQueueUrl: getResponseQueueUrl(region, accountId, FunctionName),
logGroupName: (0, aws_shared_1.getLogGroupName)(FunctionName),
Bucket: FunctionName
};
scheduleWork({ type: "DeleteResources", resources });
});
}
async function awsPacker(functionModule, options, wrapperOptions, FunctionName) {
return (0, packer_1.packer)(awsTrampoline, functionModule, {
...options,
webpackOptions: (0, webpack_merge_1.default)(options.webpackOptions ?? {}, {
externals: [new RegExp("^aws-sdk/?")]
})
}, wrapperOptions, FunctionName);
}
exports.awsPacker = awsPacker;
function getSNSTopicName(FunctionName) {
return `${FunctionName}-Requests`;
}
function getSNSTopicArn(region, accountId, FunctionName) {
const TopicName = getSNSTopicName(FunctionName);
return `arn:aws:sns:${region}:${accountId}:${TopicName}`;
}
function getSQSName(FunctionName) {
return `${FunctionName}-Responses`;
}
function getResponseQueueUrl(region, accountId, FunctionName) {
const queueName = getSQSName(FunctionName);
return `https://sqs.${region}.amazonaws.com/${accountId}/${queueName}`;
}
function createRequestQueueImpl(state, FunctionName, FunctionArn) {
const { sns, lambda } = state.services;
const { resources } = state;
log_1.log.info(`Creating SNS request topic`);
const createTopicPromise = (0, aws_queue_1.createSNSTopic)(sns, getSNSTopicName(FunctionName));
const assignRequestTopicArnPromise = createTopicPromise.then(topic => (resources.RequestTopicArn = topic));
const addPermissionsPromise = createTopicPromise.then(topic => {
log_1.log.info(`Adding SNS invoke permissions to function`);
return addSnsInvokePermissionsToFunction(FunctionName, topic, lambda);
});
const subscribePromise = createTopicPromise.then(topic => {
log_1.log.info(`Subscribing SNS to invoke lambda function`);
return sns
.subscribe({
TopicArn: topic,
Protocol: "lambda",
Endpoint: FunctionArn
})
.promise();
});
const assignSNSResponsePromise = subscribePromise.then(snsResponse => (resources.SNSLambdaSubscriptionArn = snsResponse.SubscriptionArn));
return Promise.all([
createTopicPromise,
assignRequestTopicArnPromise,
addPermissionsPromise,
subscribePromise,
assignSNSResponsePromise
]);
}
async function createResponseQueueImpl(state, FunctionName) {
const { sqs } = state.services;
const { resources } = state;
log_1.log.info(`Creating SQS response queue`);
const { QueueUrl, QueueArn } = await (0, aws_queue_1.createSQSQueue)(getSQSName(FunctionName), 60, sqs);
resources.ResponseQueueUrl = QueueUrl;
resources.ResponseQueueArn = QueueArn;
log_1.log.info(`Created response queue`);
return QueueArn;
}
exports.createResponseQueueImpl = createResponseQueueImpl;
function addSnsInvokePermissionsToFunction(FunctionName, RequestTopicArn, lambda) {
return lambda
.addPermission({
FunctionName,
Action: "lambda:InvokeFunction",
Principal: "sns.amazonaws.com",
StatementId: `${FunctionName}-Invoke`,
SourceArn: RequestTopicArn
})
.promise()
.catch(err => {
if (err.match(/already exists/)) {
}
else {
throw err;
}
});
}
const locations = {
"us-east-1": "US East (N. Virginia)",
"us-east-2": "US East (Ohio)",
"us-west-1": "US West (N. California)",
"us-west-2": "US West (Oregon)",
"ca-central-1": "Canada (Central)",
"eu-central-1": "EU (Frankfurt)",
"eu-west-1": "EU (Ireland)",
"eu-west-2": "EU (London)",
"eu-west-3": "EU (Paris)",
"ap-northeast-1": "Asia Pacific (Tokyo)",
"ap-northeast-2": "Asia Pacific (Seoul)",
"ap-northeast-3": "Asia Pacific (Osaka-Local)",
"ap-southeast-1": "Asia Pacific (Singapore)",
"ap-southeast-2": "Asia Pacific (Sydney)",
"ap-south-1": "Asia Pacific (Mumbai)",
"sa-east-1": "South America (São Paulo)"
};
exports.awsPrice = (0, throttle_1.throttle)({ concurrency: 6, rate: 5, memoize: true, cache: cache_1.caches.awsPrices }, async (pricing, ServiceCode, filter) => {
try {
function first(obj) {
return obj[Object.keys(obj)[0]];
}
function extractPrice(obj) {
const prices = Object.keys(obj.priceDimensions).map(key => Number(obj.priceDimensions[key].pricePerUnit.USD));
return Math.max(...prices);
}
const priceResult = await pricing
.getProducts({
ServiceCode,
Filters: Object.keys(filter).map(key => ({
Field: key,
Type: "TERM_MATCH",
Value: filter[key]
}))
})
.promise();
if (priceResult.PriceList.length > 1) {
log_1.log.warn(`Price query returned more than one product '${ServiceCode}' ($O)`, filter);
priceResult.PriceList.forEach(p => log_1.log.warn(`%O`, p));
}
const pList = priceResult.PriceList[0];
const price = extractPrice(first(pList.terms.OnDemand));
return price;
}
catch (err) {
/* istanbul ignore next */
{
const { message: m } = err;
if (!m.match(/Rate exceeded/) &&
!m.match(/EPROTO/) &&
!m.match(/socket hang up/)) {
log_1.log.warn(`Could not get AWS pricing for '${ServiceCode}' (%O)`, filter);
log_1.log.warn(err);
}
throw new error_1.FaastError(err, `failed to get AWS pricing for "${ServiceCode}"`);
}
}
});
const requestAwsPrices = async (pricing, region) => {
const location = locations[region];
/* istanbul ignore next */
return {
lambdaPerRequest: await (0, exports.awsPrice)(pricing, "AWSLambda", {
location,
group: "AWS-Lambda-Requests"
}).catch(_ => 0.0000002),
lambdaPerGbSecond: await (0, exports.awsPrice)(pricing, "AWSLambda", {
location,
group: "AWS-Lambda-Duration"
}).catch(_ => 0.00001667),
snsPer64kPublish: await (0, exports.awsPrice)(pricing, "AmazonSNS", {
location,
group: "SNS-Requests-Tier1"
}).catch(_ => 0.5 / 1e6),
sqsPer64kRequest: await (0, exports.awsPrice)(pricing, "AWSQueueService", {
location,
group: "SQS-APIRequest-Tier1",
queueType: "Standard"
}).catch(_ => 0.4 / 1e6),
dataOutPerGb: await (0, exports.awsPrice)(pricing, "AWSDataTransfer", {
fromLocation: location,
transferType: "AWS Outbound"
}).catch(_ => 0.09),
logsIngestedPerGb: await (0, exports.awsPrice)(pricing, "AmazonCloudWatch", {
location,
group: "Ingested Logs",
groupDescription: "Existing system, application, and custom log files"
}).catch(_ => 0.5)
};
};
exports.requestAwsPrices = requestAwsPrices;
async function costSnapshot(state, stats) {
const { region } = state.resources;
const prices = await (0, exports.requestAwsPrices)(state.services.pricing, region);
const costMetrics = [];
const { memorySize = exports.defaults.memorySize } = state.options;
const billedTimeStats = stats.estimatedBilledTime;
const seconds = (billedTimeStats.mean / 1000) * billedTimeStats.samples || 0;
const provisionedGb = memorySize / 1024;
const functionCallDuration = new cost_1.CostMetric({
name: "functionCallDuration",
pricing: prices.lambdaPerGbSecond * provisionedGb,
unit: "second",
measured: seconds,
comment: `https://aws.amazon.com/lambda/pricing (rate = ${prices.lambdaPerGbSecond.toFixed(8)}/(GB*second) * ${provisionedGb} GB = ${(prices.lambdaPerGbSecond * provisionedGb).toFixed(8)}/second)`
});
costMetrics.push(functionCallDuration);
const functionCallRequests = new cost_1.CostMetric({
name: "functionCallRequests",
pricing: prices.lambdaPerRequest,
measured: stats.invocations,
unit: "request",
comment: "https://aws.amazon.com/lambda/pricing"
});
costMetrics.push(functionCallRequests);
const { metrics } = state;
const outboundDataTransfer = new cost_1.CostMetric({
name: "outboundDataTransfer",
pricing: prices.dataOutPerGb,
measured: metrics.outboundBytes / 2 ** 30,
unit: "GB",
comment: "https://aws.amazon.com/ec2/pricing/on-demand/#Data_Transfer"
});
costMetrics.push(outboundDataTransfer);
const sqs = new cost_1.CostMetric({
name: "sqs",
pricing: prices.sqsPer64kRequest,
measured: metrics.sqs64kRequests,
unit: "request",
comment: "https://aws.amazon.com/sqs/pricing"
});
costMetrics.push(sqs);
const sns = new cost_1.CostMetric({
name: "sns",
pricing: prices.snsPer64kPublish,
measured: metrics.sns64kRequests,
unit: "request",
comment: "https://aws.amazon.com/sns/pricing"
});
costMetrics.push(sns);
const logIngestion = new cost_1.CostMetric({
name: "logIngestion",
pricing: prices.logsIngestedPerGb,
measured: 0,
unit: "GB",
comment: "https://aws.amazon.com/cloudwatch/pricing/ - Log ingestion costs not currently included.",
informationalOnly: true
});
costMetrics.push(logIngestion);
return new cost_1.CostSnapshot("aws", state.options, stats, costMetrics);
}
exports.costSnapshot = costSnapshot;
exports.AwsImpl = {
name: "aws",
initialize: exports.initialize,
defaults: exports.defaults,
cleanup,
costSnapshot,
logUrl,
invoke,
poll,
responseQueueId
};
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiYXdzLWZhYXN0LmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vc3JjL2F3cy9hd3MtZmFhc3QudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7O0FBQUEscUNBV2lCO0FBRWpCLG1DQUFvQztBQUNwQyx1Q0FBb0M7QUFDcEMsK0JBQStCO0FBQy9CLCtCQUErQjtBQUMvQixpREFBa0M7QUFDbEMsb0NBQWtDO0FBQ2xDLGtDQUFtRDtBQUNuRCxvQ0FBdUQ7QUFDdkQsb0NBQW9DO0FBQ3BDLGdDQUE2QjtBQUM3QixzQ0FBaUQ7QUFDakQsMENBUXFCO0FBQ3JCLDRDQUF5QztBQUN6QyxzQ0FPbUI7QUFDbkIsMENBQWdEO0FBRWhELG9DQUFvQztBQUVwQywyQ0FNcUI7QUFDckIsNkNBQTBEO0FBQzFELGtEQUFrRDtBQUVyQyxRQUFBLGVBQWUsR0FBRyxJQUFBLG1CQUFRLEVBQ25DLEVBQUUsV0FBVyxFQUFFLENBQUMsRUFBRSxJQUFJLEVBQUUsQ0FBQyxFQUFFLEtBQUssRUFBRSxDQUFDLEVBQUUsRUFDckMsS0FBSyxFQUFFLElBQWUsRUFBRSxRQUFxQixFQUFFLEVBQUU7SUFDN0MsUUFBUSxJQUFJLENBQUMsSUFBSSxFQUFFO1FBQ2YsS0FBSyxpQkFBaUI7WUFDbEIsSUFDSSxNQUFNLFNBQVMsQ0FDWCxRQUFRLENBQUMsVUFBVSxDQUFDLGtCQUFrQixDQUFDO2dCQUNuQyxZQUFZLEVBQUUsSUFBSSxDQUFDLFlBQVk7Z0JBQy9CLGVBQWUsRUFBRSxJQUFJLENBQUMsZUFBZSxJQUFJLENBQUM7YUFDN0MsQ0FBQyxDQUNMLEVBQ0g7Z0JBQ0UsU0FBRyxDQUFDLEVBQUUsQ0FBQywyQkFBMkIsRUFBRSxJQUFJLENBQUMsQ0FBQzthQUM3QztZQUNELE1BQU07UUFDVixLQUFLLGlCQUFpQjtZQUNsQixNQUFNLGVBQWUsQ0FBQyxJQUFJLENBQUMsU0FBUyxFQUFFLFFBQVEsRUFBRSxTQUFHLENBQUMsRUFBRSxDQUFDLENBQUM7WUFDeEQsTUFBTTtRQUNWLEtBQUssb0JBQW9CO1lBQ3JCLElBQ0ksTUFBTSxTQUFTLENBQ1gsUUFBUSxDQUFDLE1BQU0sQ0FBQyxrQkFBa0IsQ0FBQztnQkFDL0IsU0FBUyxFQUFFLElBQUksQ0FBQyxTQUFTO2dCQUN6QixhQUFhLEVBQUUsSUFBSSxDQUFDLGFBQWE7YUFDcEMsQ0FBQyxDQUNMLEVBQ0g7Z0JBQ0UsU0FBRyxDQUFDLEVBQUUsQ0FBQyxrQkFBa0IsRUFBRSxJQUFJLENBQUMsQ0FBQzthQUNwQztZQUNELE1BQU07S0FDYjtBQUNMLENBQUMsQ0FDSixDQUFDO0FBcUhTLFFBQUEsUUFBUSxHQUF5QjtJQUN4QyxHQUFHLHlCQUFjO0lBQ2pCLE1BQU0sRUFBRSxXQUFXO0lBQ25CLFFBQVEsRUFBRSwwQkFBMEI7SUFDcEMsVUFBVSxFQUFFLElBQUk7SUFDaEIsZ0JBQWdCLEVBQUUsRUFBRTtJQUNwQixTQUFTLEVBQUUsRUFBRTtJQUNiLFNBQVMsRUFBRSx1QkFBZTtDQUM3QixDQUFDO0FBV0YsTUFBYSxVQUFVO0lBQXZCO1FBQ0ksa0JBQWEsR0FBRyxDQUFDLENBQUM7UUFDbEIsbUJBQWMsR0FBRyxDQUFDLENBQUM7UUFDbkIsbUJBQWMsR0FBRyxDQUFDLENBQUM7SUFDdkIsQ0FBQztDQUFBO0FBSkQsZ0NBSUM7QUEyRE0sS0FBSyxVQUFVLFNBQVMsQ0FBSSxHQUF5QjtJQUN4RCxJQUFJO1FBQ0EsT0FBTyxNQUFNLEdBQUcsQ0FBQyxPQUFPLEVBQUUsQ0FBQztLQUM5QjtJQUFDLE9BQU8sR0FBUSxFQUFFO1FBQ2YsU0FBRyxDQUFDLElBQUksQ0FBQyxHQUFHLENBQUMsQ0FBQztRQUNkLE9BQU87S0FDVjtBQUNMLENBQUM7QUFQRCw4QkFPQztBQUVNLEtBQUssVUFBVSxPQUFPLENBQUksR0FBeUI7SUFDdEQsSUFBSTtRQUNBLE9BQU8sTUFBTSxHQUFHLENBQUMsT0FBTyxFQUFFLENBQUM7S0FDOUI7SUFBQyxPQUFPLEdBQVEsRUFBRTtRQUNmLE9BQU87S0FDVjtBQUNMLENBQUM7QUFORCwwQkFNQztBQUVZLFFBQUEsYUFBYSxHQUFHLElBQUEsbUJBQVEsRUFDakMsRUFBRSxXQUFXLEVBQUUsQ0FBQyxFQUFFLEVBQ2xCLEtBQUssRUFBRSxNQUFpQixFQUFFLFlBQWtDLEVBQUUsRUFBRSxFQUFFO0lBQzlELE1BQU0sTUFBTSxHQUFHLFNBQUcsQ0FBQyxNQUFNLENBQUMsT0FBTyxDQUFDLENBQUMsQ0FBQyxFQUFFLEdBQUcsRUFBRSxTQUFHLENBQUMsTUFBTSxFQUFFLENBQUMsQ0FBQyxDQUFDLFNBQVMsQ0FBQztJQUNwRSxNQUFNLE1BQU0sR0FBeUI7UUFDakMsVUFBVSxFQUFFLENBQUM7UUFDYixnQkFBZ0IsRUFBRSxJQUFJO1FBQ3RCLE1BQU07UUFDTixHQUFHLFNBQVM7UUFDWixNQUFNO0tBQ1QsQ0FBQztJQUNGLE1BQU0sS0FBSyxHQUFHLElBQUksS0FBSyxDQUFDLEtBQUssQ0FBQyxFQUFFLFNBQVMsRUFBRSxJQUFJLEVBQUUsVUFBVSxFQUFFLElBQUksRUFBRSxPQUFPLEVBQUUsQ0FBQyxFQUFFLENBQUMsQ0FBQztJQUNqRixNQUFNLFFBQVEsR0FBZ0I7UUFDMUIsR0FBRyxFQUFFLElBQUksYUFBRyxDQUFDLEVBQUUsVUFBVSxFQUFFLFlBQVksRUFBRSxHQUFHLE1BQU0sRUFBRSxDQUFDO1FBQ3JELE1BQU0sRUFBRSxJQUFJLGdCQUFNLENBQUMsRUFBRSxVQUFVLEVBQUUsWUFBWSxFQUFFLEdBQUcsTUFBTSxFQUFFLENBQUM7UUFDM0QsMkRBQTJEO1FBQzNELGVBQWU7UUFDZixPQUFPLEVBQUUsSUFBSSxnQkFBTSxDQUFDO1lBQ2hCLFVBQVUsRUFBRSxZQUFZO1lBQ3hCLEdBQUcsTUFBTTtZQUNULGdEQUFnRDtZQUNoRCxVQUFVLEVBQUUsQ0FBQztZQUNiLDhEQUE4RDtZQUM5RCxRQUFRO1lBQ1IsV0FBVyxFQUFFLEVBQUUsT0FBTyxFQUFFLENBQUMsRUFBRSxLQUFLLEVBQUU7U0FDckMsQ0FBQztRQUNGLFVBQVUsRUFBRSxJQUFJLHdCQUFjLENBQUMsRUFBRSxVQUFVLEVBQUUsWUFBWSxFQUFFLEdBQUcsTUFBTSxFQUFFLENBQUM7UUFDdkUsR0FBRyxFQUFFLElBQUksYUFBRyxDQUFDLEVBQUUsVUFBVSxFQUFFLFlBQVksRUFBRSxHQUFHLE1BQU0sRUFBRSxDQUFDO1FBQ3JELEdBQUcsRUFBRSxJQUFJLGFBQUcsQ0FBQyxFQUFFLFVBQVUsRUFBRSxZQUFZLEVBQUUsR0FBRyxNQUFNLEVBQUUsQ0FBQztRQUNyRCxPQUFPLEVBQUUsSUFBSSxpQkFBTyxDQUFDLEVBQUUsTUFBTSxFQUFFLFdBQVcsRUFBRSxHQUFHLE1BQU0sRUFBRSxDQUFDO1FBQ3hELEdBQUcsRUFBRSxJQUFJLGFBQUcsQ0FBQyxFQUFFLFVBQVUsRUFBRSxZQUFZLEVBQUUsR0FBRyxNQUFNLEVBQUUsQ0FBQztRQUNyRCxFQUFFLEVBQUUsSUFBSSxZQUFFLENBQUMsRUFBRSxVQUFVLEVBQUUsWUFBWSxFQUFFLEdBQUcsTUFBTSxFQUFFLENBQUM7S0FDdEQsQ0FBQztJQUNGLE9BQU8sUUFBUSxDQUFDO0FBQ3BCLENBQUMsQ0FDSixDQUFDO0FBRUssS0FBSyxVQUFVLGFBQWEsQ0FDL0IsUUFBZ0IsRUFDaEIsUUFBcUIsRUFDckIsVUFBbUI7SUFFbkIsTUFBTSxFQUFFLEdBQUcsRUFBRSxHQUFHLFFBQVEsQ0FBQztJQUN6QixTQUFHLENBQUMsSUFBSSxDQUFDLGlDQUFpQyxDQUFDLENBQUM7SUFDNUMsSUFBSTtRQUNBLE1BQU0sUUFBUSxHQUFHLE1BQU0sR0FBRyxDQUFDLE9BQU8sQ0FBQyxFQUFFLFFBQVEsRUFBRSxDQUFDLENBQUMsT0FBTyxFQUFFLENBQUM7UUFDM0QsT0FBTyxRQUFRLENBQUMsSUFBSSxDQUFDO0tBQ3hCO0lBQUMsT0FBTyxHQUFRLEVBQUU7UUFDZixJQUFJLENBQUMsVUFBVSxFQUFFO1lBQ2IsTUFBTSxJQUFJLGtCQUFVLENBQUMsR0FBRyxFQUFFLHdCQUF3QixRQUFRLEdBQUcsQ0FBQyxDQUFDO1NBQ2xFO0tBQ0o7SUFDRCxTQUFHLENBQUMsSUFBSSxDQUFDLDBCQUEwQixRQUFRLGlDQUFpQyxDQUFDLENBQUM7SUFDOUUsTUFBTSx3QkFBd0IsR0FBRyxJQUFJLENBQUMsU0FBUyxDQUFDO1FBQzVDLE9BQU8sRUFBRSxZQUFZO1FBQ3JCLFNBQVMsRUFBRTtZQUNQO2dCQUNJLFNBQVMsRUFBRSxFQUFFLE9BQU8sRUFBRSxzQkFBc0IsRUFBRTtnQkFDOUMsTUFBTSxFQUFFLGdCQUFnQjtnQkFDeEIsTUFBTSxFQUFFLE9BQU87YUFDbEI7U0FDSjtLQUNKLENBQUMsQ0FBQztJQUNILE1BQU0sVUFBVSxHQUEwQjtRQUN0Qyx3QkFBd0I7UUFDeEIsUUFBUTtRQUNSLFdBQVcsRUFBRSw0Q0FBNEM7UUFDekQsa0JBQWtCLEVBQUUsSUFBSTtLQUMzQixDQUFDO0lBQ0YsU0FBRyxDQUFDLElBQUksQ0FBQyxvQkFBb0IsQ0FBQyxDQUFDO0lBQy9CLE1BQU0sU0FBUyxHQUFHLDZDQUE2QyxDQUFDO0lBQ2hFLElBQUk7UUFDQSxNQUFNLFlBQVksR0FBRyxNQUFNLEdBQUcsQ0FBQyxVQUFVLENBQUMsVUFBVSxDQUFDLENBQUMsT0FBTyxFQUFFLENBQUM7UUFDaEUsU0FBRyxDQUFDLElBQUksQ0FBQyxxQ0FBcUMsQ0FBQyxDQUFDO1FBQ2hELE1BQU0sR0FBRyxDQUFDLGdCQUFnQixDQUFDLEVBQUUsUUFBUSxFQUFFLFNBQVMsRUFBRSxDQUFDLENBQUMsT0FBTyxFQUFFLENBQUM7UUFDOUQsT0FBTyxZQUFZLENBQUMsSUFBSSxDQUFDO0tBQzVCO0lBQUMsT0FBTyxHQUFRLEVBQUU7UUFDZixJQUFJLEdBQUcsQ0FBQyxJQUFJLEtBQUsscUJBQXFCLEVBQUU7WUFDcEMsTUFBTSxJQUFBLGNBQUssRUFBQyxJQUFJLENBQUMsQ0FBQztZQUNsQixNQUFNLFlBQVksR0FBRyxNQUFNLEdBQUcsQ0FBQyxPQUFPLENBQUMsRUFBRSxRQUFRLEVBQUUsQ0FBQyxDQUFDLE9BQU8sRUFBRSxDQUFDO1lBQy9ELE1BQU0sR0FBRyxDQUFDLGdCQUFnQixDQUFDLEVBQUUsUUFBUSxFQUFFLFNBQVMsRUFBRSxDQUFDLENBQUMsT0FBTyxFQUFFLENBQUM7WUFDOUQsT0FBTyxZQUFZLENBQUMsSUFBSSxDQUFDO1NBQzVCO1FBQ0QsTUFBTSxJQUFJLGtCQUFVLENBQUMsR0FBRyxFQUFFLDBCQUEwQixRQUFRLEdBQUcsQ0FBQyxDQUFDO0tBQ3BFO0FBQ0wsQ0FBQztBQWhERCxzQ0FnREM7QUFFWSxRQUFBLFVBQVUsR0FBRyxJQUFBLG1CQUFRLEVBQzlCLEVBQUUsV0FBVyxFQUFFLENBQUMsRUFBRSxJQUFJLEVBQUUsQ0FBQyxFQUFFLE9BQU8sRUFBRSxJQUFJLEVBQUUsS0FBSyxFQUFFLEVBQUUsRUFBRSxFQUNyRCxhQUFhLENBQ2hCLENBQUM7QUFFRixNQUFNLGVBQWUsR0FBRyxhQUFhLENBQUMscUJBQXFCLENBQUM7QUFDNUQsTUFBTSxVQUFVLEdBQUcsRUFBRSxNQUFNLEVBQUUsR0FBRyxFQUFFLFVBQVUsRUFBRSxFQUFFLEVBQUUsSUFBSSxFQUFFLEVBQUUsRUFBRSxJQUFJLEVBQUUsRUFBRSxFQUFFLGVBQWUsRUFBRSxDQUFDO0FBRWpGLEtBQUssVUFBVSxXQUFXLENBQzdCLE1BQWMsRUFDZCxXQUF3QyxFQUN4QyxvQkFBNkIsRUFDN0IsWUFBb0IsRUFDcEIsTUFBaUIsRUFDakIsZUFBdUIsRUFDdkIsZ0JBQXVEO0lBRXZELElBQUksQ0FBQyxXQUFXLEVBQUU7UUFDZCxPQUFPO0tBQ1Y7SUFDRCxTQUFHLENBQUMsSUFBSSxDQUFDLHVCQUF1QixDQUFDLENBQUM7SUFFbEMsTUFBTSxtQkFBbUIsR0FDckIsT0FBTyxXQUFXLEtBQUssUUFBUTtRQUMzQixDQUFDLENBQUMsQ0FBQyxNQUFNLElBQUEsbUJBQVEsRUFBQyxXQUFXLENBQUMsQ0FBQyxDQUFDLFFBQVEsRUFBRTtRQUMxQyxDQUFDLENBQUMsSUFBSSxDQUFDLFNBQVMsQ0FBQyxXQUFXLENBQUMsQ0FBQztJQUV0QyxJQUFJLFNBQVMsQ0FBQztJQUNkLElBQUksb0JBQW9CLEVBQUU7UUFDdEIsTUFBTSxNQUFNLEdBQUcsSUFBQSxtQkFBVSxFQUFDLFFBQVEsQ0FBQyxDQUFDO1FBQ3BDLE1BQU0sQ0FBQyxNQUFNLENBQUMsbUJBQW1CLENBQUMsQ0FBQztRQUNuQyxNQUFNLENBQUMsTUFBTSxDQUFDLElBQUksQ0FBQyxTQUFTLENBQUMsZ0JBQWdCLENBQUMsYUFBYSxJQUFJLEVBQUUsQ0FBQyxDQUFDLENBQUM7UUFDcEUsTUFBTSxRQUFRLEdBQUcsTUFBTSxDQUFDLE1BQU0sQ0FBQyxLQUFLLENBQUMsQ0FBQztRQUN0QyxTQUFTLEdBQUcsU0FBUyxRQUFRLEVBQUUsQ0FBQztRQUNoQyxNQUFNLE1BQU0sR0FBRyxNQUFNLE9BQU8sQ0FBQyxNQUFNLENBQUMsaUJBQWlCLENBQUMsRUFBRSxTQUFTLEVBQUUsQ0FBQyxDQUFDLENBQUM7UUFDdEUsSUFBSSxNQUFNLEVBQUUsYUFBYSxFQUFFLE1BQU0sSUFBSSxDQUFDLEdBQUcsQ0FBQyxFQUFFO1lBQ3hDLE1BQU0sQ0FBQyxFQUFFLE9BQU8sRUFBRSxlQUFlLEVBQUUsV0FBVyxFQUFFLENBQUMsR0FDN0MsTUFBTSxFQUFFLGFBQWEsSUFBSSxFQUFFLENBQUM7WUFDaEMsSUFBSSxDQUFDLElBQUEsbUJBQVUsRUFBQyxXQUFXLEVBQUUsZUFBZSxDQUFDLElBQUksT0FBTyxJQUFJLGVBQWUsRUFBRTtnQkFDekUsT0FBTyxFQUFFLE9BQU8sRUFBRSxlQUFlLEVBQUUsU0FBUyxFQUFFLENBQUM7YUFDbEQ7U0FDSjtLQUNKO1NBQU07UUFDSCxTQUFTLEdBQUcsWUFBWSxDQUFDO0tBQzVCO0lBRUQsSUFBSTtRQUNBLE1BQU0sV0FBVyxHQUFHLE1BQU0sSUFBQSxnQkFBUSxFQUFDLE1BQU0sRUFBRTtZQUN2QyxNQUFNO1lBQ04sT0FBTyxFQUFFLEdBQUc7WUFDWixVQUFVLEVBQUUsSUFBSTtZQUNoQixJQUFJLEVBQUUsT0FBTztZQUNiLEVBQUUsRUFBRSxLQUFLO1lBQ1QsVUFBVSxFQUFFLENBQUM7WUFDYixjQUFjLEVBQUU7Z0JBQ1osU0FBUyxFQUFFLEVBQUU7YUFDaEI7WUFDRCxnQkFBZ0I7U0FDbkIsQ0FBQyxDQUFDO1FBQ0gsSUFBSTtZQUNBLE1BQU0sV0FBVyxHQUEwQjtnQkFDdkMsbUJBQW1CO2dCQUNuQixTQUFTO2dCQUNULFlBQVk7Z0JBQ1osTUFBTTtnQkFDTixlQUFlO2FBQ2xCLENBQUM7WUFDRixNQUFNLEVBQUUsVUFBVSxFQUFFLFNBQVMsRUFBRSxHQUFHLE1BQU0sV0FBVyxDQUFDLFNBQVMsQ0FBQyxVQUFVLENBQ3BFLFdBQVcsQ0FDZCxDQUFDO1lBQ0YsU0FBRyxDQUFDLElBQUksQ0FBQyxVQUFVLENBQUMsQ0FBQztZQUNyQixPQUFPLFNBQVMsQ0FBQztTQUNwQjtnQkFBUztZQUNOLE1BQU0sV0FBVyxDQUFDLE9BQU8sRUFBRSxDQUFDO1NBQy9CO0tBQ0o7SUFBQyxPQUFPLEdBQVEsRUFBRTtRQUNmLE1BQU0sSUFBSSxrQkFBVSxDQUFDLEdBQUcsRUFBRSxnREFBZ0QsQ0FBQyxDQUFDO0tBQy9FO0FBQ0wsQ0FBQztBQXRFRCxrQ0FzRUM7QUFFRCxTQUFnQixNQUFNLENBQUMsS0FBZTtJQUNsQyxNQUFNLEVBQUUsTUFBTSxFQUFFLFlBQVksRUFBRSxHQUFHLEtBQUssQ0FBQyxTQUFTLENBQUM7SUFDakQsT0FBTyxJQUFBLHNCQUFTLEVBQUMsTUFBTSxFQUFFLFlBQVksQ0FBQyxDQUFDO0FBQzNDLENBQUM7QUFIRCx3QkFHQztBQUVZLFFBQUEsVUFBVSxHQUFHLElBQUEsbUJBQVEsRUFDOUIsRUFBRSxXQUFXLEVBQUUsUUFBUSxFQUFFLElBQUksRUFBRSxDQUFDLEVBQUUsRUFDbEMsS0FBSyxFQUFFLE9BQWUsRUFBRSxLQUFXLEVBQUUsT0FBNkIsRUFBRSxFQUFFO0lBQ2xFLE1BQU0sRUFBRSxNQUFNLEVBQUUsT0FBTyxFQUFFLFVBQVUsRUFBRSxHQUFHLEVBQUUsV0FBVyxFQUFFLElBQUksRUFBRSxHQUFHLE9BQU8sQ0FBQztJQUN4RSxJQUFJLFdBQVcsR0FBRyxHQUFHLElBQUksSUFBSSxLQUFLLE9BQU8sRUFBRTtRQUN2QyxTQUFHLENBQUMsSUFBSSxDQUFDLDZEQUE2RCxDQUFDLENBQUM7UUFDeEUsU0FBRyxDQUFDLElBQUksQ0FBQyx5REFBeUQsQ0FBQyxDQUFDO0tBQ3ZFO0lBQ0QsU0FBRyxDQUFDLElBQUksQ0FBQyxtQkFBbUIsQ0FBQyxDQUFDO0lBQzlCLE1BQU0sUUFBUSxHQUFHLE1BQU0sSUFBQSxxQkFBYSxFQUFDLE1BQU0sRUFBRSxPQUFPLENBQUMsU0FBUyxDQUFDLENBQUM7SUFDaEUsTUFBTSxFQUFFLE1BQU0sRUFBRSxHQUFHLFFBQVEsQ0FBQztJQUM1QixNQUFNLFlBQVksR0FBRyxTQUFTLEtBQUssRUFBRSxDQUFDO0lBQ3RDLE1BQU0sRUFBRSxXQUFXLEVBQUUsb0JBQW9CLEVBQUUsV0FBVyxFQUFFLEdBQUcsT0FBTyxDQUFDO0lBRW5FLEtBQUssVUFBVSxxQkFBcUIsQ0FDaEMsSUFBeUIsRUFDekIsSUFBWSxFQUNaLGdCQUF3QixFQUN4QixTQUF3QjtRQUV4QixNQUFNLEVBQUUsTUFBTSxHQUFHLEVBQUUsRUFBRSxHQUFHLElBQUksRUFBRSxHQUFHLE9BQU8sQ0FBQyxnQkFBZ0IsQ0FBQztRQUMxRCxJQUFJLFNBQVMsRUFBRTtZQUNYLE1BQU0sQ0FBQyxJQUFJLENBQUMsU0FBUyxDQUFDLGVBQWUsQ0FBQyxDQUFDO1NBQzFDO1FBQ0QsTUFBTSxPQUFPLEdBQWlDO1lBQzFDLFlBQVk7WUFDWixJQUFJO1lBQ0osT0FBTyxFQUFFLFlBQVk7WUFDckIsT0FBTyxFQUFFLGtCQUFrQjtZQUMzQixJQUFJO1lBQ0osV0FBVyxFQUFFLDJCQUEyQjtZQUN4QyxPQUFPLEVBQUUsT0FBTztZQUNoQixVQUFVLEVBQUUsVUFBVTtZQUN0QixXQUFXLEVBQUUsRUFBRSxTQUFTLEVBQUUsR0FBRyxFQUFFO1lBQy9CLE1BQU07WUFDTixHQUFHLElBQUk7U0FDVixDQUFDO1FBQ0YsU0FBRyxDQUFDLElBQUksQ0FBQywyQkFBMkIsRUFBRSxPQUFPLENBQUMsQ0FBQztRQUMvQyxJQUFJLElBQUksQ0FBQztRQUNULElBQUk7WUFDQSxJQUFJLEdBQUcsTUFBTSxNQUFNLENBQUMsY0FBYyxDQUFDLE9BQU8sQ0FBQyxDQUFDLE9BQU8sRUFBRSxDQUFDO1lBQ3RELE1BQU0sSUFBQSxrQkFBTyxFQUFDLENBQUMsRUFBRSxHQUFHLEVBQUUsQ0FDbEIsTUFBTSxDQUFDLE9BQU8sQ0FBQyxnQkFBZ0IsRUFBRSxFQUFFLFlBQVksRUFBRSxDQUFDLENBQUMsT0FBTyxFQUFFLENBQy9ELENBQUM7U0FDTDtRQUFDLE9BQU8sR0FBUSxFQUFFO1lBQ2YsSUFBSSxHQUFHLEVBQUUsT0FBTyxFQUFFLEtBQUssQ0FBQyx3QkFBd0IsQ0FBQyxFQUFFO2dCQUMvQyxJQUFJLEdBQUcsQ0FBQyxNQUFNLE1BQU0sQ0FBQyxXQUFXLENBQUMsRUFBRSxZQUFZLEVBQUUsQ0FBQyxDQUFDLE9BQU8sRUFBRSxDQ