@artilleryio/platform-fargate/lib/commands/run-cluster.js

Fargate support for Artillery

"use strict";const AWS=require("aws-sdk"),debug=require("debug")("commands:run-test"),debugVerbose=require("debug")("commands:run-test:v"),debugErr=require("debug")("commands:run-test:errors"),A=require("async"),customAlphabet=require("nanoid")["customAlphabet"],path=require("path"),fs=require("fs"),chalk=require("chalk"),defaultOptions=require("rc")("artillery"),moment=require("moment"),EnsurePlugin=require("artillery-plugin-ensure"),EventEmitter=require("events"),Dino=require("../animate-dino")["Dino"],_=require("lodash"),pkg=require("../../package.json"),utils=require("../util"),parseTags=require("../tags")["parseTags"],{Timeout,sleep,timeStringToMs}=require("../time"),SqsReporter=require("../sqs-reporter")["SqsReporter"],awaitOnEE=require("../utils/await-on-ee"),VPCSubnetFinder=require("../find-public-subnets")["VPCSubnetFinder"],awsUtil=require("../aws-util"),createTest=require("./create-test")["createTest"],TestRunObject=require("../data-api/test-run-object")["TestRunObject"],TestBundle=require("../data-api/test-object")["TestBundle"],createS3Client=require("../utils/create-s3-client"),getBucketName=require("../util")["getBucketName"],stopTest=require("./stop-test")["stopTest"],versionCompare=require("../utils/version-compare"),normalizeCLIFlags=require("../utils/normalize-cli-flags"),getBackendId=require("../utils/get-backend-id"),getBackendStore=require("../utils/get-backend-store"),createStore=require("../data-api/store"),dotenvParse=require("dotenv").parse,util=require("../util"),setDefaultAWSCredentials=require("../utils/aws-set-default-credentials");module.exports=runCluster;let consoleReporter={toggleSpinner:()=>{}};const{TASK_NAME,CF_STACK_NAME,S3_BUCKET_NAME_PREFIX,WORKER_ROLE_ARN_OUTPUT_KEY,SQS_QUEUES_NAME_PREFIX,LOGGROUP_NAME,IMAGE_VERSION,WAIT_TIMEOUT,MINIMUM_BACKEND_VERSION,ARTILLERY_CLUSTER_NAME,TEST_RUNS_MAX_TAGS}=require("../constants"),{TestNotFoundError,NoAvailableQueueError,ClientServerVersionMismatchError,ConsoleOutputSerializeError}=require("../errors");let MW=100,MW_T=10,MCR=10,IS_FARGATE=!1;const TEST_RUN_STATUS=require("../test-run-status");function getStatusString(n){for(var[key,value]of Object.entries(TEST_RUN_STATUS))if(value===n)return key}function setupConsoleReporter(){var reporterOpts={outputFormat:"classic",printPeriod:!1},reporterEvents=(global.artillery&&global.artillery.version&&global.artillery.version.startsWith("2")&&(delete reporterOpts.outputFormat,delete reporterOpts.printPeriod),new EventEmitter);return consoleReporter=global.artillery.__createReporter(reporterEvents,reporterOpts),global.artillery&&global.artillery.version&&!global.artillery.version.startsWith("2")&&(consoleReporter.spinner.stop(),consoleReporter.spinner.clear(),consoleReporter.spinner={start:()=>{},stop:()=>{},clear:()=>{}}),{reporterEvents:reporterEvents}}function runCluster(scriptPath,options){process.env.DEBUG&&(AWS.config.logger=console);var artilleryReporter=setupConsoleReporter();-1===versionCompare(global.artillery.version,"1.7.7")&&(artillery.log(chalk.red("Version mismatch: ECS/Fargate support requires Artillery v1.7.7 or later")),process.exit(1)),global.artillery.version.startsWith("2")&&-1===versionCompare(global.artillery.version,"2.0.0-dev7")&&(artillery.log(chalk.red("Version mismatch: Artillery v2.0.0-dev7 or later is required")),process.exit(1)),tryRunCluster(scriptPath,options,artilleryReporter)}function logProgress(msg,opts={}){void 0===opts.showTimestamp&&(opts.showTimestamp=!0),global.artillery&&global.artillery.log?artillery.logger(opts).log(msg):(consoleReporter.toggleSpinner(),artillery.log(msg+" "+chalk.gray("["+moment().format("HH:mm:ss")+"]")),consoleReporter.toggleSpinner())}async function tryRunCluster(scriptPath,options,artilleryReporter){normalizeCLIFlags(["task-role-arn","launch-type","launch-config","subnet-ids","security-group-ids","max-duration"],options);const MAX_RETAINED_LOG_SIZE_MB=Number(process.env.MAX_RETAINED_LOG_SIZE_MB||"50"),MAX_RETAINED_LOG_SIZE=1024*MAX_RETAINED_LOG_SIZE_MB*1024;let currentSize=0,outputLines=[],truncated=!1;console.log=function(){let orig=console.log;return function(){if(orig.apply(console,arguments),currentSize<MAX_RETAINED_LOG_SIZE){outputLines=outputLines.concat(arguments);for(const x of arguments)currentSize+=String(x).length}else{var msg;truncated||(truncated=!0,msg=`[WARNING] Artillery: maximum retained log size exceeded, max size: ${MAX_RETAINED_LOG_SIZE_MB}MB. Further logs won't be retained.

`,process.stdout.write(msg),outputLines=outputLines.concat([msg]))}}}(),console.error=function(){let orig=console.error;return function(){if(orig.apply(console,arguments),currentSize<MAX_RETAINED_LOG_SIZE){outputLines=outputLines.concat(arguments);for(const x of arguments)currentSize+=String(x).length}else{var msg;truncated||(truncated=!0,msg=`[WARNING] Artillery: maximum retained log size exceeded, max size: ${MAX_RETAINED_LOG_SIZE_MB}MB. Further logs won't be retained.

`,process.stdout.write(msg),outputLines=outputLines.concat([msg]))}}}();try{await setDefaultAWSCredentials(AWS)}catch(err){console.error(err),process.exit(1)}let context={},absoluteScriptPath;if(void 0!==scriptPath){absoluteScriptPath=path.resolve(process.cwd(),scriptPath),context.namedTest=!1;try{fs.statSync(absoluteScriptPath)}catch(statErr){artillery.log("Could not read file:",scriptPath),artillery.log("Trying to run a named test? Use the --bundle option."),process.exit(1)}}if(options.dotenv&&(contents=fs.readFileSync(path.resolve(process.cwd(),options.dotenv)),context.dotenv=dotenvParse(contents)),options.bundle&&(context.namedTest=!0),options.maxDuration)try{var maxDurationMs=timeStringToMs(options.maxDuration);context.maxDurationMs=maxDurationMs}catch(err){throw err}context.tags=parseTags(options.tags),context.tags.length>TEST_RUNS_MAX_TAGS&&(console.error(chalk.red(`A maximum of ${TEST_RUNS_MAX_TAGS} tags is allowed per test run`)),process.exit(1)),0===context.tags.filter(t=>"name"===t.name).length&&(void 0!==scriptPath?context.tags.push({name:"name",value:path.basename(scriptPath)}):context.tags.push({name:"name",value:options.bundle})),context.extraSecrets=options.secret||[];var contents=customAlphabet("3456789abcdefghjkmnpqrtwxyz"),maxDurationMs=(context.testId=`t${contents(4)}_${contents(29)}_`+contents(4),context.shortTestId=context.testId,context.namedTest&&(context.s3Prefix=options.bundle,debug("Trying to run a named test: "+context.s3Prefix)),context.namedTest||(maxDurationMs=options.context?path.resolve(options.context):path.dirname(absoluteScriptPath),debugVerbose("script:",absoluteScriptPath),debugVerbose("root:",maxDurationMs),-1!==(contents=path.join(path.relative(maxDurationMs,path.dirname(absoluteScriptPath)),path.basename(absoluteScriptPath))).indexOf("..")&&(artillery.log(chalk.red("Test script must reside inside the context dir. See Artillery docs for more details.")),process.exit(1)),context.contextDir=maxDurationMs,context.newScriptPath=contents,debug("container script path:",contents)),Number(options.count)||1);if(void 0!==options.taskRoleName){let customRoleName=options.taskRoleName;customRoleName.startsWith("arn:aws:iam")&&(customRoleName=customRoleName.split(":role/")[1]),context.customTaskRoleName=customRoleName}contents=options.cluster||ARTILLERY_CLUSTER_NAME;if(options.launchConfig){let launchConfig;try{launchConfig=JSON.parse(options.launchConfig)}catch(parseErr){debug(parseErr)}launchConfig||(artillery.log(chalk.red("Launch config could not be parsed. Please check that it's valid JSON.")),process.exit(1)),launchConfig.ulimits&&!Array.isArray(launchConfig.ulimits)&&(artillery.log(chalk.red("ulimits must be an array of objects")),artillery.log("Please see AWS documentation for more information:\nhttps://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_Ulimit.html"),process.exit(1)),options.launchConfig=launchConfig}void 0!==options.launchType&&"ecs:fargate"!==options.launchType&&"ecs:ec2"!==options.launchType&&(artillery.log('Invalid launch type - the value of --launch-type needs to be "ecs:fargate" or "ecs:ec2"'),process.exit(1)),void 0!==options.fargate&&console.error("The --fargate flag is deprecated, use --launch-type ecs:fargate instead"),options.fargate&&options.launchType&&(console.error("Either --fargate or --launch-type flag should be set, not both"),process.exit(1)),void 0===options.fargate&&void 0===options.launchType&&(options.launchType="ecs:fargate"),IS_FARGATE=void 0!==options.fargate||void 0!==options.publicSubnetIds||(options.launchType,"ecs:fargate"===options.launchType)||void 0===options.launchType,global.artillery.globalEvents.emit("test:init",{flags:options,testRunId:context.shortTestId,tags:context.tags,metadata:{testId:context.shortTestId,startedAt:Date.now(),count:maxDurationMs,tags:context.tags,launchType:options.launchType}});let packageJsonPath;if(options.packages){packageJsonPath=path.resolve(process.cwd(),options.packages);try{JSON.parse(fs.readFileSync(packageJsonPath,"utf8"))}catch(err){console.error("Could not load package dependency list"),console.error("Trying to read from:",packageJsonPath),console.error(err)}}context=Object.assign(context,{scriptPath:absoluteScriptPath,originalScriptPath:scriptPath,count:maxDurationMs,taskName:`${TASK_NAME}_${IS_FARGATE?"fargate":""}_${contents}_${IMAGE_VERSION}_`+Math.floor(1e6*Math.random()),clusterName:contents,logGroupName:LOGGROUP_NAME,cliOptions:options,isFargate:IS_FARGATE,configTableName:"",status:TEST_RUN_STATUS.INITIALIZING,packageJsonPath:packageJsonPath,taskArns:[]}),void 0!==options.region&&-1===util.supportedRegions.indexOf(options.region)&&(console.log(`Unsupported region (${options.region}) provided. Please specify one of: ${util.supportedRegions.join(", ")} `),process.exit(1)),void 0!==options.region&&(context.region=options.region);let subnetIds=[];options.publicSubnetIds&&(console.error(chalk.yellow("Warning")+": --public-subnet-ids will be deprecated. Use --subnet-ids instead."),subnetIds=options.publicSubnetIds.split(",")),options.subnetIds&&(subnetIds=options.subnetIds.split(",")),IS_FARGATE&&(context.fargatePublicSubnetIds=subnetIds,context.fargateSecurityGroupIds=void 0!==options.securityGroupIds?options.securityGroupIds.split(","):[]);await getBucketName();scriptPath=new TestRunObject(context.testId,await getBackendStore());await scriptPath.init(),context.testRunObject=scriptPath,global.artillery&&global.artillery.telemetry&&global.artillery.telemetry.capture("run-test",{version:global.artillery.version,proVersion:pkg.version,count:maxDurationMs,launchPlatform:IS_FARGATE?"ecs:fargate":"ecs:ec2",usesTags:0<context.tags.length,region:context.region,crossRegion:context.region!==context.backendRegion}),async function(artilleryReporter){let testRunCompletedSuccessfully=!0;async function initCleanup(){artillery.log("Stopping test run..."),context.status=TEST_RUN_STATUS.TERMINATING,await sleep(5e3),artillery.log("Cleaning up..."),await sleep(5e3),context.status=TEST_RUN_STATUS.EARLY_STOP,await cleanup(context,{clean:!1}),process.exit(1)}let lastCtrlC=Date.now();process.on("SIGINT",async()=>{500<Date.now()-lastCtrlC?(artillery.log("Press Ctrl+C twice to stop the test"),lastCtrlC=Date.now()):await initCleanup()}),process.on("SIGTERM",initCleanup),context.reporterEvents=artilleryReporter.reporterEvents;try{if(logProgress("Checking AWS connectivity..."),await Promise.all([getAccountId(context),getrc(context)]),await setBackend(context),await Promise.all([async function(context){var bucketName=await getBucketName();return context.s3Bucket=bucketName,context}(context)]),logProgress("Checking cluster..."),!await checkTargetCluster(context)){if(void 0!==context.cliOptions.cluster)throw new Error(`Could not find cluster ${context.clusterName} in `+context.region);await createArtilleryCluster(context)}0<context.tags.length&&(await context.testRunObject.setTags(context.tags),logProgress("Tags: "+context.tags.map(t=>t.name+":"+t.value).join(", "))),logProgress("Test run ID: "+context.shortTestId),logProgress("Preparing launch platform..."),await Promise.all([checkParallelTestRuns(context),maybeGetSubnetIdsForFargate(context),getWorkerRoleArn(context)]),logProgress(`Environment:
  Account:     ${context.accountId}
  Region:      ${context.region}
  Count:       ${context.count}
  Cluster:     ${context.clusterName}
  Launch type: ${context.cliOptions.launchType}
  Subnets:     ${context.fargatePublicSubnetIds.join(", ")} ${context.cliOptions.subnetIds?"":"(autodetected)"}
  ${context.backendSetExplicitly?"Backend: "+context.backendRegion:""}
`,{showTimestamp:!1}),await createQueue(context),await checkCustomTaskRole(context),await ensureTaskExists(context),logProgress("Preparing test bundle..."),await maybeCreateAdhocTest(context),await getManifest(context),await generateTaskOverrides(context),logProgress("Launching workers..."),await setupDefaultECSParams(context),context.cliOptions.note&&await context.testRunObject.addNote(context.cliOptions.note),listen(context,artilleryReporter.reporterEvents),context.status!==TEST_RUN_STATUS.EARLY_STOP&&context.status!==TEST_RUN_STATUS.TERMINATING&&await launchLeadTask(context),context.status!==TEST_RUN_STATUS.EARLY_STOP&&context.status!==TEST_RUN_STATUS.TERMINATING&&(logProgress(context.isFargate?"Waiting for Fargate...":"Waiting for ECS..."),await ecsRunTask(context)),context.status!==TEST_RUN_STATUS.EARLY_STOP&&context.status!==TEST_RUN_STATUS.TERMINATING&&await waitForTasks2(context),context.status!==TEST_RUN_STATUS.EARLY_STOP&&context.status!==TEST_RUN_STATUS.TERMINATING&&(logProgress("Waiting for workers to come online..."),await waitForWorkerSync(context),await sendGoSignal(context),logProgress("Workers are running, waiting for reports..."),context.maxDurationMs&&0<context.maxDurationMs&&(logProgress("Max duration for test run is set to: "+context.cliOptions.maxDuration),(testDurationTimeout=new Timeout(context.maxDurationMs)).start(),testDurationTimeout.on("timeout",async()=>{artillery.log(`Max duration of test run exceeded: ${context.cliOptions.maxDuration}
`),await initCleanup()})),context.status=TEST_RUN_STATUS.RECEIVING_REPORTS,await context.testRunObject.setStatus(getStatusString(context.status)));var workerState=await awaitOnEE(artilleryReporter.reporterEvents,"workersDone");debug(workerState),logProgress("Test run completed: "+context.shortTestId),context.status=TEST_RUN_STATUS.COMPLETED,await context.testRunObject.setStatus(getStatusString(context.status));let checks=[];global.artillery.globalEvents.once("checks",async results=>{checks=results,await context.testRunObject.setChecks(checks)}),context.ensureSpec&&new EnsurePlugin.Plugin({config:{ensure:context.ensureSpec}});for(const e of global.artillery.extensionEvents){var ps=[],testInfo={endTime:Date.now()};"beforeExit"===e.ext&&ps.push(e.method({report:context.aggregateReport,flags:context.cliOptions,runnerOpts:{environment:context.cliOptions?.environment,scriptPath:"",absoluteScriptPath:""},testInfo:testInfo})),await Promise.allSettled(ps)}if(context.cliOptions.output){var logfile=getLogFilename(context.cliOptions.output,defaultOptions.logFilenameFormat);for(const ix of context.intermediateReports)delete ix.histograms,ix.histograms=ix.summaries;delete context.aggregateReport.histograms,context.aggregateReport.histograms=context.aggregateReport.summaries;var jsonReport={intermediate:context.intermediateReports,aggregate:context.aggregateReport,testId:context.testId,metadata:{tags:context.tags,count:context.count,region:context.region,cluster:context.clusterName,artilleryVersion:{core:global.artillery.version,pro:pkg.version}},ensure:checks.map(c=>({condition:c.original,success:1===c.result,strict:c.strict}))};fs.writeFileSync(logfile,JSON.stringify(jsonReport,null,2),{flag:"w"})}debug(context.testId,"done")}catch(err){debug(err),"InvalidParameterException"===err.code?-1!==err.message.toLowerCase().indexOf("no container instances were found")?artillery.log(chalk.yellow("The cluster has no active instances. We need some instances.")):artillery.log(err):err instanceof TestNotFoundError?artillery.log(`Test ${context.s3Prefix} not found`):err instanceof NoAvailableQueueError||err instanceof ClientServerVersionMismatchError?artillery.log(chalk.red("Error:",err.message)):"LicenseKeyExpiredError"===err.name||"TrialLimitExceededError"===err.name?(artillery.log(util.formatError(err)),artillery.log("A license can be purchased on https://artillery.io/pro or by contacting sales@artillery.io")):(artillery.log(util.formatError(err)),artillery.log(err),artillery.log(err.stack)),testRunCompletedSuccessfully=!1,global.artillery.suggestedExitCode=1}finally{testRunCompletedSuccessfully?await cleanup(context,{clean:!0}):(logProgress("Cleaning up..."),context.status=TEST_RUN_STATUS.ERROR,await cleanup(context,{clean:!1}));var testDurationTimeout=Date.now();await context.testRunObject.setEndedAt(testDurationTimeout),await context.testRunObject.setReport([],context.aggregateReport);try{await context.testRunObject.setTextLog(outputLines)}catch(err){err instanceof ConsoleOutputSerializeError?(artillery.log(err),artillery.log("This could happen due to printing a circular structure with console.log")):artillery.log(err)}await context.testRunObject.setStatus(getStatusString(context.status),testDurationTimeout);var shutdownOpts={exitCode:global.artillery.suggestedExitCode};global.artillery.globalEvents.emit("shutdown:start",shutdownOpts);for(const e of global.artillery.extensionEvents){const ps=[];"onShutdown"===e.ext&&ps.push(e.method(shutdownOpts)),await Promise.allSettled(ps)}process.exit(global.artillery.suggestedExitCode)}}(artilleryReporter)}async function cleanup(context,opts){try{context.sqsReporter&&context.sqsReporter.stop();for(const taskArn of context.taskArns)try{await new AWS.ECS({apiVersion:"2014-11-13",region:testRunObject.metadata.region}).stopTask({task:taskArn,cluster:context.clusterName,reason:"Test cleanup"}).promise()}catch(err){debug(err)}await Promise.all([deleteQueue(context),deregisterTaskDefinition(context),gcQueues(context)]),opts.clean?context.status=TEST_RUN_STATUS.COMPLETED:context.status!==TEST_RUN_STATUS.ERROR&&(context.status=TEST_RUN_STATUS.EARLY_STOP),await context.testRunObject.setStatus(getStatusString(context.status))}catch(err){artillery.log(err)}}function checkFargateResourceConfig(cpu,memory){var FARGATE_VALID_CONFIGS={256:[512,1024,2048],512:[1024,2048,3072,4096],1024:[2048,3072,4096,5120,6144,7168,8192],2048:[4096,5120,6144,7168,8192,10240,11264,12288],4096:[8192,9216,10240,11264,12288]};return FARGATE_VALID_CONFIGS[cpu]?FARGATE_VALID_CONFIGS[cpu].indexOf(memory)<0?new Error(`Fargate memory override for cpu = ${cpu} must be one of: `+FARGATE_VALID_CONFIGS[cpu].join(", ")):null:new Error("Unsupported cpu override for Fargate. Must be one of: "+Object.keys(FARGATE_VALID_CONFIGS).join(", "))}async function getAccountId(context){return new Promise((resolve,reject)=>{utils.getAccountId((err,accountId)=>err?reject(err):(context.accountId=accountId,resolve(context)))})}async function setBackend(context){var backendId,isDefault;process.env.USE_NOOP_BACKEND_STORE||({backendId,isDefault}=await getBackendId(),context.backendRegion=backendId,context.backendSetExplicitly=!isDefault,void 0===context.region&&(context.region=context.backendRegion))}async function createArtilleryCluster(context){var ecs=new AWS.ECS({apiVersion:"2014-11-13",region:context.region});try{await ecs.createCluster({clusterName:ARTILLERY_CLUSTER_NAME,capacityProviders:["FARGATE"]}).promise();let retries=0;for(;retries<12;){if(await checkTargetCluster(context))break;retries++,await sleep(1e4)}}catch(err){throw err}}async function checkTargetCluster(context){var ecs=new AWS.ECS({apiVersion:"2014-11-13",region:context.region});try{var response=await ecs.describeClusters({clusters:[context.clusterName]}).promise();return debug(response),0===response.clusters.length||0<response.failures.length?(debugVerbose(response),!1):0<response.clusters.filter(c=>"ACTIVE"===c.status).length}catch(err){return debugVerbose(err),!1}}async function maybeGetSubnetIdsForFargate(context){if(context.isFargate&&!(0<context.fargatePublicSubnetIds.length)){debug("Subnet IDs not provided, looking up default VPC");var publicSubnets=await new VPCSubnetFinder({region:context.region}).findPublicSubnets();if(0===publicSubnets.length)throw new Error("Could not find public subnets in default VPC");context.fargatePublicSubnetIds=publicSubnets.map(s=>s.SubnetId),debug("Found public subnets:",context.fargatePublicSubnetIds.join(", "))}return context}async function maybeCreateAdhocTest(context){return new Promise((resolve,reject)=>{if(!0===context.namedTest)return resolve(context);createTest(context.scriptPath,{name:context.testId,config:context.cliOptions.config,packageJsonPath:context.packageJsonPath},function(err,result){return err?reject(err):resolve(context)})})}async function ensureTaskExists(context){return new Promise((resolve,reject)=>{const ecs=new AWS.ECS({apiVersion:"2014-11-13",region:context.region});let cpu=4096,memory=8192;context.isFargate||(memory=2048);const defaultUlimits={nofile:{softLimit:8192,hardLimit:8192}};if(context.cliOptions.launchConfig){var lc=context.cliOptions.launchConfig;if(lc.cpu&&(cpu=parseInt(lc.cpu,10)),lc.memory&&(memory=parseInt(lc.memory,10)),lc.ulimits&&lc.ulimits.forEach(u=>{defaultUlimits[u.name]||(defaultUlimits[u.name]={}),defaultUlimits[u.name]={softLimit:u.softLimit,hardLimit:"number"==typeof u.hardLimit?u.hardLimit:u.softLimit}}),context.isFargate){var lc=checkFargateResourceConfig(cpu,memory);if(lc)return reject(lc)}}lc=Object.keys(defaultUlimits).map(name=>({name:name,softLimit:defaultUlimits[name].softLimit,hardLimit:defaultUlimits[name].hardLimit}));let imageUrl=process.env.WORKER_IMAGE_URL||`301676560329.dkr.ecr.${context.region}.amazonaws.com/artillery-pro/aws-ecs-node:`+IMAGE_VERSION;global.artillery&&global.artillery.version&&global.artillery.version.startsWith("2")&&(imageUrl=process.env.WORKER_IMAGE_URL||`301676560329.dkr.ecr.${context.region}.amazonaws.com/artillery-pro/aws-ecs-node:v2-`+IMAGE_VERSION),process.env.WORKER_NODEJS_VERSION&&(imageUrl+="-"+process.env.WORKER_NODEJS_VERSION);var secrets=[].concat(context.extraSecrets).map(secretName=>({name:secretName,valueFrom:`arn:aws:ssm:${context.backendRegion}:${context.accountId}:parameter/artilleryio/`+secretName}));let taskDefinition={family:context.taskName,containerDefinitions:[{name:"artillery",image:imageUrl,cpu:cpu,command:[],memory:memory,secrets:secrets,ulimits:lc,essential:!0,logConfiguration:{logDriver:"awslogs",options:{"awslogs-group":context.logGroupName+"/"+context.clusterName,"awslogs-region":context.region,"awslogs-stream-prefix":"artilleryio/"+context.shortTestId,"awslogs-create-group":"true",mode:"non-blocking"}}}],executionRoleArn:context.taskRoleArn};context.taskDefinition=taskDefinition,context.isFargate&&(taskDefinition.networkMode="awsvpc",taskDefinition.requiresCompatibilities=["FARGATE"],taskDefinition.cpu=String(cpu),taskDefinition.memory=String(memory),taskDefinition.executionRoleArn=context.taskRoleArn);secrets={taskDefinition:context.taskName};debug("Task definition\n",JSON.stringify(taskDefinition,null,4)),ecs.describeTaskDefinition(secrets,function(err,data){if(!err)return debug("OK: ECS task exists"),process.env.ECR_IMAGE_VERSION&&debug("ECR_IMAGE_VERSION is set, but the task definition was already in place."),resolve(context);ecs.registerTaskDefinition(taskDefinition,function(err,response){return err?(artillery.log(err),artillery.log("Could not create ECS task, please try again"),reject(err)):(debug("OK: ECS task registered"),debugVerbose(JSON.stringify(response,null,4)),context.taskDefinitionArn=response.taskDefinition.taskDefinitionArn,debug("Task definition ARN: "+context.taskDefinitionArn),resolve(context))})})})}async function getrc(context){return new Promise((resolve,reject)=>{resolve()})}async function checkCustomTaskRole(context){if(context.customTaskRoleName){var iam=new AWS.IAM;try{var roleData=await iam.getRole({RoleName:context.customTaskRoleName}).promise();context.customRoleArn=roleData.Role.Arn,context.taskRoleArn=roleData.Role.Arn,debug(roleData)}catch(err){throw err}}}async function getWorkerRoleArn(context){return new Promise((resolve,reject)=>{if(context.customTaskRoleName)return resolve(context);var cf=new AWS.CloudFormation({apiVersion:"2010-05-15",region:context.backendRegion}),params={StackName:CF_STACK_NAME};cf.describeStacks(params,function(err,cfResponse){return err?(debug(err),reject(err)):1===cfResponse.Stacks.length?(cfResponse.Stacks[0].Outputs.forEach(o=>{o.OutputKey===WORKER_ROLE_ARN_OUTPUT_KEY&&(context.taskRoleArn=o.OutputValue,debug("taskRoleArn:",context.taskRoleArn))}),resolve(context)):(debugVerbose(cfResponse),reject(new Error("Expected to find exactly one stack with the name "+CF_STACK_NAME)))})})}async function gcQueues(context){var sqs=new AWS.SQS({region:context.region});let data;try{data=await sqs.listQueues({QueueNamePrefix:SQS_QUEUES_NAME_PREFIX,MaxResults:1e3}).promise()}catch(err){}if(data&&data.QueueUrls&&0<data.QueueUrls.length)for(const qu of data.QueueUrls)try{const data=await sqs.getQueueAttributes({QueueUrl:qu,AttributeNames:["CreatedTimestamp"]}).promise();var ts=1e3*Number(data.Attributes.CreatedTimestamp);3456e5<Date.now()-ts&&await sqs.deleteQueue({QueueUrl:qu}).promise()}catch(err){debug(err)}}async function checkParallelTestRuns(context){}async function deleteQueue(context){if(context.sqsQueueUrl){var sqs=new AWS.SQS({region:context.region});try{await sqs.deleteQueue({QueueUrl:context.sqsQueueUrl}).promise()}catch(err){console.error("Unable to clean up SQS queue. URL: "+context.sqsQueueUrl),debug(err)}}}async function createQueue(context){var sqs=new AWS.SQS({region:context.region}),queueName=`${SQS_QUEUES_NAME_PREFIX}_${context.testId.slice(0,30)}.fifo`,params={QueueName:queueName,Attributes:{FifoQueue:"true",ContentBasedDeduplication:"false",MessageRetentionPeriod:"1800",VisibilityTimeout:"600"}};try{var result=await sqs.createQueue(params).promise();context.sqsQueueUrl=result.QueueUrl}catch(err){throw err}let waited=0,ok=!1;for(;waited<12e4;)try{var results=await sqs.listQueues({QueueNamePrefix:queueName}).promise();if(results.QueueUrls&&1===results.QueueUrls.length){debug("SQS queue created:",queueName),ok=!0;break}await sleep(1e4),waited+=1e4}catch(err){await sleep(1e4),waited+=1e4}if(!ok)throw debug("Time out waiting for SQS queue:",queueName),new Error("SQS queue could not be created")}async function getManifest(context){try{var metadata=await new TestBundle(context.namedTest?context.s3Prefix:context.testId).getManifest();return context.newScriptPath=metadata.scriptPath,metadata.configPath&&(context.configPath=metadata.configPath),context}catch(err){throw"NoSuchKey"===err.code?new TestNotFoundError:err}}async function generateTaskOverrides(context){var cliArgs=["run"].concat(context.cliOptions.environment?["--environment",context.cliOptions.environment]:[],context.cliOptions["scenario-name"]?["--scenario-name",context.cliOptions["scenario-name"]]:[],context.cliOptions.insecure?["-k"]:[],context.cliOptions.target?["-t",context.cliOptions.target]:[],context.cliOptions.overrides?["--overrides",context.cliOptions.overrides]:[],context.configPath?["--config",context.configPath]:[]),s3path=(cliArgs.push(context.newScriptPath),debug("cliArgs",cliArgs,cliArgs.join(" ")),`s3://${context.s3Bucket}/tests/`+(context.namedTest?context.s3Prefix:context.testId)),s3path={containerOverrides:[{name:"artillery",command:["-p",s3path,"-a",util.btoa(JSON.stringify(cliArgs)),"-r",context.region||context.backendRegion,"-q",process.env.SQS_QUEUE_URL||context.sqsQueueUrl,"-i",context.testId,"-d",`s3://${context.s3Bucket}/test-runs`,"-t",String(WAIT_TIMEOUT)],environment:[{name:"AWS_SDK_JS_SUPPRESS_MAINTENANCE_MODE_MESSAGE",value:"1"}]}],taskRoleArn:context.taskRoleArn};if(context.customRoleArn&&(s3path.taskRoleArn=context.customRoleArn),global.artillery&&global.artillery.version&&global.artillery.version.startsWith("2")&&s3path.containerOverrides[0].environment.push({name:"USE_V2",value:"true"}),context.dotenv){var name,value,extraEnv=[];for([name,value]of Object.entries(context.dotenv))extraEnv.push({name:name,value:value});s3path.containerOverrides[0].environment=s3path.containerOverrides[0].environment.concat(extraEnv)}return context.cliOptions.launchConfig&&((cliArgs=context.cliOptions.launchConfig).environment&&(s3path.containerOverrides[0].environment=s3path.containerOverrides[0].environment.concat(cliArgs.environment)),cliArgs.taskRoleArn&&(s3path.taskRoleArn=cliArgs.taskRoleArn),cliArgs.command)&&(s3path.containerOverrides[0].command=cliArgs.command),debug("OK: Overrides generated"),debugVerbose(JSON.stringify(s3path,null,4)),context.taskOverrides=s3path,context}async function setupDefaultECSParams(context){var defaultParams={taskDefinition:context.taskName,cluster:context.clusterName,overrides:context.taskOverrides};return context.isFargate&&(defaultParams.launchType="FARGATE",defaultParams.networkConfiguration={awsvpcConfiguration:{assignPublicIp:"ENABLED",securityGroups:context.fargateSecurityGroupIds,subnets:context.fargatePublicSubnetIds}}),context.defaultECSParams=defaultParams,context}async function launchLeadTask(context){var metadata={testId:context.shortTestId,startedAt:Date.now(),cluster:context.clusterName,region:context.region,launchType:context.cliOptions.launchType,count:context.count,sqsQueueUrl:context.sqsQueueUrl,tags:context.tags,secrets:JSON.stringify(Array.isArray(context.extraSecrets)?context.extraSecrets:[context.extraSecrets]),platformConfig:JSON.stringify({memory:context.taskDefinition.containerDefinitions[0].memory,cpu:context.taskDefinition.containerDefinitions[0].cpu}),artilleryVersion:JSON.stringify({core:global.artillery.version,pro:pkg.version})},metadata=(artillery.globalEvents.emit("metadata",metadata),await context.testRunObject.setMetadata(metadata),context.status=TEST_RUN_STATUS.LAUNCHING_WORKERS,await context.testRunObject.setStatus(getStatusString(context.status)),new AWS.ECS({apiVersion:"2014-11-13",region:context.region})),leaderParams=Object.assign({count:1},JSON.parse(JSON.stringify(context.defaultECSParams)));leaderParams.overrides.containerOverrides[0].environment.push({name:"IS_LEADER",value:"true"});try{var uniqueReasons,runData=await metadata.runTask(leaderParams).promise();if(0<runData.failures.length)throw runData.failures.length===context.count?(artillery.log("ERROR: Worker start failure"),uniqueReasons=[...new Set(runData.failures.map(f=>f.reason))],artillery.log("Reason:",uniqueReasons),new Error("Could not start workers")):(artillery.log("WARNING: Some workers failed to start"),artillery.log(chalk.red(JSON.stringify(runData.failures,null,4))),new Error("Not enough capacity - terminating"));context.taskArns=context.taskArns.concat(runData.tasks.map(task=>task.taskArn)),await context.testRunObject.setTasks([runData.tasks[0].taskArn]),artillery.globalEvents.emit("metadata",{platformMetadata:{taskArns:context.taskArns}})}catch(runErr){throw runErr}return context}async function ecsRunTask(context){var ecs=new AWS.ECS({apiVersion:"2014-11-13",region:context.region});let tasksRemaining=context.count-1,retries=0;for(;0<tasksRemaining&&context.status!==TEST_RUN_STATUS.TERMINATING&&context.status!==TEST_RUN_STATUS.EARLY_STOP;){if(10<=retries)throw artillery.log("Max retries for ECS (10) exceeded"),new Error("Max retries exceeded");var launchCount=tasksRemaining<=10?tasksRemaining:10,params=Object.assign({count:launchCount},JSON.parse(JSON.stringify(context.defaultECSParams)));params.overrides.containerOverrides[0].environment.push({name:"IS_LEADER",value:"false"});try{var uniqueReasons,runData=await ecs.runTask(params).promise();if(0<runData.failures.length)throw artillery.log("Some workers failed to start"),uniqueReasons=[...new Set(runData.failures.map(f=>f.reason))],artillery.log(chalk.red(uniqueReasons)),artillery.log("Retrying..."),await sleep(1e4),new Error("Not enough ECS capacity");0<runData.tasks?.length?(await context.testRunObject.setTasks(runData.tasks.map(x=>x.taskArn)),context.taskArns=context.taskArns.concat(runData.tasks.map(task=>task.taskArn)),debug(`Launched ${launchCount} tasks`),tasksRemaining-=launchCount,await sleep(250)):retries++}catch(runErr){if("ThrottlingException"===runErr.code?(artillery.log("ThrottlingException returned from ECS, retrying"),await sleep(2e3*retries),debug("runTask throttled, retrying"),debug(runErr)):runErr.message.match(/Not enough ECS capacity/gi)||artillery.log(runErr),10<=++retries)throw artillery.log("Max retries for ECS (10) exceeded"),runErr}}return context}async function waitForTasks2(context){var ecs=new AWS.ECS({apiVersion:"2014-11-13",region:context.region}),params={tasks:context.taskArns,cluster:context.clusterName};let failedTasks=[],stoppedTasks=[],maybeErr=null;for(var statusCounts,silentWaitTimeout=new Timeout(3e4).start(),waitTimeout=new Timeout(6e4).start();context.status!==TEST_RUN_STATUS.TERMINATING;){let ecsData;try{ecsData=await awsUtil.ecsDescribeTasks(params,ecs)}catch(err){debug(err),await sleep(5e3);continue}if(_.every(ecsData.tasks,s=>"RUNNING"===s.lastStatus)){logProgress("All workers started..."),debug("All tasks in RUNNING state");break}if(0<(stoppedTasks=ecsData.tasks.filter(t=>"STOPPED"===t.lastStatus)).length){debug("Some tasks in STOPPED state"),debugErr(stoppedTasks),maybeErr=new Error("Worker init failure, aborting test");break}if(0<ecsData.failures.length){failedTasks=ecsData.failures,debug("Some tasks failed to start"),debugErr(ecsData.failures),maybeErr=new Error("Worker start up failure, aborting test");break}if(debug("Waiting on pending tasks"),silentWaitTimeout.timedout()&&(statusCounts=_.countBy(ecsData.tasks,"lastStatus"),statusCounts=_.map(statusCounts,(count,status)=>{status="RUNNING"===status?"ready":status.toLowerCase();let displayStatusChalked=status;return"ready"===status?displayStatusChalked=chalk.green(status):"pending"===status&&(displayStatusChalked=chalk.yellow(status)),displayStatusChalked+": "+count}).join(" / "),moment().format("HH:mm:ss"),logProgress("Waiting for workers to start: "+statusCounts)),waitTimeout.timedout())break;await sleep(1e4)}if(waitTimeout.stop(),maybeErr)throw 0<stoppedTasks.length&&artillery.log(stoppedTasks),0<failedTasks.length&&artillery.log(failedTasks),maybeErr;return context}async function waitForWorkerSync(context){return new Promise((resolve,reject)=>{createS3Client();const prefix=`test-runs/${context.testId}/synced_`;var times=WAIT_TIMEOUT/10;A.retry({times:times,interval:1e4},function(next){util.listAllObjectsWithPrefix(context.s3Bucket,prefix,(err,objects)=>{if(!err)return debug({objects:objects}),objects.length!==context.count?(debug(`expected ${context.count} sync acks but got `+objects.length),next(new Error("Timed out waiting for workers to sync"))):next(null);next(err)})},err=>err?reject(err):(debug("all workers synced"),resolve(context)))})}async function sendGoSignal(context){var s3=createS3Client();try{await s3.putObject({Body:context.testId,Bucket:context.s3Bucket,Key:`test-runs/${context.testId}/go.json`}).promise()}catch(err){throw err}return context}async function listen(context,ee){return new Promise((resolve,reject)=>{context.intermediateReports=[],context.aggregateReport=null;var r=new SqsReporter(context);(context.sqsReporter=r).on("workersDone",state=>(ee.emit("workersDone",state),resolve(context))),r.on("done",stats=>{stats.report?context.aggregateReport=stats.report():context.aggregateReport=stats,global.artillery.globalEvents.emit("done",stats),ee.emit("done",stats)}),r.on("error",err=>{}),r.on("workerDone",(body,attrs)=>{process.env.LOG_WORKER_MESSAGES&&artillery.log(chalk.green(`[${attrs.workerId.StringValue} ${JSON.stringify(body,null,4)}]`))}),r.on("workerError",(body,attrs)=>{process.env.LOG_WORKER_MESSAGES&&artillery.log(chalk.red(`[${attrs.workerId.StringValue} ${JSON.stringify(body,null,4)}]`)),artillery.log(chalk.yellow("Worker exited with an error, worker ID = "+attrs.workerId.StringValue)),global.artillery.suggestedExitCode=body.exitCode||1}),r.on("workerMessage",(body,attrs)=>{if(process.env.LOG_WORKER_MESSAGES&&artillery.log(chalk.yellow(`[${attrs.workerId.StringValue}] ${body.msg} `+body.type)),"stopped"===body.type&&(context.status!==TEST_RUN_STATUS.EARLY_STOP&&artillery.log("Test run has been requested to stop"),context.status=TEST_RUN_STATUS.EARLY_STOP),"ensure"===body.type)try{context.ensureSpec=JSON.parse(util.atob(body.msg))}catch(parseErr){console.error("Error processing ensure directive")}}),r.on("stats",async stats=>{let timestamp,report;stats.report?(report=stats.report(),timestamp=report.timestamp,context.intermediateReports.push(report)):(context.intermediateReports.push(stats),timestamp=Number(stats.period),report=stats),global.artillery.globalEvents.emit("stats",stats),ee.emit("stats",stats),await context.testRunObject.recordIntermediateReport(report,timestamp)}),r.start()})}async function deregisterTaskDefinition(context){if(context.taskDefinitionArn){var ecs=new AWS.ECS({apiVersion:"2014-11-13",region:context.region});try{await ecs.deregisterTaskDefinition({taskDefinition:context.taskDefinitionArn}).promise();debug("Deregistered "+context.taskDefinitionArn)}catch(err){artillery.log(err),debug(err)}return context}}function getLogFilename(output,userDefaultFilenameFormat){let logfile,isDir=!1;if(output)try{isDir=fs.statSync(output).isDirectory()}catch(err){}return!isDir&&output?logfile=output:(isDir||output)&&(logfile=path.join(output,moment().format(userDefaultFilenameFormat||"[artillery_report_]YMMDD_HHmmSS[.json]"))),logfile}