@artilleryio/platform-fargate/lib/data-api/store.js

Fargate support for Artillery

const debug=require("debug")("store"),_=require("lodash"),{listAllObjectsWithPrefix,getBucketName,tagsFilterQueryParams,statusFilterQueryParams,defaultQueryParams,queryByTags,queryByStatus}=require("../util"),promisify=require("util")["promisify"],ObjectStore=require("../cloud/object-store")["ObjectStore"],util=require("util"),uuidv4=require("uuid/v4"),getBackendId=require("../utils/get-backend-id"),TestNotFoundError=require("../errors")["TestNotFoundError"],AWS=require("aws-sdk"),getrc=require("../utils/get-rc"),{getAccountId,getTestRunsTableName}=require("../util"),createDynamoDocumentClient=require("../utils/create-dynamo-document-client")["createDynamoDocumentClient"],writeInBatches=require("./write-in-batches")["writeInBatches"],AuroraV1Store=require("./aurora-serverless")["AuroraV1Store"];class DynamoStore{constructor(opts){return this}async init(){var backendId=(await getBackendId())["backendId"],storedTableName=(this.region=backendId,this.tableName||(backendId=await getAccountId(),storedTableName=await getTestRunsTableName(this.region),this.tableName=process.env.ARTILLERY_TEST_RUNS_TABLE||storedTableName||"artilleryio-test-runs-"+backendId,this.commonObjectsTable="artilleryio-common-objects-"+backendId),AWS.config.update({region:this.region}),this.ddb=createDynamoDocumentClient(),this.objectPrefix="data-"+this.region,await getBucketName());return this.objectStore=new ObjectStore({backend:"aws",bucket:storedTableName}),this}async getRunningTests(){try{var statuses=["INITIALIZING","LAUNCHING_WORKERS","RECEIVING_REPORTS"],params={TableName:this.tableName,IndexName:"test-runs-laststatus-index",KeyConditions:{statusName:{ComparisonOperator:"EQ",AttributeValueList:[]}}};let items=[];for(const s of statuses){params.KeyConditions.statusName.AttributeValueList=[s];var data=await this.ddb.query(params).promise();items=items.concat(data.Items)}var result={};for(const item of items){var testRunId=item.testRunId,testRunData=await this.getTestRun(testRunId);result[testRunId]={testId:testRunId,region:testRunData.metadata.region,cluster:testRunData.metadata.cluster,launchType:testRunData.metadata.launchType,count:testRunData.metadata.count,startedAt:testRunData.metadata.startedAt,lastStatus:item.statusName}}return result}catch(err){throw err}}async listTestRuns({tags,limit=10,offset,status,created}){try{if(0<tags.length){let data;const params=tagsFilterQueryParams({tags:tags,tableName:this.tableName,indexName:"test-runs-kind-index",offset:offset});return data=await queryByTags({ddb:this.ddb,created:created,queryParams:params,status:status,limit:limit}),1===tags.length&&0===data.Items.length&&(params.KeyConditions.kind.AttributeValueList[0]=`tag#${tags[0].name}:`+tags[0].value,data=await queryByTags({ddb:this.ddb,created:created,queryParams:params,status:status,limit:limit})),{items:data.Items.map(item=>({id:item.testRunId,endedAt:Number(new Date(item.createdTime))})),nextOffset:data.LastEvaluatedKey}}if(status){const params=statusFilterQueryParams({tableName:this.tableName,indexName:"test-runs-laststatus-index",status:status,offset:offset}),data=await queryByStatus({ddb:this.ddb,created:created,queryParams:params,status:status,limit:limit});return debug(data),{items:data.Items.map(item=>({id:item.testRunId,endedAt:Number(new Date(item.endTime))})),nextOffset:data.LastEvaluatedKey}}var params=defaultQueryParams({tableName:this.tableName,indexName:"test-runs-createdtime-index",created:created,limit:limit,offset:offset}),data=await this.ddb.query(params).promise();return debug(data),{items:data.Items.map(item=>({id:item.testRunId,endedAt:Number(new Date(item.createdTime))})),nextOffset:data.LastEvaluatedKey}}catch(err){throw err}}async deleteTestRun(testRunId){var params={TableName:this.tableName,KeyConditions:{testRunId:{ComparisonOperator:"EQ",AttributeValueList:[testRunId]}}},params=await this.ddb.query(params).promise();if(0===params.Items.length)throw new TestNotFoundError(`Test run ${testRunId} could not be found`);await writeInBatches(this.ddb,params.Items,batch=>{var batchWriteRequest={RequestItems:{}};return batchWriteRequest.RequestItems[this.tableName]=batch.map(item=>({DeleteRequest:{Key:{testRunId:item.testRunId,kind:item.kind}}})),batchWriteRequest});try{await Promise.all([this.objectStore.del(this.objectPrefix+`/${testRunId}/console-log.json`),this.objectStore.del(this.objectPrefix+`/${testRunId}/aggregate.json`)])}catch(s3Err){debug(s3Err)}}async getTestRun(testRunId){debug("store getTestRun",testRunId);var result={metadata:{},tasks:[],report:{intermediate:[],aggregate:{}},consoleLog:null,status:null,notes:null},params={TableName:this.tableName,KeyConditions:{testRunId:{ComparisonOperator:"EQ",AttributeValueList:[testRunId]}}};const data=await this.ddb.query(params).promise(),itemByKind={};for(const item of data.Items){var kind=item["kind"];itemByKind[kind]=item}void 0!==itemByKind["metadata#testId"]&&itemByKind["metadata#testId"].stringValue,result.metadata={testId:itemByKind["metadata#testId"].stringValue,startedAt:itemByKind["metadata#startedAt"]?.numberValue,cluster:itemByKind["metadata#cluster"]?.stringValue,region:itemByKind["metadata#region"]?.stringValue,launchType:itemByKind["metadata#launchType"]?.stringValue,count:itemByKind["metadata#count"]?.numberValue,tags:itemByKind.tags?itemByKind.tags.tags.split(",").map(x=>({name:x.split(":")[0],value:x.split(":")[1]})):[],cliArgs:itemByKind["metadata#cliArgs"]?.stringValue||null,secrets:JSON.parse(itemByKind["metadata#secrets"]?.stringValue||null),platformConfig:JSON.parse(itemByKind["metadata#platformConfig"]?.stringValue||null),artilleryVersion:JSON.parse(itemByKind["metadata#artilleryVersion"]?.stringValue||null)},result.tasks=Object.keys(itemByKind).filter(k=>k.startsWith("metadata#task#")).map(k=>itemByKind[k].value),result.report.intermediate=[].concat(Object.keys(itemByKind).filter(k=>k.startsWith("report#intermediate#")).map(k=>itemByKind[k].data).map(s=>JSON.parse(s)));params=Object.keys(itemByKind).filter(k=>k.startsWith("report#aggregate")).sort();let aggregateReportJson;if(1===params.length&&itemByKind[params[0]].data?.startsWith("s3://")){debug("aggregate field is in S3");var objectPath=this.objectPrefix+`/${testRunId}/aggregate.json`;aggregateReportJson=await this.objectStore.get(objectPath,{json:!0})}else{debug("aggregate field is in Dynamo (legacy)");const data=params.map(k=>itemByKind[k].data).join("");debug({aggregateKeys:params,data:data}),""!==data&&(aggregateReportJson=JSON.parse(data))}result.report.aggregate=aggregateReportJson;objectPath=Object.keys(itemByKind).filter(k=>k.startsWith("consoleOutput#")).sort();if(debug({consoleLogKeys:objectPath}),1===objectPath.length&&itemByKind[objectPath[0]].data?.startsWith("s3://")){debug("text log is in S3");const objectPath=this.objectPrefix+`/${testRunId}/console-log.json`;result.consoleLog=(await this.objectStore.get(objectPath))?.Body.toString("utf8")}else if(debug("text log is in Dynamo (legacy)"),0<objectPath.length){let textLog="";for(const args of objectPath.map(k=>JSON.parse(itemByKind[k].data))[0])textLog+=util.format(...Object.keys(args).map(k=>args[k]))+"\n";result.consoleLog=textLog}return result.status={endedAt:Number(new Date(itemByKind.lastStatus?.endTime))||void 0,status:itemByKind.lastStatus?.statusName||void 0},result.notes=await this.getNotes(testRunId),result}async listTags(){var params={TableName:this.tableName,KeyConditions:{testRunId:{ComparisonOperator:"EQ",AttributeValueList:["__artilleryio:virtual:key"]},kind:{ComparisonOperator:"BEGINS_WITH",AttributeValueList:["tag#"]}}};return(await this.ddb.query(params).promise()).Items.map(x=>({name:x.name,value:x.stringValue}))}async addNote(testRunId,note){var noteId=uuidv4(),ts=Date.now();return await this.ddb.put({TableName:this.tableName,Item:{testRunId:testRunId,kind:"note#"+ts,noteId:noteId,createdTime:new Date(ts).toISOString(),data:note}}).promise(),noteId}async getNotes(testRunId){return(await this.ddb.query({TableName:this.tableName,KeyConditionExpression:"testRunId = :id AND begins_with(kind, :prefix)",ExpressionAttributeValues:{":id":testRunId,":prefix":"note#"},ScanIndexForward:!1,Limit:10}).promise()).Items.map(x=>({createdOn:Number(new Date(x.createdTime)),id:x.noteId,text:x.data,userId:x.userId}))}async checkMigrations(){return[]}}class Store{constructor(opts){return opts&&"aws:aurora-v1"===opts.type?this.backend=new AuroraV1Store(opts):opts&&"cloud"===opts.type?this.backend=new CloudStore(opts):this.backend=new DynamoStore,this}async init(){return this.backend.init()}async listTestRuns(tags,limit,offset,status,created){return this.backend.listTestRuns({tags:tags,limit:limit,offset:offset,status:status,created:created})}async getTestRun(testRunId){return this.backend.getTestRun(testRunId)}async deleteTestRun(testRunId){return this.backend.deleteTestRun(testRunId)}async listTags(){return this.backend.listTags()}async addNote(testRunId,text,opts){return this.backend.addNote(testRunId,text,opts)}async putNote(testRunId,noteId,userId,text,opts){return this.backend.putNote(testRunId,noteId,userId,text,opts)}async deleteNote(testRunId,noteId,userId,opts){return this.backend.deleteNote(testRunId,noteId,userId,opts)}async getNotes(testRunId){return this.backend.getNotes(testRunId)}async getRunningTests(){return this.backend.getRunningTests()}async checkMigrations(){return this.backend.checkMigrations()}async createUser(providerId,accountId,username,email,image,opts){return this.backend.createUser(providerId,accountId,username,email,image,opts)}async getUser(providerId,accountId,opts){return this.backend.getUser(providerId,accountId,opts)}}let store;function getOrCreateStore(backend){return new Store(backend)}module.exports=getOrCreateStore;