@artilleryio/platform-fargate/lib/data-api/aurora-serverless/index.js

Fargate support for Artillery

const DataSource=require("typeorm")["DataSource"],{TestRun,Tag,User,Provider,ProviderAccount}=require("./models"),{TestRunSchema,TagSchema,UserSchema,ProviderSchema,ProviderAccountSchema}=require("./entities"),ConsoleOutputSerializeError=require("../../errors")["ConsoleOutputSerializeError"],PromisePool=require("@supercharge/promise-pool")["PromisePool"],ObjectStore=require("../../cloud/object-store")["ObjectStore"],{getBucketName,listAllObjectsWithPrefix}=require("../../util"),debug=require("debug")("store"),debugPerf=require("debug")("perf"),util=require("util"),uuidv4=require("uuid/v4"),migrationNames=require("./migrations")["migrationNames"],TEST_RUN_STATUS=require("../../test-run-status");class StorageBackendAuroraV1{static properties={type:"aws:aurora-v1",description:"AWS Aurora v1 Serverless backend"};constructor(id,opts){return this.id=id,this.secretArn=opts.config.secretArn,this.resourceArn=opts.config.resourceArn,this.engine="pg",this.region=opts.config.region,this.database=opts.config.database,this.synchronize=void 0!==process.env.SYNCH,this.logging=void 0!==process.env.LOG_SQL,this}async init(id,opts={}){this.conn=new DataSource({type:"aurora-postgres",secretArn:this.secretArn,resourceArn:this.resourceArn,region:this.region,engine:"pg",database:this.database,entities:[TestRunSchema,TagSchema,UserSchema,ProviderSchema,ProviderAccountSchema],synchronize:this.synchronize,logging:this.logging}),await this.conn.initialize(),this.repo=this.conn.getRepository(TestRun),opts.testRunId?await this.load(opts.testRunId):await this.create(),this.objectPrefix="data-"+this.region;opts=await getBucketName();return this.objectStore=new ObjectStore({backend:"aws",bucket:opts}),this}async create(){var t=new TestRun(this.id);return this.dbObject=await this.repo.save(t),this}async load(testRunId){return this.dbObject=await this.repo.findOneBy({testRunId:testRunId}),this}async setEndedAt(ts){return this.dbObject.endedAt=new Date(ts),await this.repo.save(this.dbObject),this}async setChecks(checks){return this.dbObject.checks=checks.map(c=>({condition:c.original,success:1===c.result,strict:c.strict})),await this.repo.save(this.dbObject),this}async bulkInsertTags(tags){await this.conn.createQueryBuilder().insert().into(Tag).values(tags.map(t=>({name:t.name,value:t.value,tagString:t.name+":"+t.value}))).execute()}async setTags(tags){var tagsObject={};for(const tag of tags)tagsObject[tag.name]=tag.value;this.dbObject.tags=tagsObject,await this.repo.save(this.dbObject);for(const t of tags){var tag=new Tag(t.name,t.value);try{await this.conn.getRepository(Tag).save(tag)}catch(err){}}return this}async setReport({aggregate}){var objectPath;return aggregate&&(aggregate=JSON.stringify(aggregate),objectPath=`${this.objectPrefix}/${this.id}/aggregate.json`,await this.objectStore.put(objectPath,aggregate),this.dbObject.aggregateMetricReport="s3://"+objectPath,await this.repo.save(this.dbObject)),this}async setTextLog(lines){try{JSON.stringify(lines)}catch(stringifyErr){throw new ConsoleOutputSerializeError("Failed trying to serialize console output")}let textLog="";for(const args of lines)textLog+=util.format(...Object.keys(args).map(k=>args[k]))+"\n";lines=`${this.objectPrefix}/${this.id}/console-log.json`;return await this.objectStore.put(lines,textLog),this.dbObject.consoleLog="s3://"+lines,await this.repo.save(this.dbObject),this}async setStatus(status){return this.dbObject.status=status,await this.repo.save(this.dbObject),this}async setMetadata(metadataObj){return this.dbObject.metadata=metadataObj,this.dbObject.startedAt=new Date(metadataObj.startedAt),this.dbObject.createdTime=new Date(metadataObj.startedAt),await this.repo.save(this.dbObject),this}async setTasks(taskArns){return this.dbObject.tasks=(this.dbObject.tasks||[]).concat(taskArns),await this.repo.save(this.dbObject),this}async recordIntermediateReport(report,ts){var objectPath=`${this.objectPrefix}/${this.id}/intermediate/${ts}.json`;await this.objectStore.put(objectPath,JSON.stringify(report)),this.dbObject.intermediateMetricReports||(this.dbObject.intermediateMetricReports=[],await this.repo.save(this.dbObject)),this.dbObject.intermediateMetricReports.push(ts),await this.repo.save(this.dbObject)}async addNote(text){this.dbObject.notes||(this.dbObject.notes=[],await this.repo.save(this.dbObject));text={id:uuidv4(),data:text,createdTime:Date.now()};return this.dbObject.notes.push(text),await this.repo.save(this.dbObject),this}async setTagsSearchStrings(tags,additionalAttributes){return this}}class AuroraV1Store{constructor(opts){return this.secretArn=opts.config.secretArn,this.resourceArn=opts.config.resourceArn,this.engine="pg",this.region=opts.config.region,this.database=opts.config.database,this.synchronize=void 0!==process.env.SYNCH,this.logging=void 0!==process.env.LOG_SQL,this.opts=opts,this}async init(){this.conn=new DataSource({type:"aurora-postgres",secretArn:this.secretArn,resourceArn:this.resourceArn,region:this.region,engine:this.engine,database:this.database,entities:[TestRunSchema,TagSchema,UserSchema,ProviderSchema,ProviderAccountSchema],synchronize:this.synchronize,logging:this.logging}),await this.conn.initialize();var migrationsErr=new Error("Database schema needs to be upgraded. Please run artillery aws:run-db-migrations");try{if(0<(await this.checkMigrations()).length)throw migrationsErr}catch(err){if("BadRequestException"===err.code||-1<err.message.indexOf("migrations"))throw migrationsErr}this.objectPrefix="data-"+this.region;migrationsErr=await getBucketName();return this.objectStore=new ObjectStore({backend:"aws",bucket:migrationsErr}),this.repo=this.conn.getRepository(TestRun),this}async getRunningTests(){let results=[];let offset=0;for(;;){var[inProgressTests,count]=await this.conn.getRepository(TestRun).createQueryBuilder("testrun").select().where("testrun.status IN (:...statuses)",{statuses:["INITIALIZING","RECEIVING_REPORTS","LAUNCHING_WORKERS","TERMINATING"]}).limit(10).offset(offset).getManyAndCount();if(results=results.concat(inProgressTests),count<=offset)break;offset+=10}return results}async listTestRuns({tags,limit=10,offset=0,status,created}){"object"==typeof offset&&offset.testRunId&&(offset=parseInt(offset.testRunId,10));var qb=await this.conn.getRepository(TestRun).createQueryBuilder("testrun").select().where("testrun.testRunId IS NOT NULL");let parameters={};var tagsKv={};for(const tag of tags)tagsKv[tag.name]=tag.value;0<tags.length&&(qb.andWhere(this._createQueryForTags("tags",tagsKv)),parameters=Object.assign({},parameters,tagsKv)),status&&qb.andWhere("status = :status",{status:status}),created&&(qb.andWhere("testrun.startedAt >= :startedAt").andWhere("testrun.endedAt <= :endedAt"),parameters=Object.assign({},parameters,{startedAt:new Date(created.start),endedAt:new Date(created.end)})),qb.setParameters(parameters);var[tags,status]=await qb.skip(offset).take(limit).orderBy("testrun.createdTime","DESC").getManyAndCount(),created={items:tags.map(r=>{var item={id:r.testRunId,endedAt:Number(r.endedAt),metadata:r.metadata||{},status:{endedAt:Number(r.endedAt),status:r.status}};return item.metadata.platformConfig=r.metadata?.platformConfig?JSON.parse(r.metadata.platformConfig):{},item.metadata.artilleryVersion=r.metadata?.artilleryVersion?JSON.parse(r.metadata.artilleryVersion):{},item.metadata.secrets=r.metadata?.secrets?JSON.parse(r.metadata.secrets):[],item.metadata.testId=r.testRunId,item})};return offset+limit<status&&(created.nextOffset=offset+limit),created}_createQueryForTags(field,tags){return Object.keys(tags).map(tagName=>field+`->>'${tagName}' = :`+tagName).join(" AND ")}async deleteTestRun(testRunId){return this.repo.delete({testRunId:testRunId})}async getTestRun(testRunId){var _ts,ix,errors,result={metadata:{},tasks:[],report:{intermediate:[],aggregate:{}},consoleLog:null,status:null,notes:[],checks:[]},t=await this.repo.findOneBy({testRunId:testRunId});if(!t)throw new Error("Test run not found: "+testRunId);result.testId=testRunId,result.tasks=t.tasks||[],result.status={endedAt:Number(t.endedAt),status:t.status},result.metadata=t.metadata||{},result.checks=t.checks,result.metadata.platformConfig=t.metadata?.platformConfig?JSON.parse(t.metadata.platformConfig):{},result.metadata.artilleryVersion=t.metadata?.artilleryVersion?JSON.parse(t.metadata.artilleryVersion):{},result.metadata.secrets=t.metadata?.secrets?JSON.parse(t.metadata.secrets):[];try{var objectPath=this.objectPrefix+`/${testRunId}/aggregate.json`;result.report.aggregate=await this.objectStore.get(objectPath,{json:!0})}catch(s3Err){}delete result.report.aggregate.histograms;for([_ts,ix]of Object.entries(result.report.intermediate))for(const k of Object.keys(ix))-1===["counters","rates","summaries","period"].indexOf(k)&&delete ix[k];try{const objectPath=this.objectPrefix+`/${testRunId}/console-log.json`;result.consoleLog=(await this.objectStore.get(objectPath))?.Body.toString("utf8")}catch(s3Err){}let intermediateMetrics,alreadyCombined=!(result.report.intermediate=[]);if(0<t.intermediateMetricReports?.length){debugPerf("getting intermediate metric reports");try{const objectPath=this.objectPrefix+`/${testRunId}/intermediates_combined.json`;intermediateMetrics=await this.objectStore.get(objectPath,{json:!0}),alreadyCombined=!0,debugPerf("intermediate metric reports already combined")}catch(s3Err){}if(Array.isArray(intermediateMetrics)||(debugPerf("fetching individual intermediate metric reports"),{results:objectPath,errors}=await PromisePool.withConcurrency(100).for(t.intermediateMetricReports).process(async(ts,_idx,_pool)=>{var objectPath=this.objectPrefix+`/${testRunId}/intermediate/${ts}.json`,objectPath=await this.objectStore.get(objectPath,{json:!0});return delete objectPath.histograms,{ts:ts,report:objectPath}}),objectPath=objectPath.sort((a,b)=>a.ts-b.ts),intermediateMetrics=objectPath.map(r=>r.report)),!alreadyCombined&&("EARLY_STOP"===t.status||"ERROR"===t.status||"COMPLETED"===t.status)){const objectPath=this.objectPrefix+`/${testRunId}/intermediates_combined.json`;await this.objectStore.put(objectPath,JSON.stringify(intermediateMetrics)),debugPerf("wrote combined intermediates object")}}if(debugPerf("got intermediate metric reports"),result.report.intermediate=intermediateMetrics,t.notes)for(const note of t.notes)result.notes.push({id:note.id,createdOn:note.createdTime,text:note.data,user:void 0!==note.userId?await this.getUserById(note.userId):void 0});return result}async getUserById(userId){return this.conn.getRepository(User).findOneBy({userId:userId})}async listTags(pattern){debugPerf("listTags start");let results=[],limit=1e3,offset=0,errorCount=0;const qb=this.conn.getRepository(Tag).createQueryBuilder("tag");for(debugPerf("listTags loop start");;){let tags=[],count=0;try{[tags,count]=pattern?await qb.select().where("tag.tagString like :tagStringPattern",{tagStringPattern:`%${pattern}%`}).limit(limit).offset(offset).getManyAndCount():await qb.select().where("id IS NOT NULL").andWhere("name <> 'test_run_id'").limit(limit).offset(offset).getManyAndCount()}catch(err){if(limit=Math.max(100,parseInt(limit/2,10)),10<errorCount++)throw err;continue}if(results=results.concat(...new Set(tags.map(t=>({name:t.name,value:t.value})))),0===tags.length)break;offset+=limit}if(debugPerf("listTags loop end"),!pattern){debugPerf("listTags test_run_id lookup start");const qb=await this.conn.getRepository(TestRun).createQueryBuilder("testrun").select("testrun.testRunId").where("testrun.testRunId IS NOT NULL");var[testRunIds,,]=await qb.take(250).orderBy("testrun.createdTime","DESC").getManyAndCount();results=results.concat((testRunIds||[]).map(t=>({name:"test_run_id",value:t.testRunId}))),debugPerf("listTags test_run_id lookup end")}return[...new Set(results)]}async addNote(testRunId,text,userId){var testRunId=await this.repo.findOneBy({testRunId:testRunId}),noteId=(testRunId.notes||(testRunId.notes=[]),uuidv4()),text={id:noteId,data:text,createdTime:Date.now(),userId:userId};return testRunId.notes.push(text),await this.repo.save(testRunId),noteId}async putNote(testRunId,noteId,text,userId){var testRunId=await this.repo.findOneBy({testRunId:testRunId}),noteToUpdate=testRunId.notes.find(n=>n.id===noteId);if(void 0===noteToUpdate)throw new Error("Note not found: "+noteId);if(noteToUpdate.userId!==userId)throw new Error(`Only the owner of note ${noteId} can modify it`);return noteToUpdate.data=text,await this.repo.save(testRunId),noteId}async deleteNote(testRunId,noteId,userId){var testRunId=await this.repo.findOneBy({testRunId:testRunId}),indexToDelete=testRunId.notes.findIndex(n=>n.id===noteId);if(-1===indexToDelete)throw new Error("Note not found: "+noteId);if(testRunId.notes[indexToDelete].userId!==userId)throw new Error(`Only the owner of note ${noteId} can delete it`);return testRunId.notes.splice(indexToDelete,1),await this.repo.save(testRunId),noteId}async getNotes(testRunId){return(await this.getTestRun(testRunId)).notes}async createUser(providerId,accountId,username,email,image){if(!this.conn.getRepository(Provider).findOneBy({providerId:providerId}))throw new Error(`Provider ${providerId} not found`);var providerAccount,existingUser=await this.conn.getRepository(User).findOneBy({email:email});if(existingUser)return providerAccount=new ProviderAccount(existingUser.userId,providerId,accountId),await this.conn.getRepository(ProviderAccount).save(providerAccount),existingUser.userId;{existingUser=uuidv4();const providerAccount=new ProviderAccount(existingUser,providerId,accountId),user=new User(existingUser,username,email,image);return await this.conn.manager.transaction(async transactionalEntityManager=>{await transactionalEntityManager.save(user),await transactionalEntityManager.save(providerAccount)}),existingUser}}async getUser(providerId,accountId){var providerId=await this.conn.getRepository(Provider).findOneBy({providerId:providerId});return(providerId=providerId&&await this.conn.getRepository(ProviderAccount).findOneBy({providerId:providerId.providerId,accountId:accountId}))?this.conn.getRepository(User).findOneBy({userId:providerId.userId}):null}async checkMigrations(){var qr=await this.conn.createQueryRunner();try{const executedMigrations=(await qr.manager.query("SELECT * FROM migrations")).records.map(r=>r.name);return migrationNames.filter(x=>!executedMigrations.includes(x))}catch(err){return debug(err),migrationNames}}}module.exports={AuroraV1Store:AuroraV1Store,StorageBackendAuroraV1:StorageBackendAuroraV1};