@datawheel/bespoke-cms-warmup/dist/cluster.js

Warmup utility for Bespoke CMS pages

#!/usr/bin/env node
'use strict';var $=(t=>typeof require<"u"?require:typeof Proxy<"u"?new Proxy(t,{get:(e,r)=>(typeof require<"u"?require:e)[r]}):t)(function(t){if(typeof require<"u")return require.apply(this,arguments);throw Error('Dynamic require of "'+t+'" is not supported')});var se=(t,e)=>()=>(t&&(e=t(t=0)),e);var _=(t,e)=>()=>(e||t((e={exports:{}}).exports,e),e.exports);var S=se(()=>{});var K=_((Ae,L)=>{S();var x=$("fs"),D=$("path"),U=class{errorCount;failures;outputPath;successCount;writer;constructor(e){this.resetCounters(),e&&this.setOutput(e);}setOutput(e){let r=D.resolve(process.cwd(),e);x.existsSync(r)||x.mkdirSync(r),process.stdout.write(`Reports will be saved on:
  ${r}
`),this.outputPath=r;let o=D.resolve(r,"results.json");this.writer=x.createWriteStream(o,{flags:"w"});}log(e){this.writer.write(`${e}
`);}close(){return new Promise((e,r)=>{this.writer?(this.writer.end(()=>e()),this.writer.on("error",r)):e();})}writeReport(e){let r=z(this.failures),o=D.resolve(this.outputPath,`profile_${e}.json`);return x.promises.writeFile(o,r)}print(e,r=true){process.stdout.write(`${e}${r?`
`:""}`);}printLine(e="-"){let r=process.stdout.columns*1||80,o=new Array(r).fill(e).join("").slice(0,r);process.stdout.write(`${o}
`);}countResult(e){e.status==="ERROR"&&this.errorCount++,e.status==="SUCCESS"&&this.successCount++,e.status==="FAILURE"&&this.failures.push(e);}resetCounters(){this.errorCount=0,this.failures=[],this.successCount=0;}printCount(e){let r="";this.errorCount>0&&(r=`with ${this.errorCount} errors`),this.failures.length>0&&(r=`${r?", and":"with"} ${this.failures.length} failures`),this.print(`
Profile "${e}" completed ${r||"without errors"}.`);}step(e){return process.stdout.write(`WAIT: ${e}...`),{reject(r){throw process.stdout.write("\r\x1B[K"),process.stdout.write(`ERR : ${e}
`),r},resolve(r){return process.stdout.write("\r\x1B[K"),process.stdout.write(`OK  : ${e}
`),r}}}};function z(t){let e=new Map([["",[""].slice(0,0)]]);e.clear();let r=o=>{let s=e.get(o);if(s!==void 0)return s;let d=[];return e.set(o,d),d};for(let o of t){let{test_na:s}=o.result;if(s&&s.length>0){let{url:d}=o.job;for(let u of s)r(u).push(d);}}return JSON.stringify([...e.entries()],null,2)}function ie(t){return [].concat(t).reduce((e,r)=>{let o=r.indexOf(":"),s=r.substr(0,o).trim();return e[s]=r.substr(o+1).trim(),e},{})}var M=class{constructor(){this.cache=new Map;}getCanonicalKey(e,r){let[o,s]=[e,r].sort((d,u)=>d-u);return `${o}-${s}`}hasBeenProcessed(e,r,o){if(!this.cache.has(e))return  false;let s=this.getCanonicalKey(r,o);return this.cache.get(e).has(s)}markAsProcessed(e,r,o){this.cache.has(e)||this.cache.set(e,new Set);let s=this.getCanonicalKey(r,o);this.cache.get(e).add(s);}};L.exports={Reporter:U,parseHeaders:ie,transposeResults:z,BilateralCache:M};});var W=_((qe,H)=>{S();var q=$("path"),I=__dirname.endsWith("models")?__dirname:q.resolve(__dirname,"models");H.exports={modelPaths:{report:q.resolve(I,"report.js"),dimension:q.resolve(I,"dimension.js"),variant:q.resolve(I,"variant.js"),search:q.resolve(I,"search.js")}};});var B=_((xe,V)=>{S();var{Sequelize:J,DataTypes:j}=$("sequelize");V.exports={hydrateModels:ae};async function ae(t,e){let r=J.Op,o=t.db||"postgresql://".concat(t["db-user"],":",t["db-pass"],"@",t["db-host"],"/",t["db-name"]);e.print(`Database Connection: ${o}`);let s;s=e.step("Creating new sequelize instance");let d=new J(o,{logging:false,pool:{max:10,min:1,acquire:5*1e3,idle:10*1e3,evict:1*1e3}});s.resolve(),s=e.step("Testing connection"),await d.authenticate().then(s.resolve,s.reject),s=e.step("Retrieving database models");let{modelPaths:u}=W(),i=$(u.report)(d,j),h=$(u.dimension)(d,j),n=$(u.variant)(d,j),a=$(u.search)(d,j);return i.associate&&i.associate({dimension:h}),h.associate&&h.associate({report:i,variant:n}),n.associate&&n.associate({dimension:h}),a.associate&&a.associate({dimension:h,report:i,variant:n}),s.resolve(),i.findAllIn=function(c=[],l=[]){let f=e.step("Requesting configured profiles");return i.findAll({where:{id:c&&c.length>0?{[r.in]:c}:{[r.ne]:0},visible:true},include:[{association:"dimensions",separate:true,include:[{association:"variants",separate:true,where:{id:l&&Array.isArray(l)&&l.length>0?{[r.in]:l}:{[r.ne]:0}}}],order:[["ordering","ASC"]]}]}).then(f.resolve,f.reject).then(v=>(c.length>0&&e.print(`User requested for profiles: ${c.join(", ")}`),l.length>0&&e.print(`User requested for variants: ${l.join(", ")}`),v.sort((g,C)=>c.indexOf(g.id)-c.indexOf(C.id)).map(g=>(e.print(`Profile found: ${g.id} ${g.name}`),g.dimensions.forEach(C=>{e.print(`  Dimension found: ${C.id} ${C.name}`),C.variants.forEach(A=>{e.print(`    Variant found: ${A.id} ${A.name}`);});}),g))))},a.findAllFromProfile=function(c,l){let f={};return l&&Array.isArray(l)&&l.length>0&&(f.variant_id={[r.in]:l}),a.findAll({where:{report_id:c.id,visible:true,...f},order:[["zvalue","DESC"]]})},a.findAllMembers=function(c,l,f){return a.findAll({where:{report_id:c,dimension_id:l,variant_id:f,visible:true},attributes:["id","slug"],raw:true})},a.findAllFromBilateralPair=function(c,l,f,v,g,C,A){return a.findAll({where:{report_id:c,[r.or]:[{dimension_id:l,variant_id:f,id:v},{dimension_id:g,variant_id:C,id:A}],visible:true},order:[["zvalue","DESC"]]})},{Report:i,Search:a}}});var X=_((je,G)=>{S();var{default:oe}=$("axios");G.exports={naApiTest:ue,naTestHandler:ce};async function ce(t){return t.evaluate(e);function e(){let r=document.querySelector("#bespoke-report"),o=document.createTreeWalker(r,NodeFilter.SHOW_TEXT,{acceptNode:i=>/\bN\/A\b|undefined/g.test(i.wholeText)?NodeFilter.FILTER_ACCEPT:NodeFilter.FILTER_SKIP}),s=[];for(;;){let i=o.nextNode();if(!i)break;s.push(d(i.parentNode));}return s;function d(i){let h,n,a;if(i.id&&(a=`#${CSS.escape(i.id)}`,document.querySelectorAll(a).length===1))return a;let c=i.localName;if(i.classList.length>0){for(let l=0;l<i.classList.length;l++)if(a=`.${CSS.escape(i.classList.item(l))}`,n=document.querySelectorAll(a),n.length===1||(a=c+a,n=document.querySelectorAll(a),n.length===1)||(h=u(i)+1,a=`${a}:nth-child(${h})`,n=document.querySelectorAll(a),n.length===1))return a}return i.parentNode!==document&&(h=u(i)+1,a=`${d(i.parentNode)} > ${c}:nth-child(${h})`),a}function u(i){return [...i.parentNode.children].indexOf(i)}}}async function ue(t,e,r,o){let s=await oe.get({baseURL:t,url:"api/profile/",params:{slug:e,id:r,locale:o}});return {url:s.config.url,paths:d([],s.data)};function d(u,i){let h=[],n=Object.keys(i);for(let a of n){let c=i[a];if(Array.isArray(c)||typeof c=="object"){let l=d(u.concat(a),c);h.push(...l);continue}if(/\bN\/A\b/g.test(`${c}`)){let l=u.concat(a).map(f=>f==parseInt(f,10)?`[${f}]`:`["${f}"]`).join("");h.push(l);}}return h}}});var be=_((Oe,Q)=>{S();var le=$("fs"),de=$("path"),fe=$("readline"),{Cluster:Y}=$("puppeteer-cluster"),{Reporter:he,parseHeaders:pe,BilateralCache:me}=K(),{hydrateModels:we}=B(),{naTestHandler:ge}=X(),p=new he;Q.exports=async function(t){p.setOutput(t.output),p.printLine();let e=t.username&&t.password?{username:t.username,password:t.password}:void 0,r=pe(t.header),o=parseInt(t.timeout,10)*1e3,s=await Y.launch({concurrency:Y.CONCURRENCY_PAGE,maxConcurrency:parseInt(t.workers,10),monitor:true,timeout:o});s.on("taskerror",(d,u)=>{let i={status:"ERROR",error:d.message,job:u};p.countResult(i),p.log(JSON.stringify(i));}),await s.task(async d=>{let{page:u,data:i}=d;await u.setDefaultNavigationTimeout(o),await u.setExtraHTTPHeaders(r),await u.setViewport({width:1280,height:720}),e&&await u.authenticate(e),await u.goto(i.url,{waitUntil:"load",timeout:o}),await u.waitForSelector("#bespoke-report",{timeout:o});let h=await ge(u).then(n=>({status:n.length===0?"SUCCESS":"FAILURE",result:{test_na:n},job:i}),n=>({status:"ERROR",error:n.message,job:i}));p.countResult(h),h.status!=="SUCCESS"&&p.log(JSON.stringify(h));}),t.input?await $e(t,s):await ve(t,s),await s.close();};async function ve(t,e){let r=t.base||"",{Report:o,Search:s}=await we(t,p),d=t.profile?`${t.profile}`.split(",").filter(n=>n.trim()):[],u=t.variant?`${t.variant}`.split(",").filter(n=>n.trim()):[],i=await o.findAllIn(d,u),h=new me;for(let n of i)if(p.printLine(),p.print(`Starting scan of profile ID ${n.id}: "${n.name}"`),n.dimensions.length===1){let a=await s.findAllFromProfile(n,u);for(let c of a)if(n&&n.dimensions[0]&&n.dimensions[0].variants.length>0){let l=n.dimensions[0].variants.find(v=>v.id===c.variant_id),f={locale:t.locale,page:c.slug,profile:l.slug};f.url=r.replace(/:(\D\w*)\b/g,(v,g)=>f[g]||`:${g}`),e.queue(f);}await e.idle(),p.writeReport(n.name),p.printCount(n.name),p.resetCounters();}else if(n.dimensions.length>=2){let a=n.dimensions[0],c;if(t["variant1-id"]){if(c=a.variants.find(m=>m.id===parseInt(t["variant1-id"],10)),!c){p.print(`  Variant ID ${t["variant1-id"]} not found for dimension ${a.name}. Skipping.`);continue}}else c=a.variants[0];if(!c)continue;let l=await s.findAllMembers(n.id,a.id,c.id),f=n.dimensions[1],v;if(t["variant2-id"]){if(v=f.variants.find(m=>m.id===parseInt(t["variant2-id"],10)),!v){p.print(`  Variant ID ${t["variant2-id"]} not found for dimension ${f.name}. Skipping.`);continue}}else v=f.variants[0];if(!v)continue;let g=await s.findAllMembers(n.id,f.id,v.id),C=l.length*g.length,A=await s.findAll({where:{report_id:n.id,dimension_id:a.id,variant_id:c.id,visible:true}}),Z=await s.findAll({where:{report_id:n.id,dimension_id:f.id,variant_id:v.id,visible:true}}),P=t["random-pairs"],ee=P?l.sort(()=>Math.random()-.5):l.map(m=>{var w;return {...m,zvalue:((w=A.find(b=>b.id===m.id))==null?void 0:w.zvalue)||0}}).sort((m,w)=>(w.zvalue||0)-(m.zvalue||0)),te=P?g.sort(()=>Math.random()-.5):g.map(m=>{var w;return {...m,zvalue:((w=Z.find(b=>b.id===m.id))==null?void 0:w.zvalue)||0}}).sort((m,w)=>(w.zvalue||0)-(m.zvalue||0)),y=[],k=new Set;for(let m of ee)for(let w of te){if(m.id===w.id)continue;let b=h.getCanonicalKey(m.id,w.id);k.has(b)||(k.add(b),y.push({member1:m,member2:w,canonicalKey:b}));}let E=y,O="",R=t["limit-pairs"]?parseInt(t["limit-pairs"],10):null;R&&R>0&&y.length>R&&(P?(E=y.sort(()=>Math.random()-.5).slice(0,R),O=`(${R} random of ${y.length})`):(E=y.slice(0,R),O=`(top ${R} of ${y.length})`));let re=P?"random":"importance";p.print(`Processing bilateral: ${a.name}(${l.length})\xD7${f.name}(${g.length}) \u2192 ${C} pairs ${O}, ordered by ${re}`);for(let m of E){let{member1:w,member2:b,canonicalKey:Se}=m;if(h.hasBeenProcessed(n.id,w.id,b.id))continue;h.markAsProcessed(n.id,w.id,b.id);let T={locale:t.locale,variant1:c.slug,profile1:w.slug,variant2:v.slug,profile2:b.slug};T.url=r.replace(/:(\D\w*)\b/g,(Ce,F)=>T[F]||`:${F}`),e.queue(T);}await e.idle(),p.printCount(n.name),p.writeReport(n.name),p.resetCounters();}}async function $e(t,e){let r=t.base||"",o=de.resolve(process.cwd(),t.input);p.print(`Retrying warming errors from file:
  ${o}`);let s=fe.createInterface({crlfDelay:1/0,input:le.createReadStream(o)});for await(let d of s){let u=JSON.parse(d);if(u.status==="SUCCESS")continue;let i=u.job;i.url=r.replace(/:(\w+)\b/g,(h,n)=>i[n]),e.queue(i);}await e.idle(),p.writeReport("retry"),p.printCount("retry"),p.resetCounters();}});var cluster = be();module.exports=cluster;