@datawheel/bespoke-cms-warmup/dist/cluster.js

Warmup utility for Bespoke CMS pages

#!/usr/bin/env node
'use strict';

var h=(t=>typeof require<"u"?require:typeof Proxy<"u"?new Proxy(t,{get:(e,r)=>(typeof require<"u"?require:e)[r]}):t)(function(t){if(typeof require<"u")return require.apply(this,arguments);throw new Error('Dynamic require of "'+t+'" is not supported')});var H=(t,e)=>()=>(t&&(e=t(t=0)),e);var $=(t,e)=>()=>(e||t((e={exports:{}}).exports,e),e.exports);var w=H(()=>{});var I=$((ie,E)=>{w();var C=h("fs"),A=h("path"),x=class{errorCount;failures;outputPath;successCount;writer;constructor(e){this.resetCounters(),e&&this.setOutput(e);}setOutput(e){let r=A.resolve(process.cwd(),e);C.existsSync(r)||C.mkdirSync(r),process.stdout.write(`Reports will be saved on:
  ${r}
`),this.outputPath=r;let o=A.resolve(r,"results.json");this.writer=C.createWriteStream(o,{flags:"w"});}log(e){this.writer.write(`${e}
`);}writeReport(e){let r=j(this.failures),o=A.resolve(this.outputPath,`profile_${e}.json`);return C.promises.writeFile(o,r)}print(e,r=!0){process.stdout.write(`${e}${r?`
`:""}`);}printLine(e="-"){let r=process.stdout.columns*1||80,o=new Array(r).fill(e).join("").slice(0,r);process.stdout.write(`${o}
`);}countResult(e){e.status==="ERROR"&&this.errorCount++,e.status==="SUCCESS"&&this.successCount++,e.status==="FAILURE"&&this.failures.push(e);}resetCounters(){this.errorCount=0,this.failures=[],this.successCount=0;}printCount(e){let r="";this.errorCount>0&&(r=`with ${this.errorCount} errors`),this.failures.length>0&&(r=`${r?", and":"with"} ${this.failures.length} failures`),this.print(`
Profile "${e}" completed ${r||"without errors"}.`);}step(e){return process.stdout.write(`WAIT: ${e}...`),{reject(r){throw process.stdout.write("\r\x1B[K"),process.stdout.write(`ERR : ${e}
`),r},resolve(r){return process.stdout.write("\r\x1B[K"),process.stdout.write(`OK  : ${e}
`),r}}}};function j(t){let e=new Map([["",[""].slice(0,0)]]);e.clear();let r=o=>{let i=e.get(o);if(i!==void 0)return i;let u=[];return e.set(o,u),u};for(let o of t){let{test_na:i}=o.result;if(i&&i.length>0){let{url:u}=o.job;for(let c of i)r(c).push(u);}}return JSON.stringify([...e.entries()],null,2)}function W(t){return [].concat(t).reduce((e,r)=>{let o=r.indexOf(":"),i=r.substr(0,o).trim();return e[i]=r.substr(o+1).trim(),e},{})}E.exports={Reporter:x,parseHeaders:W,transposeResults:j};});var _=$((ae,O)=>{w();var v=h("path"),b=__dirname.endsWith("models")?__dirname:v.resolve(__dirname,"models");O.exports={modelPaths:{report:v.resolve(b,"report.js"),dimension:v.resolve(b,"dimension.js"),variant:v.resolve(b,"variant.js"),search:v.resolve(b,"search.js")}};});var U=$((ue,P)=>{w();var{Sequelize:T,DataTypes:q}=h("sequelize");P.exports={hydrateModels:M};async function M(t,e){let r=T.Op,o=t.db||"postgresql://".concat(t["db-user"],":",t["db-pass"],"@",t["db-host"],"/",t["db-name"]);e.print(`Database Connection: ${o}`);let i;i=e.step("Creating new sequelize instance");let u=new T(o,{logging:!1,pool:{max:10,min:1,acquire:5*1e3,idle:10*1e3,evict:1*1e3}});i.resolve(),i=e.step("Testing connection"),await u.authenticate().then(i.resolve,i.reject),i=e.step("Retrieving database models");let{modelPaths:c}=_(),n=h(c.report)(u,q),s=h(c.dimension)(u,q),d=h(c.variant)(u,q),a=h(c.search)(u,q);return n.associate&&n.associate({dimension:s}),s.associate&&s.associate({report:n,variant:d}),d.associate&&d.associate({dimension:s}),a.associate&&a.associate({dimension:s,report:n,variant:d}),i.resolve(),n.findAllIn=function(f=[],l=[]){let m=e.step("Requesting configured profiles");return n.findAll({where:{id:f&&f.length>0?{[r.in]:f}:{[r.ne]:0},visible:!0},include:[{association:"dimensions",separate:!0,include:[{association:"variants",separate:!0,where:{id:l&&Array.isArray(l)&&l.length>0?{[r.in]:l}:{[r.ne]:0}}}],order:[["ordering","ASC"]]}]}).then(m.resolve,m.reject).then(y=>(f.length>0&&e.print(`User requested for profiles: ${f.join(", ")}`),l.length>0&&e.print(`User requested for variants: ${l.join(", ")}`),y.sort((g,S)=>f.indexOf(g.id)-f.indexOf(S.id)).map(g=>(e.print(`Profile found: ${g.id} ${g.name}`),g.dimensions.forEach(S=>{e.print(`  Dimension found: ${S.id} ${S.name}`),S.variants.forEach(N=>{e.print(`    Variant found: ${N.id} ${N.name}`);});}),g))))},a.findAllFromProfile=function(f,l){let m={};return l&&Array.isArray(l)&&l.length>0&&(m.variant_id={[r.in]:l}),a.findAll({where:{report_id:f.id,visible:!0,...m},order:[["zvalue","DESC"]]})},{Report:n,Search:a}}});var L=$((fe,F)=>{w();var{default:z}=h("axios");F.exports={naApiTest:K,naTestHandler:J};async function J(t){return t.evaluate(e);function e(){let r=document.querySelector("#bespoke-report"),o=document.createTreeWalker(r,NodeFilter.SHOW_TEXT,{acceptNode:n=>/\bN\/A\b|undefined/g.test(n.wholeText)?NodeFilter.FILTER_ACCEPT:NodeFilter.FILTER_SKIP}),i=[];for(;;){let n=o.nextNode();if(!n)break;i.push(u(n.parentNode));}return i;function u(n){let s,d,a;if(n.id&&(a=`#${CSS.escape(n.id)}`,document.querySelectorAll(a).length===1))return a;let f=n.localName;if(n.classList.length>0){for(let l=0;l<n.classList.length;l++)if(a=`.${CSS.escape(n.classList.item(l))}`,d=document.querySelectorAll(a),d.length===1||(a=f+a,d=document.querySelectorAll(a),d.length===1)||(s=c(n)+1,a=`${a}:nth-child(${s})`,d=document.querySelectorAll(a),d.length===1))return a}return n.parentNode!==document&&(s=c(n)+1,a=`${u(n.parentNode)} > ${f}:nth-child(${s})`),a}function c(n){return [...n.parentNode.children].indexOf(n)}}}async function K(t,e,r,o){let i=await z.get({baseURL:t,url:"api/profile/",params:{slug:e,id:r,locale:o}});return {url:i.config.url,paths:u([],i.data)};function u(c,n){let s=[],d=Object.keys(n);for(let a of d){let f=n[a];if(Array.isArray(f)||typeof f=="object"){let l=u(c.concat(a),f);s.push(...l);continue}if(/\bN\/A\b/g.test(`${f}`)){let l=c.concat(a).map(m=>m==parseInt(m,10)?`[${m}]`:`["${m}"]`).join("");s.push(l);}}return s}}});var re=$((he,k)=>{w();var B=h("fs"),G=h("path"),X=h("readline"),{Cluster:D}=h("puppeteer-cluster"),{Reporter:Y,parseHeaders:Q}=I(),{hydrateModels:V}=U(),{naTestHandler:Z}=L(),p=new Y;k.exports=async function(t){p.setOutput(t.output),p.printLine();let e=t.username&&t.password?{username:t.username,password:t.password}:void 0,r=Q(t.header),o=parseInt(t.timeout,10)*1e3,i=await D.launch({concurrency:D.CONCURRENCY_PAGE,maxConcurrency:parseInt(t.workers,10),monitor:!0,timeout:o});i.on("taskerror",(u,c)=>{let n={status:"ERROR",error:u.message,job:c};p.countResult(n),p.log(JSON.stringify(n));}),await i.task(async u=>{let{page:c,data:n}=u;await c.setDefaultNavigationTimeout(o),await c.setExtraHTTPHeaders(r),await c.setViewport({width:1280,height:720}),e&&await c.authenticate(e),await c.goto(n.url,{waitUntil:"load",timeout:o}),await c.waitForSelector("#bespoke-report",{timeout:o});let s=await Z(c).then(d=>({status:d.length===0?"SUCCESS":"FAILURE",result:{test_na:d},job:n}),d=>({status:"ERROR",error:d.message,job:n}));p.countResult(s),s.status!=="SUCCESS"&&p.log(JSON.stringify(s));}),t.input?await te(t,i):await ee(t,i),await i.close();};async function ee(t,e){let r=t.base||"",{Report:o,Search:i}=await V(t,p),u=t.profile?`${t.profile}`.split(",").filter(s=>s.trim()):[],c=t.variant?`${t.variant}`.split(",").filter(s=>s.trim()):[],n=await o.findAllIn(u,c);for(let s of n)if(p.printLine(),p.print(`Starting scan of profile ID ${s.id}: "${s.name}"`),s.dimensions.length>1)p.print(`Bilateral not supported yet: ID ${s.id}: "${s.name}"`);else {let d=await i.findAllFromProfile(s,c);for(let a of d)if(s&&s.dimensions[0]&&s.dimensions[0].variants.length>0){let f=s.dimensions[0].variants.find(m=>m.id===a.variant_id),l={locale:t.locale,page:a.slug,profile:f.slug};l.url=r.replace(/:(\D\w*)\b/g,(m,y)=>l[y]||`:${y}`),e.queue(l);}await e.idle(),p.writeReport(s.name),p.printCount(s.name),p.resetCounters();}}async function te(t,e){let r=t.base||"",o=G.resolve(process.cwd(),t.input);p.print(`Retrying warming errors from file:
  ${o}`);let i=X.createInterface({crlfDelay:1/0,input:B.createReadStream(o)});for await(let u of i){let c=JSON.parse(u);if(c.status==="SUCCESS")continue;let n=c.job;n.url=r.replace(/:(\w+)\b/g,(s,d)=>n[d]),e.queue(n);}await e.idle(),p.writeReport("retry"),p.printCount("retry"),p.resetCounters();}});var cluster = re();

module.exports = cluster;