@datawheel/bespoke-cms-warmup/dist/index.js

Warmup utility for Bespoke CMS pages

#!/usr/bin/env node
'use strict';var S=(e=>typeof require<"u"?require:typeof Proxy<"u"?new Proxy(e,{get:(t,i)=>(typeof require<"u"?require:t)[i]}):e)(function(e){if(typeof require<"u")return require.apply(this,arguments);throw Error('Dynamic require of "'+e+'" is not supported')});var ce=(e,t)=>()=>(e&&(t=e(e=0)),t);var x=(e,t)=>()=>(t||e((t={exports:{}}).exports,t),t.exports);var C=ce(()=>{});var G=x((Ne,J)=>{C();var j=S("path"),N=__dirname.endsWith("models")?__dirname:j.resolve(__dirname,"models");J.exports={modelPaths:{report:j.resolve(N,"report.js"),dimension:j.resolve(N,"dimension.js"),variant:j.resolve(N,"variant.js"),search:j.resolve(N,"search.js")}};});var L=x((ze,Q)=>{C();var{Sequelize:X,DataTypes:O}=S("sequelize");Q.exports={hydrateModels:de};async function de(e,t){let i=X.Op,o=e.db||"postgresql://".concat(e["db-user"],":",e["db-pass"],"@",e["db-host"],"/",e["db-name"]);t.print(`Database Connection: ${o}`);let a;a=t.step("Creating new sequelize instance");let h=new X(o,{logging:false,pool:{max:10,min:1,acquire:5*1e3,idle:10*1e3,evict:1*1e3}});a.resolve(),a=t.step("Testing connection"),await h.authenticate().then(a.resolve,a.reject),a=t.step("Retrieving database models");let{modelPaths:d}=G(),r=S(d.report)(h,O),u=S(d.dimension)(h,O),s=S(d.variant)(h,O),n=S(d.search)(h,O);return r.associate&&r.associate({dimension:u}),u.associate&&u.associate({report:r,variant:s}),s.associate&&s.associate({dimension:u}),n.associate&&n.associate({dimension:u,report:r,variant:s}),a.resolve(),r.findAllIn=function(l=[],c=[]){let f=t.step("Requesting configured profiles");return r.findAll({where:{id:l&&l.length>0?{[i.in]:l}:{[i.ne]:0},visible:true},include:[{association:"dimensions",separate:true,include:[{association:"variants",separate:true,where:{id:c&&Array.isArray(c)&&c.length>0?{[i.in]:c}:{[i.ne]:0}}}],order:[["ordering","ASC"]]}]}).then(f.resolve,f.reject).then($=>(l.length>0&&t.print(`User requested for profiles: ${l.join(", ")}`),c.length>0&&t.print(`User requested for variants: ${c.join(", ")}`),$.sort((w,_)=>l.indexOf(w.id)-l.indexOf(_.id)).map(w=>(t.print(`Profile found: ${w.id} ${w.name}`),w.dimensions.forEach(_=>{t.print(`  Dimension found: ${_.id} ${_.name}`),_.variants.forEach(P=>{t.print(`    Variant found: ${P.id} ${P.name}`);});}),w))))},n.findAllFromProfile=function(l,c){let f={};return c&&Array.isArray(c)&&c.length>0&&(f.variant_id={[i.in]:c}),n.findAll({where:{report_id:l.id,visible:true,...f},order:[["zvalue","DESC"]]})},n.findAllMembers=function(l,c,f){return n.findAll({where:{report_id:l,dimension_id:c,variant_id:f,visible:true},attributes:["id","slug"],raw:true})},n.findAllFromBilateralPair=function(l,c,f,$,w,_,P){return n.findAll({where:{report_id:l,[i.or]:[{dimension_id:c,variant_id:f,id:$},{dimension_id:w,variant_id:_,id:P}],visible:true},order:[["zvalue","DESC"]]})},{Report:r,Search:n}}});var W=x((Fe,ee)=>{C();var z=S("fs"),K=S("path"),H=class{errorCount;failures;outputPath;successCount;writer;constructor(t){this.resetCounters(),t&&this.setOutput(t);}setOutput(t){let i=K.resolve(process.cwd(),t);z.existsSync(i)||z.mkdirSync(i),process.stdout.write(`Reports will be saved on:
  ${i}
`),this.outputPath=i;let o=K.resolve(i,"results.json");this.writer=z.createWriteStream(o,{flags:"w"});}log(t){this.writer.write(`${t}
`);}close(){return new Promise((t,i)=>{this.writer?(this.writer.end(()=>t()),this.writer.on("error",i)):t();})}writeReport(t){let i=Z(this.failures),o=K.resolve(this.outputPath,`profile_${t}.json`);return z.promises.writeFile(o,i)}print(t,i=true){process.stdout.write(`${t}${i?`
`:""}`);}printLine(t="-"){let i=process.stdout.columns*1||80,o=new Array(i).fill(t).join("").slice(0,i);process.stdout.write(`${o}
`);}countResult(t){t.status==="ERROR"&&this.errorCount++,t.status==="SUCCESS"&&this.successCount++,t.status==="FAILURE"&&this.failures.push(t);}resetCounters(){this.errorCount=0,this.failures=[],this.successCount=0;}printCount(t){let i="";this.errorCount>0&&(i=`with ${this.errorCount} errors`),this.failures.length>0&&(i=`${i?", and":"with"} ${this.failures.length} failures`),this.print(`
Profile "${t}" completed ${i||"without errors"}.`);}step(t){return process.stdout.write(`WAIT: ${t}...`),{reject(i){throw process.stdout.write("\r\x1B[K"),process.stdout.write(`ERR : ${t}
`),i},resolve(i){return process.stdout.write("\r\x1B[K"),process.stdout.write(`OK  : ${t}
`),i}}}};function Z(e){let t=new Map([["",[""].slice(0,0)]]);t.clear();let i=o=>{let a=t.get(o);if(a!==void 0)return a;let h=[];return t.set(o,h),h};for(let o of e){let{test_na:a}=o.result;if(a&&a.length>0){let{url:h}=o.job;for(let d of a)i(d).push(h);}}return JSON.stringify([...t.entries()],null,2)}function ue(e){return [].concat(e).reduce((t,i)=>{let o=i.indexOf(":"),a=i.substr(0,o).trim();return t[a]=i.substr(o+1).trim(),t},{})}var V=class{constructor(){this.cache=new Map;}getCanonicalKey(t,i){let[o,a]=[t,i].sort((h,d)=>h-d);return `${o}-${a}`}hasBeenProcessed(t,i,o){if(!this.cache.has(t))return  false;let a=this.getCanonicalKey(i,o);return this.cache.get(t).has(a)}markAsProcessed(t,i,o){this.cache.has(t)||this.cache.set(t,new Set);let a=this.getCanonicalKey(i,o);this.cache.get(t).add(a);}};ee.exports={Reporter:H,parseHeaders:ue,transposeResults:Z,BilateralCache:V};});var ie=x((Ke,te)=>{C();var{hydrateModels:fe}=L(),{Reporter:he,BilateralCache:me}=W(),q=new he;te.exports=async function(e){q.setOutput(e.output),q.printLine();let t=e.base||"",{Report:i,Search:o}=await fe(e,q),a=e.profile?`${e.profile}`.split(",").filter(r=>r.trim()):[],h=await i.findAllIn(a,[]),d=new me;for(let r of h)if(r.dimensions.length===1){let u=await o.findAllFromProfile(r,[]);for(let s of u){let n={page:s.slug,profile:s.id},l=t.replace(/:(\w+)\b/g,(c,f)=>n[f]);q.log(l);}}else if(r.dimensions.length>=2){let u=r.dimensions[0],s;if(e["variant1-id"]){if(s=u.variants.find(m=>m.id===parseInt(e["variant1-id"],10)),!s){q.print(`  Variant ID ${e["variant1-id"]} not found for dimension ${u.name}. Skipping.`);continue}}else s=u.variants[0];if(!s)continue;let n=await o.findAllMembers(r.id,u.id,s.id),l=r.dimensions[1],c;if(e["variant2-id"]){if(c=l.variants.find(m=>m.id===parseInt(e["variant2-id"],10)),!c){q.print(`  Variant ID ${e["variant2-id"]} not found for dimension ${l.name}. Skipping.`);continue}}else c=l.variants[0];if(!c)continue;let f=await o.findAllMembers(r.id,l.id,c.id);q.print(`Bilateral ${r.name}: ${u.name}(${n.length}) \xD7 ${l.name}(${f.length})`);let $=await o.findAll({where:{report_id:r.id,dimension_id:u.id,variant_id:s.id,visible:true}}),w=await o.findAll({where:{report_id:r.id,dimension_id:l.id,variant_id:c.id,visible:true}}),_=e["random-pairs"],P=_?n.sort(()=>Math.random()-.5):n.map(m=>{var b;return {...m,zvalue:((b=$.find(y=>y.id===m.id))==null?void 0:b.zvalue)||0}}).sort((m,b)=>(b.zvalue||0)-(m.zvalue||0)),E=_?f.sort(()=>Math.random()-.5):f.map(m=>{var b;return {...m,zvalue:((b=w.find(y=>y.id===m.id))==null?void 0:b.zvalue)||0}}).sort((m,b)=>(b.zvalue||0)-(m.zvalue||0)),I=[],M=new Set;for(let m of P)for(let b of E){if(m.id===b.id)continue;let y=d.getCanonicalKey(m.id,b.id);M.has(y)||(M.add(y),I.push({member1:m,member2:b,canonicalKey:y}));}let k=I,R=e["limit-pairs"]?parseInt(e["limit-pairs"],10):null;R&&R>0&&I.length>R&&(_?k=I.sort(()=>Math.random()-.5).slice(0,R):k=I.slice(0,R));for(let m of k){let{member1:b,member2:y}=m;if(d.hasBeenProcessed(r.id,b.id,y.id))continue;d.markAsProcessed(r.id,b.id,y.id);let T={variant1:s.slug,profile1:b.slug,variant2:c.slug,profile2:y.slug},U=t.replace(/:(\w+)\b/g,(B,p)=>T[p]);q.log(U);}}await q.close();};});var se=x((Ve,re)=>{C();var{default:pe}=S("axios");re.exports={naApiTest:be,naTestHandler:ge};async function ge(e){return e.evaluate(t);function t(){let i=document.querySelector("#bespoke-report"),o=document.createTreeWalker(i,NodeFilter.SHOW_TEXT,{acceptNode:r=>/\bN\/A\b|undefined/g.test(r.wholeText)?NodeFilter.FILTER_ACCEPT:NodeFilter.FILTER_SKIP}),a=[];for(;;){let r=o.nextNode();if(!r)break;a.push(h(r.parentNode));}return a;function h(r){let u,s,n;if(r.id&&(n=`#${CSS.escape(r.id)}`,document.querySelectorAll(n).length===1))return n;let l=r.localName;if(r.classList.length>0){for(let c=0;c<r.classList.length;c++)if(n=`.${CSS.escape(r.classList.item(c))}`,s=document.querySelectorAll(n),s.length===1||(n=l+n,s=document.querySelectorAll(n),s.length===1)||(u=d(r)+1,n=`${n}:nth-child(${u})`,s=document.querySelectorAll(n),s.length===1))return n}return r.parentNode!==document&&(u=d(r)+1,n=`${h(r.parentNode)} > ${l}:nth-child(${u})`),n}function d(r){return [...r.parentNode.children].indexOf(r)}}}async function be(e,t,i,o){let a=await pe.get({baseURL:e,url:"api/profile/",params:{slug:t,id:i,locale:o}});return {url:a.config.url,paths:h([],a.data)};function h(d,r){let u=[],s=Object.keys(r);for(let n of s){let l=r[n];if(Array.isArray(l)||typeof l=="object"){let c=h(d.concat(n),l);u.push(...c);continue}if(/\bN\/A\b/g.test(`${l}`)){let c=d.concat(n).map(f=>f==parseInt(f,10)?`[${f}]`:`["${f}"]`).join("");u.push(c);}}return u}}});var oe=x((Be,ae)=>{C();var we=S("fs"),ve=S("path"),$e=S("readline"),{Cluster:ne}=S("puppeteer-cluster"),{Reporter:ye,parseHeaders:Se,BilateralCache:Ce}=W(),{hydrateModels:_e}=L(),{naTestHandler:Re}=se(),g=new ye;ae.exports=async function(e){g.setOutput(e.output),g.printLine();let t=e.username&&e.password?{username:e.username,password:e.password}:void 0,i=Se(e.header),o=parseInt(e.timeout,10)*1e3,a=await ne.launch({concurrency:ne.CONCURRENCY_PAGE,maxConcurrency:parseInt(e.workers,10),monitor:true,timeout:o});a.on("taskerror",(h,d)=>{let r={status:"ERROR",error:h.message,job:d};g.countResult(r),g.log(JSON.stringify(r));}),await a.task(async h=>{let{page:d,data:r}=h;await d.setDefaultNavigationTimeout(o),await d.setExtraHTTPHeaders(i),await d.setViewport({width:1280,height:720}),t&&await d.authenticate(t),await d.goto(r.url,{waitUntil:"load",timeout:o}),await d.waitForSelector("#bespoke-report",{timeout:o});let u=await Re(d).then(s=>({status:s.length===0?"SUCCESS":"FAILURE",result:{test_na:s},job:r}),s=>({status:"ERROR",error:s.message,job:r}));g.countResult(u),u.status!=="SUCCESS"&&g.log(JSON.stringify(u));}),e.input?await Ie(e,a):await Ae(e,a),await a.close();};async function Ae(e,t){let i=e.base||"",{Report:o,Search:a}=await _e(e,g),h=e.profile?`${e.profile}`.split(",").filter(s=>s.trim()):[],d=e.variant?`${e.variant}`.split(",").filter(s=>s.trim()):[],r=await o.findAllIn(h,d),u=new Ce;for(let s of r)if(g.printLine(),g.print(`Starting scan of profile ID ${s.id}: "${s.name}"`),s.dimensions.length===1){let n=await a.findAllFromProfile(s,d);for(let l of n)if(s&&s.dimensions[0]&&s.dimensions[0].variants.length>0){let c=s.dimensions[0].variants.find($=>$.id===l.variant_id),f={locale:e.locale,page:l.slug,profile:c.slug};f.url=i.replace(/:(\D\w*)\b/g,($,w)=>f[w]||`:${w}`),t.queue(f);}await t.idle(),g.writeReport(s.name),g.printCount(s.name),g.resetCounters();}else if(s.dimensions.length>=2){let n=s.dimensions[0],l;if(e["variant1-id"]){if(l=n.variants.find(p=>p.id===parseInt(e["variant1-id"],10)),!l){g.print(`  Variant ID ${e["variant1-id"]} not found for dimension ${n.name}. Skipping.`);continue}}else l=n.variants[0];if(!l)continue;let c=await a.findAllMembers(s.id,n.id,l.id),f=s.dimensions[1],$;if(e["variant2-id"]){if($=f.variants.find(p=>p.id===parseInt(e["variant2-id"],10)),!$){g.print(`  Variant ID ${e["variant2-id"]} not found for dimension ${f.name}. Skipping.`);continue}}else $=f.variants[0];if(!$)continue;let w=await a.findAllMembers(s.id,f.id,$.id),_=c.length*w.length,P=await a.findAll({where:{report_id:s.id,dimension_id:n.id,variant_id:l.id,visible:true}}),E=await a.findAll({where:{report_id:s.id,dimension_id:f.id,variant_id:$.id,visible:true}}),I=e["random-pairs"],M=I?c.sort(()=>Math.random()-.5):c.map(p=>{var v;return {...p,zvalue:((v=P.find(A=>A.id===p.id))==null?void 0:v.zvalue)||0}}).sort((p,v)=>(v.zvalue||0)-(p.zvalue||0)),k=I?w.sort(()=>Math.random()-.5):w.map(p=>{var v;return {...p,zvalue:((v=E.find(A=>A.id===p.id))==null?void 0:v.zvalue)||0}}).sort((p,v)=>(v.zvalue||0)-(p.zvalue||0)),R=[],m=new Set;for(let p of M)for(let v of k){if(p.id===v.id)continue;let A=u.getCanonicalKey(p.id,v.id);m.has(A)||(m.add(A),R.push({member1:p,member2:v,canonicalKey:A}));}let b=R,y="",T=e["limit-pairs"]?parseInt(e["limit-pairs"],10):null;T&&T>0&&R.length>T&&(I?(b=R.sort(()=>Math.random()-.5).slice(0,T),y=`(${T} random of ${R.length})`):(b=R.slice(0,T),y=`(top ${T} of ${R.length})`));let U=I?"random":"importance";g.print(`Processing bilateral: ${n.name}(${c.length})\xD7${f.name}(${w.length}) \u2192 ${_} pairs ${y}, ordered by ${U}`);for(let p of b){let{member1:v,member2:A,canonicalKey:De}=p;if(u.hasBeenProcessed(s.id,v.id,A.id))continue;u.markAsProcessed(s.id,v.id,A.id);let F={locale:e.locale,variant1:l.slug,profile1:v.slug,variant2:$.slug,profile2:A.slug};F.url=i.replace(/:(\D\w*)\b/g,(ke,Y)=>F[Y]||`:${Y}`),t.queue(F);}await t.idle(),g.printCount(s.name),g.writeReport(s.name),g.resetCounters();}}async function Ie(e,t){let i=e.base||"",o=ve.resolve(process.cwd(),e.input);g.print(`Retrying warming errors from file:
  ${o}`);let a=$e.createInterface({crlfDelay:1/0,input:we.createReadStream(o)});for await(let h of a){let d=JSON.parse(h);if(d.status==="SUCCESS")continue;let r=d.job;r.url=i.replace(/:(\w+)\b/g,(u,s)=>r[s]),t.queue(r);}await t.idle(),g.writeReport("retry"),g.printCount("retry"),g.resetCounters();}});C();var Te=S("getopts"),le=`Bespoke CMS / Warmup script
`,qe=`${le}
Usage: npx bespoke-cms-warmup [command] [args]

Available commands are "scan" and "list".
If the command is not set, the script will execute the "scan" command.

Commands:
    scan    The "scan" command checks the available pages in the available
            profiles, and run the tests on each page.
            It has 2 modes: the "run" mode and the "retry" mode. The presence
            of the --input argument determines which mode the script will run.

            In run mode, the script needs to connect to the database and
            retrieve the items the profiles are built with, then sets the
            additional parameters.
                Required : base, db[-props]

            In retry mode, the script will use the results.json file generated
            by a previous run. All the parameters were saved inside, so passing
            them again is not needed.
                Required : input

    list    The "list" command is a reduced version of the scan command.
            Instead of generating the URLs, loading, and executing tests on
            them, it just generates the URLs and saves them in a file.
            This file can later be used in other tools, like siege.
                Required : base, db[-props]

Arguments:
    -b, --base      The root url to use as template in the generation.
                    These variables will be replaced:
                    - ':profile' for the profile name
                    - ':page' for the page slug
    -d, --debug     Prints some extra variables for debugging purposes.
    -H, --header    Set a header for all requests. Multiple headers are allowed
                    but each must be preceeded by the flag, like in curl.
                    The 'Host' header can't be modified.
    -h, --help      Shows this information.
                    This parameter must be used once for each "key: value" combo.
    -i, --input     The path to the 'results.json' file of the previous run.
    -o, --output    The path to the folder where the reports will be saved.
                    Defaults to './cms_warmup_YYYYMMDDhhmmss'.
    -p, --password  The password in case of needing basic authentication.
        --profile   A comma-separated numbers with report_id: 1,2,3
                    If omitted or empty, all available profiles will be used.
    -t, --timeout   Time limit to consider a page load failed, in seconds.
    -u, --username  The username in case of needing basic authentication.
        --db-host   The host and port where to connect to the database.
                    Defaults to "localhost:5432".
        --db-name   The name of the database where the info is stored.
        --db-user   The username to connect to the database.
        --db-pass   The password to connect to the database, if needed.
        --db        The full connection URI string to connect to the database.
                    Format is "engine://dbUser:dbPswd@dbHost/dbName".
                    If this variable is set, the previous ones are ignored.
    -w, --workers   The number of concurrent connections to work with. Default: 2
        --limit-pairs For bilateral profiles, limit testing to this many pairs.
                    If omitted, all pairs will be tested.
        --random-pairs For bilateral profiles, randomly select pairs instead of
                    ordering by importance (zvalue). Default: false (ordered by importance).
        --variant1-id For bilateral profiles, specify the variant ID for dimension 1.
                    If omitted, the first variant of dimension 1 will be used.
        --variant2-id For bilateral profiles, specify the variant ID for dimension 2.
                    If omitted, the first variant of dimension 2 will be used.
`,Pe=Te(process.argv.slice(2),{alias:{base:"b",debug:"d",header:"H",help:"h",input:"i",output:"o",password:"p",timeout:"t",username:"u",verbose:"v",workers:"w"},default:{"db-host":"localhost:5432",header:[],output:`./cms_warmup_${new Date().toISOString().replace(/\D/g,"").slice(0,14)}`,timeout:20,workers:"2"},boolean:["debug","help","verbose","random-pairs"],string:["base","db","db-host","db-name","db-pass","db-user","header","input","limit-pairs","output","password","profile","timeout","username","workers"]});xe(Pe).then(()=>{process.exit(0);},e=>{console.error(e),process.exit(1);});async function xe(e){e.help||e._.includes("help")?console.log(qe):(console.log(le),e.debug&&console.debug("Options:",e),e._.includes("list")?await ie()(e):await oe()(e));}