@datawheel/bespoke-cms-warmup/dist/index.js

Warmup utility for Bespoke CMS pages

#!/usr/bin/env node
'use strict';

var f=(t=>typeof require<"u"?require:typeof Proxy<"u"?new Proxy(t,{get:(e,s)=>(typeof require<"u"?require:e)[s]}):t)(function(t){if(typeof require<"u")return require.apply(this,arguments);throw new Error('Dynamic require of "'+t+'" is not supported')});var J=(t,e)=>()=>(t&&(e=t(t=0)),e);var b=(t,e)=>()=>(e||t((e={exports:{}}).exports,e),e.exports);var m=J(()=>{});var O=b((me,N)=>{m();var y=f("path"),$=__dirname.endsWith("models")?__dirname:y.resolve(__dirname,"models");N.exports={modelPaths:{report:y.resolve($,"report.js"),dimension:y.resolve($,"dimension.js"),variant:y.resolve($,"variant.js"),search:y.resolve($,"search.js")}};});var x=b((ge,k)=>{m();var{Sequelize:P,DataTypes:q}=f("sequelize");k.exports={hydrateModels:K};async function K(t,e){let s=P.Op,a=t.db||"postgresql://".concat(t["db-user"],":",t["db-pass"],"@",t["db-host"],"/",t["db-name"]);e.print(`Database Connection: ${a}`);let o;o=e.step("Creating new sequelize instance");let u=new P(a,{logging:!1,pool:{max:10,min:1,acquire:5*1e3,idle:10*1e3,evict:1*1e3}});o.resolve(),o=e.step("Testing connection"),await u.authenticate().then(o.resolve,o.reject),o=e.step("Retrieving database models");let{modelPaths:c}=O(),r=f(c.report)(u,q),n=f(c.dimension)(u,q),l=f(c.variant)(u,q),i=f(c.search)(u,q);return r.associate&&r.associate({dimension:n}),n.associate&&n.associate({report:r,variant:l}),l.associate&&l.associate({dimension:n}),i.associate&&i.associate({dimension:n,report:r,variant:l}),o.resolve(),r.findAllIn=function(h=[],d=[]){let w=e.step("Requesting configured profiles");return r.findAll({where:{id:h&&h.length>0?{[s.in]:h}:{[s.ne]:0},visible:!0},include:[{association:"dimensions",separate:!0,include:[{association:"variants",separate:!0,where:{id:d&&Array.isArray(d)&&d.length>0?{[s.in]:d}:{[s.ne]:0}}}],order:[["ordering","ASC"]]}]}).then(w.resolve,w.reject).then(R=>(h.length>0&&e.print(`User requested for profiles: ${h.join(", ")}`),d.length>0&&e.print(`User requested for variants: ${d.join(", ")}`),R.sort((g,S)=>h.indexOf(g.id)-h.indexOf(S.id)).map(g=>(e.print(`Profile found: ${g.id} ${g.name}`),g.dimensions.forEach(S=>{e.print(`  Dimension found: ${S.id} ${S.name}`),S.variants.forEach(j=>{e.print(`    Variant found: ${j.id} ${j.name}`);});}),g))))},i.findAllFromProfile=function(h,d){let w={};return d&&Array.isArray(d)&&d.length>0&&(w.variant_id={[s.in]:d}),i.findAll({where:{report_id:h.id,visible:!0,...w},order:[["zvalue","DESC"]]})},{Report:r,Search:i}}});var _=b((Se,U)=>{m();var C=f("fs"),A=f("path"),I=class{errorCount;failures;outputPath;successCount;writer;constructor(e){this.resetCounters(),e&&this.setOutput(e);}setOutput(e){let s=A.resolve(process.cwd(),e);C.existsSync(s)||C.mkdirSync(s),process.stdout.write(`Reports will be saved on:
  ${s}
`),this.outputPath=s;let a=A.resolve(s,"results.json");this.writer=C.createWriteStream(a,{flags:"w"});}log(e){this.writer.write(`${e}
`);}writeReport(e){let s=E(this.failures),a=A.resolve(this.outputPath,`profile_${e}.json`);return C.promises.writeFile(a,s)}print(e,s=!0){process.stdout.write(`${e}${s?`
`:""}`);}printLine(e="-"){let s=process.stdout.columns*1||80,a=new Array(s).fill(e).join("").slice(0,s);process.stdout.write(`${a}
`);}countResult(e){e.status==="ERROR"&&this.errorCount++,e.status==="SUCCESS"&&this.successCount++,e.status==="FAILURE"&&this.failures.push(e);}resetCounters(){this.errorCount=0,this.failures=[],this.successCount=0;}printCount(e){let s="";this.errorCount>0&&(s=`with ${this.errorCount} errors`),this.failures.length>0&&(s=`${s?", and":"with"} ${this.failures.length} failures`),this.print(`
Profile "${e}" completed ${s||"without errors"}.`);}step(e){return process.stdout.write(`WAIT: ${e}...`),{reject(s){throw process.stdout.write("\r\x1B[K"),process.stdout.write(`ERR : ${e}
`),s},resolve(s){return process.stdout.write("\r\x1B[K"),process.stdout.write(`OK  : ${e}
`),s}}}};function E(t){let e=new Map([["",[""].slice(0,0)]]);e.clear();let s=a=>{let o=e.get(a);if(o!==void 0)return o;let u=[];return e.set(a,u),u};for(let a of t){let{test_na:o}=a.result;if(o&&o.length>0){let{url:u}=a.job;for(let c of o)s(c).push(u);}}return JSON.stringify([...e.entries()],null,2)}function B(t){return [].concat(t).reduce((e,s)=>{let a=s.indexOf(":"),o=s.substr(0,a).trim();return e[o]=s.substr(a+1).trim(),e},{})}U.exports={Reporter:I,parseHeaders:B,transposeResults:E};});var L=b((Re,D)=>{m();var{hydrateModels:G}=x(),{Reporter:X}=_(),T=new X;D.exports=async function(t){T.setOutput(t.output),T.printLine();let e=t.base||"",{ProfileMeta:s,Search:a}=await G(t,T),o=await s.findAllIn([],[]);for(let u of o){let c=await a.findAllFromProfile(u,[]);for(let r of c){let n={page:r.slug,profile:u.slug},l=e.replace(/:(\w+)\b/g,(i,h)=>n[h]);T.log(l);}}};});var H=b((qe,F)=>{m();var{default:Q}=f("axios");F.exports={naApiTest:Z,naTestHandler:V};async function V(t){return t.evaluate(e);function e(){let s=document.querySelector("#bespoke-report"),a=document.createTreeWalker(s,NodeFilter.SHOW_TEXT,{acceptNode:r=>/\bN\/A\b|undefined/g.test(r.wholeText)?NodeFilter.FILTER_ACCEPT:NodeFilter.FILTER_SKIP}),o=[];for(;;){let r=a.nextNode();if(!r)break;o.push(u(r.parentNode));}return o;function u(r){let n,l,i;if(r.id&&(i=`#${CSS.escape(r.id)}`,document.querySelectorAll(i).length===1))return i;let h=r.localName;if(r.classList.length>0){for(let d=0;d<r.classList.length;d++)if(i=`.${CSS.escape(r.classList.item(d))}`,l=document.querySelectorAll(i),l.length===1||(i=h+i,l=document.querySelectorAll(i),l.length===1)||(n=c(r)+1,i=`${i}:nth-child(${n})`,l=document.querySelectorAll(i),l.length===1))return i}return r.parentNode!==document&&(n=c(r)+1,i=`${u(r.parentNode)} > ${h}:nth-child(${n})`),i}function c(r){return [...r.parentNode.children].indexOf(r)}}}async function Z(t,e,s,a){let o=await Q.get({baseURL:t,url:"api/profile/",params:{slug:e,id:s,locale:a}});return {url:o.config.url,paths:u([],o.data)};function u(c,r){let n=[],l=Object.keys(r);for(let i of l){let h=r[i];if(Array.isArray(h)||typeof h=="object"){let d=u(c.concat(i),h);n.push(...d);continue}if(/\bN\/A\b/g.test(`${h}`)){let d=c.concat(i).map(w=>w==parseInt(w,10)?`[${w}]`:`["${w}"]`).join("");n.push(d);}}return n}}});var Y=b((Te,W)=>{m();var ee=f("fs"),te=f("path"),se=f("readline"),{Cluster:M}=f("puppeteer-cluster"),{Reporter:re,parseHeaders:ne}=_(),{hydrateModels:oe}=x(),{naTestHandler:ae}=H(),p=new re;W.exports=async function(t){p.setOutput(t.output),p.printLine();let e=t.username&&t.password?{username:t.username,password:t.password}:void 0,s=ne(t.header),a=parseInt(t.timeout,10)*1e3,o=await M.launch({concurrency:M.CONCURRENCY_PAGE,maxConcurrency:parseInt(t.workers,10),monitor:!0,timeout:a});o.on("taskerror",(u,c)=>{let r={status:"ERROR",error:u.message,job:c};p.countResult(r),p.log(JSON.stringify(r));}),await o.task(async u=>{let{page:c,data:r}=u;await c.setDefaultNavigationTimeout(a),await c.setExtraHTTPHeaders(s),await c.setViewport({width:1280,height:720}),e&&await c.authenticate(e),await c.goto(r.url,{waitUntil:"load",timeout:a}),await c.waitForSelector("#bespoke-report",{timeout:a});let n=await ae(c).then(l=>({status:l.length===0?"SUCCESS":"FAILURE",result:{test_na:l},job:r}),l=>({status:"ERROR",error:l.message,job:r}));p.countResult(n),n.status!=="SUCCESS"&&p.log(JSON.stringify(n));}),t.input?await ce(t,o):await ie(t,o),await o.close();};async function ie(t,e){let s=t.base||"",{Report:a,Search:o}=await oe(t,p),u=t.profile?`${t.profile}`.split(",").filter(n=>n.trim()):[],c=t.variant?`${t.variant}`.split(",").filter(n=>n.trim()):[],r=await a.findAllIn(u,c);for(let n of r)if(p.printLine(),p.print(`Starting scan of profile ID ${n.id}: "${n.name}"`),n.dimensions.length>1)p.print(`Bilateral not supported yet: ID ${n.id}: "${n.name}"`);else {let l=await o.findAllFromProfile(n,c);for(let i of l)if(n&&n.dimensions[0]&&n.dimensions[0].variants.length>0){let h=n.dimensions[0].variants.find(w=>w.id===i.variant_id),d={locale:t.locale,page:i.slug,profile:h.slug};d.url=s.replace(/:(\D\w*)\b/g,(w,R)=>d[R]||`:${R}`),e.queue(d);}await e.idle(),p.writeReport(n.name),p.printCount(n.name),p.resetCounters();}}async function ce(t,e){let s=t.base||"",a=te.resolve(process.cwd(),t.input);p.print(`Retrying warming errors from file:
  ${a}`);let o=se.createInterface({crlfDelay:1/0,input:ee.createReadStream(a)});for await(let u of o){let c=JSON.parse(u);if(c.status==="SUCCESS")continue;let r=c.job;r.url=s.replace(/:(\w+)\b/g,(n,l)=>r[l]),e.queue(r);}await e.idle(),p.writeReport("retry"),p.printCount("retry"),p.resetCounters();}});m();var ue=f("getopts"),z=`Bespoke CMS / Warmup script
`,le=`${z}
Usage: npx bespoke-cms-warmup [command] [args]

Available commands are "scan" and "list".
If the command is not set, the script will execute the "scan" command.

Commands:
    scan    The "scan" command checks the available pages in the available
            profiles, and run the tests on each page.
            It has 2 modes: the "run" mode and the "retry" mode. The presence
            of the --input argument determines which mode the script will run.

            In run mode, the script needs to connect to the database and
            retrieve the items the profiles are built with, then sets the
            additional parameters.
                Required : base, db[-props]

            In retry mode, the script will use the results.json file generated
            by a previous run. All the parameters were saved inside, so passing
            them again is not needed.
                Required : input

    list    The "list" command is a reduced version of the scan command.
            Instead of generating the URLs, loading, and executing tests on
            them, it just generates the URLs and saves them in a file.
            This file can later be used in other tools, like siege.
                Required : base, db[-props]

Arguments:
    -b, --base      The root url to use as template in the generation.
                    These variables will be replaced:
                    - ':profile' for the profile name
                    - ':page' for the page slug
    -d, --debug     Prints some extra variables for debugging purposes.
    -H, --header    Set a header for all requests. Multiple headers are allowed
                    but each must be preceeded by the flag, like in curl.
                    The 'Host' header can't be modified.
    -h, --help      Shows this information.
                    This parameter must be used once for each "key: value" combo.
    -i, --input     The path to the 'results.json' file of the previous run.
    -o, --output    The path to the folder where the reports will be saved.
                    Defaults to './cms_warmup_YYYYMMDDhhmmss'.
    -p, --password  The password in case of needing basic authentication.
        --profile   A comma-separated numbers with report_id: 1,2,3
                    If omitted or empty, all available profiles will be used.
    -t, --timeout   Time limit to consider a page load failed, in seconds.
    -u, --username  The username in case of needing basic authentication.
        --db-host   The host and port where to connect to the database.
                    Defaults to "localhost:5432".
        --db-name   The name of the database where the info is stored.
        --db-user   The username to connect to the database.
        --db-pass   The password to connect to the database, if needed.
        --db        The full connection URI string to connect to the database.
                    Format is "engine://dbUser:dbPswd@dbHost/dbName".
                    If this variable is set, the previous ones are ignored.
    -w, --workers   The number of concurrent connections to work with. Default: 2
`,de=ue(process.argv.slice(2),{alias:{base:"b",debug:"d",header:"H",help:"h",input:"i",output:"o",password:"p",timeout:"t",username:"u",verbose:"v",workers:"w"},default:{"db-host":"localhost:5432",header:[],output:`./cms_warmup_${new Date().toISOString().replace(/\D/g,"").slice(0,14)}`,timeout:20,workers:"2"},boolean:["debug","help","verbose"],string:["base","db","db-host","db-name","db-pass","db-user","header","input","output","password","profile","timeout","username","workers"]});he(de).then(()=>{process.exit(0);},t=>{console.error(t),process.exit(1);});async function he(t){t.help||t._.includes("help")?console.log(le):(console.log(z),t.debug&&console.debug("Options:",t),t._.includes("list")?await L()(t):await Y()(t));}