puppeteer-infinite-scroller/lib/index.js

Provides a simple and efficient solution for scraping data loaded through infinite scrolling on web pages using Puppeteer.

"use strict";async function y(t,o){let{scrollDelay:n=1e3,itemCount:a=10,pageFunction:e,selector:r}=o;if(!r&&!e)throw new Error("Either selector or pageFunction must be provided as an option");if(r&&typeof r!="string")throw new Error("Selector must be a string");if(e&&typeof e!="function")throw new Error("PageFunction must be a function");if(typeof a!="number")throw new Error("ItemCount must be a number");if(typeof n!="number")throw new Error("ScrollDelay must be a number");try{return await l(t,[],a,r,e,n)}catch(i){return console.log("Error:",i.message),[]}}async function l(t,o,n,a,e,r){if(n<=o.length)return o.slice(0,n);let i=[];a?i=await t.evaluate(f=>{let w=document.querySelectorAll(f);return Array.from(w).map(u=>{let b=Array.from(u.attributes).reduce((s,c)=>(s[c.name]=u.getAttribute(c.name),s),{});return{tagName:u.tagName,...b}})},a):e&&(i=await t.evaluate(e)),o.push(...i);let m=await t.evaluate("document.body.scrollHeight");return await t.evaluate("window.scrollTo(0, document.body.scrollHeight)"),await t.waitForFunction(`document.body.scrollHeight > ${m}`),await t.waitForTimeout(r),l(t,o,n,a,e,r)}module.exports=y;