pup-crawler
Version:
A simple web crawler that extracts data from websites, using Node.js and Puppeteer.
326 lines (325 loc) • 13.1 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.PupCrawler = void 0;
const puppeteer_1 = __importDefault(require("puppeteer"));
const jsdom_1 = __importDefault(require("jsdom"));
const { JSDOM } = jsdom_1.default;
// 把对象数组合成对象
function formatArrToObj(result) {
if (!result.length)
return {};
const obj = {};
result.forEach((item) => {
Object.assign(obj, item);
});
return obj;
}
// 页面内部脚本方法转换,
// async function evaluateFunc(page: Page, func: Function, args?: any) {
// const selectorCoreStr = func?.toString()
// const startIdx = selectorCoreStr.indexOf('{') + 1
// const endIdx = selectorCoreStr.lastIndexOf('}')
// const execStr = selectorCoreStr.slice(startIdx, endIdx)
// return page
// .evaluate((execStr) => {
// const exec = new Function('window', execStr) // 还原 after 函数
// return exec(window)
// }, execStr)
// .catch((err) => {
// console.error('evaluateFunc error:', err)
// })
// }
class PupCrawler {
browser;
url;
host;
states;
showLog;
constructor(props) {
this.browser = undefined;
this.url = undefined;
this.host = props?.host || '';
this.states = {}; // 存放状态数据
this.showLog = props?.showLog || false;
}
async open(options) {
this.browser = await puppeteer_1.default.launch(options || { protocolTimeout: 60000 });
this.browser.userAgent();
}
async close() {
await this.browser?.close();
}
setValue(key, value) {
this.states[key] = value;
}
getValue(key) {
return this.states[key];
}
/** 循环获取 */
async loopQueue(result, options) {
const { target } = options;
const { loopOpt, label: loopAttr } = target;
const mode = loopOpt?.mode || 'dynamic';
if (loopAttr && loopOpt) {
const links = typeof result[loopAttr] === 'string' ? [result[loopAttr]] : result[loopAttr];
this.showLog && console.log(`loopQueue info: len=${links.length}, attr=${loopAttr}, mode=${mode}`);
const looplist = [];
for await (let link of links) {
let loopResult = {};
if (mode === 'static') {
loopResult = await this.crawlStaticPage({ ...loopOpt, url: link, mode });
}
else {
loopResult = await this.crawlPage({ ...loopOpt, url: link, mode });
}
looplist.push(loopResult);
}
result[loopAttr] = looplist;
}
return result;
}
/** 自循环 */
async recursionRun(result, options) {
const { target, name, delayTime, recursion } = options;
const { loopKey, loopVals = [] } = recursion || {};
if (loopKey) {
const links = result[loopKey] || [];
const resArr = [];
// 多个循环
if (links.length > 1) {
for await (let link of links) {
// 递归不能要recursion，否则从第一开始, 和callback要处理和返回对象那个一致
const loopRes = await this.crawlPage({ url: link, target, name, delayTime });
if (!loopRes || !Object.keys(loopRes).length)
continue;
const tempObj = {};
loopVals.forEach((attr) => {
tempObj[attr] = loopRes[attr];
});
resArr.push(tempObj);
}
result[loopKey] = resArr;
}
else if (links.length === 1) {
// 如果只有一个值，前面就以及处理了
const tempObj = {};
loopVals.forEach((attr) => {
tempObj[attr] = result[attr];
});
result[loopKey] = [tempObj];
}
}
return result;
}
/** 自定义dom解析器 */
domParser(win, values) {
if (!values.length)
return [];
const res = values.map(({ css, all = false, attr, label, allIdx }) => {
// 0、可能直接执行命令
if (attr?.startsWith('window.')) {
return { [label]: win.eval(attr) };
}
// 1、如果没有css选择器，则返回空
if (!css)
return;
const [mainCss, subCss] = typeof css === 'string' ? [css] : css;
const elements = win.document.querySelectorAll(mainCss) || [];
// 2、单个查询
if (!all) {
let el = elements[0];
el = subCss ? el.querySelector(subCss) || el : el; // 如果子选择器没有，则取父元素
const text = attr ? el?.getAttribute(attr) : el?.textContent?.trim();
return { [label]: text };
}
// 3、多个查询
if (all) {
if (allIdx !== undefined) {
// 有index
const idx = allIdx < 0 ? elements.length + allIdx : allIdx;
const el = elements[idx];
// 3.1、有index && 没有子选择器：=>则取元素的值
if (!subCss) {
const text = attr ? el?.getAttribute(attr) : el?.textContent?.trim();
return { [label]: text };
}
else {
// 3.2、有index && 有子选择器：=> 则取子元素的值
const els = el.querySelectorAll(subCss) || [];
const subVals = [];
for (let i = 0; i < els.length; i++) {
const sel = els[i];
const text = attr ? sel?.getAttribute(attr) : sel?.textContent?.trim();
subVals.push(text);
}
return { [label]: subVals };
}
}
// 3.3、没有index && all： => 返回所有
const vals = [];
for (let i = 0; i < elements.length; i++) {
const el = elements[i];
const text = attr ? el?.getAttribute(attr) : el?.textContent?.trim();
vals.push(text);
}
return { [label]: vals };
}
});
const valArr = res.filter((item) => item !== undefined);
return valArr;
}
/** 页面操作 */
async actionsPlug(page, actions) {
if (!actions.length)
return true; // 没有操作，直接返回true
// 操作页面
for await (let action of actions) {
const { type, css, delay, waitCss, value } = action;
// 等待元素加载
if (waitCss) {
await page.waitForSelector(waitCss);
}
// 延迟 delayTime /1000 秒
if (delay) {
await new Promise((resolve) => setTimeout(resolve, delay));
}
switch (type) {
case 'click':
await page.click(css);
break;
case 'input':
await page.type(css, value);
break;
case 'scroll':
const [x, y] = value.split(',');
await page.evaluate((x, y) => window.scroll(x, y), x, y);
break;
default:
console.log('unknown action type:', type);
break;
}
}
}
/** 爬取内容核心 */
async crawlCore(params) {
const { name = 'default', url, target, delayTime = 0, plugs } = params;
const { crawlPlug, beforePlug, afterPlug, callbackPlug, finallyPlug, errorPlug } = plugs || {};
this.url = url?.includes(this.host) ? url : this.host + url;
console.log(`${name} crawling... =>> `, this.url);
const loopTarget = target.values.find((item) => item?.loopOpt); // 是否有循环属性
const isRecursion = !!params.recursion; // 是否是递归
// 前置处理插件
const beforeFlag = beforePlug ? await beforePlug?.() : true;
if (!beforeFlag)
return {};
// 延迟 delayTime /1000 秒
delayTime > 0 && (await new Promise((resolve) => setTimeout(resolve, delayTime)));
try {
let crawlResult = (await crawlPlug?.()) || [];
// 序列化对象
let result = formatArrToObj(crawlResult);
// 后置处理插件
const afterFlag = afterPlug ? await afterPlug?.(result) : result;
if (afterFlag === false)
return result;
result = afterFlag;
// 处理循环爬取: 不能传关于callback的东西，会从新执行
if (loopTarget) {
result = await this.loopQueue(result, { target: loopTarget });
}
// 自循环：把当前formatResult里面指定的key值的数组循环
if (isRecursion) {
result = await this.recursionRun(result, params);
}
return callbackPlug ? callbackPlug(result) : {};
}
catch (error) {
errorPlug?.(error);
return {};
}
finally {
await finallyPlug?.();
}
}
/** 爬取静态页面 */
async crawlStaticPage(params) {
const { url, target, delayTime = 0 } = params;
if (!target.values.length)
return [];
this.url = url?.includes(this.host) ? url : this.host + url;
// 延迟加载
delayTime > 0 && (await new Promise((resolve) => setTimeout(resolve, delayTime)));
const options = {
runScripts: 'outside-only',
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36',
};
// 开启一个请求
const { window } = (await JSDOM.fromURL(this.url, options)) || {};
if (!window)
return [];
return this.crawlCore({
...params,
plugs: {
errorPlug: params.error,
crawlPlug: () => this.domParser(window, target.values),
callbackPlug: params.callback,
afterPlug: params.after,
beforePlug: params.before,
},
});
}
/** 爬取页面属性：CrawlPageOptions */
async crawlPage(params) {
const { url, mode = 'dynamic', target, timeout = 60000, pageOpts } = params;
const { actions = [] } = target || {};
this.url = url?.includes(this.host) ? url : this.host + url;
// 【静态页面】去，去静态页面去
if (mode === 'static')
return this.crawlStaticPage(params);
// 【动态页面】爬取动态页面
const page = await this.browser?.newPage();
if (!page)
return {};
// 打开页面
await page.goto(this.url, { timeout: timeout, waitUntil: 'networkidle2', ...pageOpts });
page.on('console', (msg) => console.log('Console:', msg.text()));
// 禁止某些请求资源
page.on('requrst', (req) => {
const url = req.url();
if (url.includes('.ttf') || url.includes('.css') || url.endsWith('.woff')) {
req.abort();
}
else {
req.continue();
}
});
// 等待元素加载
target?.waitCss && (await page.waitForSelector(target.waitCss));
// 执行actions
await this.actionsPlug(page, actions);
const selectorCoreStr = this.domParser.toString();
const startIdx = selectorCoreStr.indexOf('{') + 1;
const endIdx = selectorCoreStr.lastIndexOf('}');
const execStr = selectorCoreStr.slice(startIdx, endIdx);
return await this.crawlCore({
...params,
plugs: {
callbackPlug: params.callback,
afterPlug: params.after,
beforePlug: params.before,
finallyPlug: params.finally || (() => page.close()),
errorPlug: params.error,
crawlPlug: () => page
.evaluate(({ execStr, vals }) => {
const exec = new Function('win', 'values', execStr); // 还原 selectorCore 函数
return exec(window, vals);
}, { execStr: execStr, vals: target.values })
.catch((e) => params?.error?.(e)),
},
});
}
}
exports.PupCrawler = PupCrawler;