pup-crawler/index.ts

A simple web crawler that extracts data from websites, using Node.js and Puppeteer.

import puppeteer, { Browser, LaunchOptions, Page } from 'puppeteer'
import jsdom from 'jsdom'
import type { IProps, CrawlOptions, CrawlOptionsCore, Target, obj } from './types.d.ts'
const { JSDOM } = jsdom

// 把对象数组合成对象
function formatArrToObj(result: obj[]): obj {
  if (!result.length) return {}
  const obj = {}
  result.forEach((item) => {
    Object.assign(obj, item)
  })
  return obj
}

// 页面内部脚本方法转换,
// async function evaluateFunc(page: Page, func: Function, args?: any) {
//   const selectorCoreStr = func?.toString()
//   const startIdx = selectorCoreStr.indexOf('{') + 1
//   const endIdx = selectorCoreStr.lastIndexOf('}')
//   const execStr = selectorCoreStr.slice(startIdx, endIdx)
//   return page
//     .evaluate((execStr) => {
//       const exec = new Function('window', execStr) // 还原 after 函数
//       return exec(window)
//     }, execStr)
//     .catch((err) => {
//       console.error('evaluateFunc error:', err)
//     })
// }

class PupCrawler {
  browser: Browser | undefined
  url: string | undefined
  host: string
  states: obj
  showLog?: boolean
  constructor(props?: IProps) {
    this.browser = undefined
    this.url = undefined
    this.host = props?.host || ''
    this.states = {} // 存放状态数据
    this.showLog = props?.showLog || false
  }
  async open(options?: LaunchOptions) {
    this.browser = await puppeteer.launch(options || { protocolTimeout: 60000 })
    this.browser.userAgent()
  }
  async close() {
    await this.browser?.close()
  }
  setValue(key: string, value: any) {
    this.states[key] = value
  }
  getValue(key: string) {
    return this.states[key]
  }

  /** 循环获取 */
  async loopQueue(result: obj, options: { target: any }) {
    const { target } = options
    const { loopOpt, label: loopAttr } = target as { loopOpt: CrawlOptions; label: string }
    const mode = loopOpt?.mode || 'dynamic'
    if (loopAttr && loopOpt) {
      const links = typeof result[loopAttr] === 'string' ? [result[loopAttr]] : result[loopAttr]
      this.showLog && console.log(`loopQueue info: len=${links.length}, attr=${loopAttr}, mode=${mode}`)
      const looplist: any[] = []
      for await (let link of links) {
        let loopResult = {}
        if (mode === 'static') {
          loopResult = await this.crawlStaticPage({ ...loopOpt, url: link, mode })
        } else {
          loopResult = await this.crawlPage({ ...loopOpt, url: link, mode })
        }
        looplist.push(loopResult)
      }
      result[loopAttr] = looplist
    }
    return result
  }

  /** 自循环 */
  async recursionRun(result: obj, options: CrawlOptions) {
    const { target, name, delayTime, recursion } = options
    const { loopKey, loopVals = [] } = recursion || {}
    if (loopKey) {
      const links = result[loopKey] || []
      const resArr: any[] = []
      // 多个循环
      if (links.length > 1) {
        for await (let link of links) {
          // 递归不能要recursion，否则从第一开始, 和callback要处理和返回对象那个一致
          const loopRes = await this.crawlPage({ url: link, target, name, delayTime })
          if (!loopRes || !Object.keys(loopRes).length) continue
          const tempObj: obj = {}
          loopVals.forEach((attr) => {
            tempObj[attr] = loopRes[attr]
          })
          resArr.push(tempObj)
        }
        result[loopKey] = resArr
      } else if (links.length === 1) {
        // 如果只有一个值，前面就以及处理了
        const tempObj = {} as obj
        loopVals.forEach((attr) => {
          tempObj[attr] = result[attr]
        })
        result[loopKey] = [tempObj]
      }
    }
    return result
  }

  /** 自定义dom解析器 */
  domParser(win: any, values: Target['values']): obj[] {
    if (!values.length) return []
    const res = values.map(({ css, all = false, attr, label, allIdx }) => {
      // 0、可能直接执行命令
      if (attr?.startsWith('window.')) {
        return { [label]: win.eval(attr) }
      }

      // 1、如果没有css选择器，则返回空
      if (!css) return

      const [mainCss, subCss] = typeof css === 'string' ? [css] : css
      const elements = win.document.querySelectorAll(mainCss) || []

      // 2、单个查询
      if (!all) {
        let el = elements[0]
        el = subCss ? el.querySelector(subCss) || el : el // 如果子选择器没有，则取父元素
        const text = attr ? el?.getAttribute(attr) : el?.textContent?.trim()
        return { [label]: text }
      }

      // 3、多个查询
      if (all) {
        if (allIdx !== undefined) {
          // 有index
          const idx = allIdx < 0 ? elements.length + allIdx : allIdx
          const el = elements[idx]
          // 3.1、有index && 没有子选择器：=>则取元素的值
          if (!subCss) {
            const text = attr ? el?.getAttribute(attr) : el?.textContent?.trim()
            return { [label]: text }
          } else {
            // 3.2、有index && 有子选择器：=> 则取子元素的值
            const els = el.querySelectorAll(subCss) || []
            const subVals: string[] = []
            for (let i = 0; i < els.length; i++) {
              const sel = els[i]
              const text = attr ? sel?.getAttribute(attr) : sel?.textContent?.trim()
              subVals.push(text)
            }
            return { [label]: subVals }
          }
        }

        // 3.3、没有index && all： => 返回所有
        const vals: string[] = []
        for (let i = 0; i < elements.length; i++) {
          const el = elements[i]
          const text = attr ? el?.getAttribute(attr) : el?.textContent?.trim()
          vals.push(text)
        }
        return { [label]: vals }
      }
    })
    const valArr = res.filter((item) => item !== undefined)
    return valArr
  }

  /** 页面操作 */
  async actionsPlug(page: Page, actions: any[]) {
    if (!actions.length) return true // 没有操作，直接返回true

    // 操作页面
    for await (let action of actions) {
      const { type, css, delay, waitCss, value } = action

      // 等待元素加载
      if (waitCss) {
        await page.waitForSelector(waitCss)
      }

      // 延迟 delayTime /1000 秒
      if (delay) {
        await new Promise((resolve) => setTimeout(resolve, delay))
      }

      switch (type) {
        case 'click':
          await page.click(css)
          break
        case 'input':
          await page.type(css, value)
          break
        case 'scroll':
          const [x, y] = value.split(',')
          await page.evaluate((x, y) => window.scroll(x, y), x, y)
          break

        default:
          console.log('unknown action type:', type)
          break
      }
    }
  }

  /** 爬取内容核心 */
  async crawlCore(params: CrawlOptionsCore): Promise<obj> {
    const { name = 'default', url, target, delayTime = 0, plugs } = params
    const { crawlPlug, beforePlug, afterPlug, callbackPlug, finallyPlug, errorPlug } = plugs || {}
    this.url = url?.includes(this.host) ? url : this.host + url
    console.log(`${name} crawling... =>> `, this.url)

    const loopTarget = target.values.find((item) => item?.loopOpt) // 是否有循环属性
    const isRecursion = !!params.recursion // 是否是递归

    // 前置处理插件
    const beforeFlag = beforePlug ? await beforePlug?.() : true
    if (!beforeFlag) return {}

    // 延迟 delayTime /1000 秒
    delayTime > 0 && (await new Promise((resolve) => setTimeout(resolve, delayTime)))

    try {
      let crawlResult = (await crawlPlug?.()) || []
      // 序列化对象
      let result = formatArrToObj(crawlResult)

      // 后置处理插件
      const afterFlag = afterPlug ? await afterPlug?.(result) : result
      if (afterFlag === false) return result
      result = afterFlag as obj

      // 处理循环爬取: 不能传关于callback的东西，会从新执行
      if (loopTarget) {
        result = await this.loopQueue(result, { target: loopTarget })
      }
      // 自循环：把当前formatResult里面指定的key值的数组循环
      if (isRecursion) {
        result = await this.recursionRun(result, params)
      }

      return callbackPlug ? callbackPlug(result) : {}
    } catch (error) {
      errorPlug?.(error)
      return {}
    } finally {
      await finallyPlug?.()
    }
  }

  /** 爬取静态页面 */
  async crawlStaticPage(params: CrawlOptions): Promise<obj> {
    const { url, target, delayTime = 0 } = params
    if (!target.values.length) return []
    this.url = url?.includes(this.host) ? url : this.host + url
    // 延迟加载
    delayTime > 0 && (await new Promise((resolve) => setTimeout(resolve, delayTime)))

    const options: any = {
      runScripts: 'outside-only',
      userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36',
    }

    // 开启一个请求
    const { window } = (await JSDOM.fromURL(this.url, options)) || {}
    if (!window) return []

    return this.crawlCore({
      ...params,
      plugs: {
        errorPlug: params.error,
        crawlPlug: () => this.domParser(window, target.values),
        callbackPlug: params.callback,
        afterPlug: params.after,
        beforePlug: params.before,
      },
    })
  }

  /** 爬取页面属性：CrawlPageOptions */
  async crawlPage(params: CrawlOptions): Promise<obj> {
    const { url, mode = 'dynamic', target, timeout = 60000, pageOpts } = params
    const { actions = [] } = target || {}
    this.url = url?.includes(this.host) ? url : this.host + url

    // 【静态页面】去，去静态页面去
    if (mode === 'static') return this.crawlStaticPage(params)

    // 【动态页面】爬取动态页面
    const page = await this.browser?.newPage()

    if (!page) return {}
    // 打开页面
    await page.goto(this.url, { timeout: timeout, waitUntil: 'networkidle2', ...pageOpts })
    page.on('console', (msg) => console.log('Console:', msg.text()))
    // 禁止某些请求资源
    page.on('requrst', (req: any) => {
      const url = req.url()
      if (url.includes('.ttf') || url.includes('.css') || url.endsWith('.woff')) {
        req.abort()
      } else {
        req.continue()
      }
    })
    // 等待元素加载
    target?.waitCss && (await page.waitForSelector(target.waitCss))
    // 执行actions
    await this.actionsPlug(page, actions)

    const selectorCoreStr = this.domParser.toString()
    const startIdx = selectorCoreStr.indexOf('{') + 1
    const endIdx = selectorCoreStr.lastIndexOf('}')
    const execStr = selectorCoreStr.slice(startIdx, endIdx)

    return await this.crawlCore({
      ...params,
      plugs: {
        callbackPlug: params.callback,
        afterPlug: params.after,
        beforePlug: params.before,
        finallyPlug: params.finally || (() => page.close()),
        errorPlug: params.error,
        crawlPlug: () =>
          page
            .evaluate(
              ({ execStr, vals }) => {
                const exec = new Function('win', 'values', execStr) // 还原 selectorCore 函数
                return exec(window, vals)
              },
              { execStr: execStr, vals: target.values }
            )
            .catch((e) => params?.error?.(e)),
      },
    })
  }
}

export { PupCrawler, CrawlOptions }