@blocklet/crawler/lib/cjs/middlewares/crawler.js

blocklet crawler lib

"use strict";

Object.defineProperty(exports, "__esModule", {
  value: true
});
exports.getPageContent = exports.crawlUrl = exports.crawlBlocklet = exports.cancelCronCrawlBlocklet = void 0;
Object.defineProperty(exports, "getRelativePath", {
  enumerable: true,
  get: function () {
    return _utils.getRelativePath;
  }
});
exports.setUrlInfoToCache = exports.initCronCrawlBlocklet = exports.getUrlInfoFromCache = void 0;
var PQueue = _interopRequireWildcard(require("p-queue"));
var _ufo = require("ufo");
var _utils = require("./utils");
var _cron = _interopRequireDefault(require("@abtnode/cron"));
var _debounce = _interopRequireDefault(require("lodash/debounce"));
function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function (e) { return e ? t : r; })(e); }
function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != typeof e && "function" != typeof e) return { default: e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && {}.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n.default = e, t && t.set(e, n), n; }
const formatHtml = htmlString => {
  if (typeof htmlString !== "string") {
    return "";
  }
  if (htmlString.includes("<h2>Unexpected Application Error!</h2>")) {
    return "";
  }
  return htmlString;
};
const crawlQueue = new PQueue.default({
  concurrency: 1,
  timeout: 60 * 1e3
});
const getPageContent = async ({
  url,
  formatPageContent
}) => {
  let page = await (0, _utils.initPage)();
  let pageContent = null;
  try {
    const response = await page.goto(url, {
      timeout: 60 * 1e3
      // 60s
    });
    if (!response) {
      throw new Error(`Failed to load page: response is null for ${url}`);
    }
    const statusCode = response.status();
    if (![200, 304].includes(statusCode)) {
      throw new Error(`Request failed with status ${statusCode}, in ${url}`);
    }
    await page.waitForNetworkIdle({
      idleTime: 1 * 1e3
      // 1s
    });
    if (formatPageContent) {
      pageContent = await formatPageContent({
        page,
        url
      });
    } else {
      pageContent = await page.content();
    }
  } catch (error) {
    _utils.logger.error("Get page content error:", error.message || error);
  } finally {
    await page.close();
  }
  return formatHtml(pageContent);
};
exports.getPageContent = getPageContent;
const getUrlInfoFromCache = async url => {
  const cache = await _utils.useCache.get((0, _utils.getRelativePath)(url));
  return cache;
};
exports.getUrlInfoFromCache = getUrlInfoFromCache;
function getNextCrawlDate(lastmod) {
  const now = /* @__PURE__ */new Date();
  const lastModTime = lastmod ? new Date(lastmod).getTime() || 0 : 0;
  const daysDiff = Math.max(0, (now.getTime() - lastModTime) / (24 * 60 * 60 * 1e3));
  const CRAWL_INTERVALS = /* @__PURE__ */new Map([[[-1, 0], 1],
  // 无 lastmod
  [[0, 3], 1],
  // 3 天内活跃
  [[3, 7], 3],
  // 7 天内活跃
  [[7, 30], 14],
  // 30 天内活跃
  [[30, 90], 30],
  // 90 天内活跃
  [[90, Infinity], 365]
  // 长期不活跃
  ]);
  const interval = Array.from(CRAWL_INTERVALS.entries()).find(([[min, max]]) => lastmod ? daysDiff > min && daysDiff <= max : min === -1)?.[1] || 90;
  return new Date(now.getTime() + interval * 24 * 60 * 60 * 1e3).toISOString();
}
const setUrlInfoToCache = async ({
  url,
  content,
  lastmod,
  nextDate
}) => {
  if (!content || !url) {
    return;
  }
  const lastmodValue = lastmod || (/* @__PURE__ */new Date()).toISOString();
  return await _utils.useCache.set((0, _utils.getRelativePath)(url), {
    content,
    lastmod: lastmodValue,
    updatedAt: (/* @__PURE__ */new Date()).toISOString(),
    nextDate: nextDate || getNextCrawlDate(lastmodValue)
  });
};
exports.setUrlInfoToCache = setUrlInfoToCache;
const crawlUrl = async ({
  urls,
  lastmodMap,
  formatPageContent,
  autoCloseBrowserCount = 50
}) => {
  if (typeof urls === "string") {
    urls = [urls];
  }
  const crawlUrlJob = ({
    url,
    retryCount = 0,
    index
  }) => {
    return async () => {
      try {
        if (index % autoCloseBrowserCount === 0) {
          await (0, _utils.closeBrowser)({
            trimCache: false
          });
        }
        const canCrawl = await (0, _utils.isAcceptCrawler)(url);
        if (canCrawl) {
          const pageContent = await getPageContent({
            url,
            formatPageContent
          });
          if (pageContent) {
            const lastmodValue = lastmodMap?.get(url) || (/* @__PURE__ */new Date()).toISOString();
            await setUrlInfoToCache({
              url,
              content: pageContent,
              lastmod: lastmodValue,
              nextDate: getNextCrawlDate(lastmodValue)
            });
            _utils.logger.info(`Crawler[${index}] ${url} success`);
          } else if (retryCount < 3) {
            retryCount++;
            const timeout = 1e3 * 5 * retryCount;
            _utils.logger.info(`Crawler[${index}] ${url} fail, continue after ${timeout}ms`);
            await (0, _utils.sleep)(timeout);
            crawlQueue.add(crawlUrlJob({
              url,
              retryCount,
              index
            }), {
              priority: retryCount
              // Operations with greater priority will be scheduled first.
            });
          } else {
            _utils.logger.info(`Crawler[${index}] ${url} fail reach 3 times, skip it`);
          }
        } else {
          _utils.logger.info(`Crawler[${index}] ${url} skip`);
        }
        if (global.gc) {
          global.gc();
        }
      } catch (error) {
        _utils.logger.info(`Crawler[${index}] ${url} abort by error, skip it: ${error?.message || error?.reason || error}`);
      }
    };
  };
  for (const [index, url] of urls.entries()) {
    await crawlQueue.add(crawlUrlJob({
      url,
      index: index + 1
    }));
  }
};
exports.crawlUrl = crawlUrl;
const crawlBlockletRunningMap = /* @__PURE__ */new Map();
const crawlBlocklet = async () => {
  const {
    mountPoint,
    did
  } = (0, _utils.getComponentInfo)();
  if (crawlBlockletRunningMap.has(did) && crawlBlockletRunningMap.get(did)) {
    _utils.logger.info(`Crawler blocklet ${did} is running, skip it`);
    return;
  }
  try {
    const browser = await (0, _utils.getBrowser)();
    if (!browser) {
      throw new Error("No Browser can use");
    }
    _utils.logger.info("Crawler blocklet existing can use browser");
  } catch (error) {
    _utils.logger.info(`Crawler blocklet abort by error: ${error?.message || error?.reason || error}`);
    return;
  }
  const {
    appUrl
  } = _utils.env;
  if (!appUrl) {
    throw new Error("appUrl not found");
  }
  const sitemapList = await (0, _utils.getSitemapList)(appUrl);
  const matchMountPoint = (0, _ufo.joinURL)(appUrl, !mountPoint || mountPoint === "/" ? "" : mountPoint);
  const otherMountPointList = _utils.components.filter(item => item.mountPoint && item.mountPoint !== mountPoint).map(item => item.mountPoint);
  const blockletLocList = sitemapList.filter(item => {
    if (mountPoint !== "/") {
      return item?.url?.indexOf(matchMountPoint) > -1;
    }
    return otherMountPointList.every(mountPoint2 => item?.url?.indexOf(mountPoint2) === -1);
  });
  const canUseBlockletLocList = [];
  const lastmodMap = /* @__PURE__ */new Map();
  let skipBlockletLocTotal = 0;
  let blockletLocTotal = 0;
  await Promise.all(blockletLocList.map(async item => {
    let tempLocList = [];
    if (item.url) {
      tempLocList.push(item.url);
    }
    if (item?.links?.length > 0) {
      tempLocList.push(...item.links?.map(ytem => ytem.url));
    }
    blockletLocTotal += tempLocList.length;
    tempLocList = (await Promise.all(tempLocList.map(async loc => {
      try {
        const {
          lastmod: cacheLastmod,
          nextDate
        } = await _utils.useCache.get((0, _utils.getRelativePath)(loc));
        if (item.lastmod && new Date(cacheLastmod).getTime() === new Date(item.lastmod).getTime()) {
          skipBlockletLocTotal++;
          return false;
        }
        if (nextDate && new Date(nextDate).getTime() >= (/* @__PURE__ */new Date()).getTime()) {
          skipBlockletLocTotal++;
          return false;
        }
        return loc;
      } catch (error) {}
      return loc;
    }))).filter(Boolean);
    tempLocList.forEach(loc => {
      if (item.lastmod) lastmodMap.set(loc, item.lastmod);
    });
    canUseBlockletLocList.push(...tempLocList);
  }));
  const crawlerLogText = (step = "") => [`Crawler sitemap.xml about ${did} ${step}: `, {
    blockletLocTotal,
    canUseBlockletLocTotal: canUseBlockletLocList.length,
    skipBlockletLocTotal,
    lastmodMapTotal: lastmodMap.size
  }];
  _utils.logger.info(...crawlerLogText("start"));
  try {
    crawlBlockletRunningMap.set(did, true);
    await crawlUrl({
      // @ts-ignore
      urls: canUseBlockletLocList,
      lastmodMap,
      formatPageContent: async ({
        page,
        url
      }) => {
        const pageContent = await page.evaluate(() => {
          const removeElements = tagName => {
            const elements = document.querySelectorAll(tagName);
            for (let i = elements.length - 1; i >= 0; i--) {
              try {
                elements[i]?.parentNode?.removeChild(elements[i]);
              } catch (error) {}
            }
          };
          removeElements('[id="uploader-container"]');
          removeElements('[class^="uppy-"]');
          removeElements('[id="point-up-component"]');
          const meta = document.createElement("meta");
          meta.name = "blocklet-crawler";
          meta.content = "true";
          document.head.appendChild(meta);
          return document.documentElement.outerHTML;
        });
        return pageContent;
      }
    });
    _utils.logger.info(...crawlerLogText("success"));
    await (0, _utils.closeBrowser)({
      trimCache: true
    });
  } catch (error) {
    _utils.logger.info(`Crawler blocklet abort by error`, error);
  } finally {
    crawlBlockletRunningMap.delete(did);
  }
};
exports.crawlBlocklet = crawlBlocklet;
const CRON_CRAWL_BLOCKLET_KEY = "cron-crawl-blocklet";
let cronCrawlBlockletJob = null;
const initCronCrawlBlocklet = ({
  time = "0 0 */12 * * *",
  // every 12 hours
  options
} = {}) => {
  if (!cronCrawlBlockletJob) {
    cronCrawlBlockletJob = _cron.default.init({
      context: {},
      jobs: [{
        name: CRON_CRAWL_BLOCKLET_KEY,
        time,
        fn: (0, _debounce.default)(crawlBlocklet),
        options: {
          runOnInit: false,
          ...options
        }
      }],
      onError: err => {
        console.error("run job failed", err);
      }
    });
  }
  return cronCrawlBlockletJob;
};
exports.initCronCrawlBlocklet = initCronCrawlBlocklet;
const cancelCronCrawlBlocklet = () => {
  if (cronCrawlBlockletJob) {
    cronCrawlBlockletJob.jobs[CRON_CRAWL_BLOCKLET_KEY].stop();
    cronCrawlBlockletJob = null;
    crawlQueue.clear();
    _utils.logger.info("Cron crawl blocklet stop, clear crawl queue");
  }
};
exports.cancelCronCrawlBlocklet = cancelCronCrawlBlocklet;