@blocklet/crawler/lib/es/middlewares/crawler.js

blocklet crawler lib

import * as PQueue from "p-queue";
import { joinURL } from "ufo";
import {
  logger,
  initPage,
  useCache,
  getSitemapList,
  isAcceptCrawler,
  env,
  components,
  getComponentInfo,
  sleep,
  closeBrowser,
  getBrowser,
  getRelativePath
} from "./utils.js";
import Cron from "@abtnode/cron";
import debounce from "lodash/debounce";
export { getRelativePath };
const formatHtml = (htmlString) => {
  if (typeof htmlString !== "string") {
    return "";
  }
  if (htmlString.includes("<h2>Unexpected Application Error!</h2>")) {
    return "";
  }
  return htmlString;
};
const crawlQueue = new PQueue.default({ concurrency: 1, timeout: 60 * 1e3 });
export const getPageContent = async ({ url, formatPageContent }) => {
  let page = await initPage();
  let pageContent = null;
  try {
    const response = await page.goto(url, {
      timeout: 60 * 1e3
      // 60s
    });
    if (!response) {
      throw new Error(`Failed to load page: response is null for ${url}`);
    }
    const statusCode = response.status();
    if (![200, 304].includes(statusCode)) {
      throw new Error(`Request failed with status ${statusCode}, in ${url}`);
    }
    await page.waitForNetworkIdle({
      idleTime: 1 * 1e3
      // 1s
    });
    if (formatPageContent) {
      pageContent = await formatPageContent({ page, url });
    } else {
      pageContent = await page.content();
    }
  } catch (error) {
    logger.error("Get page content error:", error.message || error);
  } finally {
    await page.close();
  }
  return formatHtml(pageContent);
};
export const getUrlInfoFromCache = async (url) => {
  const cache = await useCache.get(getRelativePath(url));
  return cache;
};
function getNextCrawlDate(lastmod) {
  const now = /* @__PURE__ */ new Date();
  const lastModTime = lastmod ? new Date(lastmod).getTime() || 0 : 0;
  const daysDiff = Math.max(0, (now.getTime() - lastModTime) / (24 * 60 * 60 * 1e3));
  const CRAWL_INTERVALS = /* @__PURE__ */ new Map([
    [[-1, 0], 1],
    // 无 lastmod
    [[0, 3], 1],
    // 3 天内活跃
    [[3, 7], 3],
    // 7 天内活跃
    [[7, 30], 14],
    // 30 天内活跃
    [[30, 90], 30],
    // 90 天内活跃
    [[90, Infinity], 365]
    // 长期不活跃
  ]);
  const interval = Array.from(CRAWL_INTERVALS.entries()).find(
    ([[min, max]]) => lastmod ? daysDiff > min && daysDiff <= max : min === -1
  )?.[1] || 90;
  return new Date(now.getTime() + interval * 24 * 60 * 60 * 1e3).toISOString();
}
export const setUrlInfoToCache = async ({
  url,
  content,
  lastmod,
  nextDate
}) => {
  if (!content || !url) {
    return;
  }
  const lastmodValue = lastmod || (/* @__PURE__ */ new Date()).toISOString();
  return await useCache.set(getRelativePath(url), {
    content,
    lastmod: lastmodValue,
    updatedAt: (/* @__PURE__ */ new Date()).toISOString(),
    nextDate: nextDate || getNextCrawlDate(lastmodValue)
  });
};
export const crawlUrl = async ({
  urls,
  lastmodMap,
  formatPageContent,
  autoCloseBrowserCount = 50
}) => {
  if (typeof urls === "string") {
    urls = [urls];
  }
  const crawlUrlJob = ({ url, retryCount = 0, index }) => {
    return async () => {
      try {
        if (index % autoCloseBrowserCount === 0) {
          await closeBrowser({
            trimCache: false
          });
        }
        const canCrawl = await isAcceptCrawler(url);
        if (canCrawl) {
          const pageContent = await getPageContent({
            url,
            formatPageContent
          });
          if (pageContent) {
            const lastmodValue = lastmodMap?.get(url) || (/* @__PURE__ */ new Date()).toISOString();
            await setUrlInfoToCache({
              url,
              content: pageContent,
              lastmod: lastmodValue,
              nextDate: getNextCrawlDate(lastmodValue)
            });
            logger.info(`Crawler[${index}] ${url} success`);
          } else if (retryCount < 3) {
            retryCount++;
            const timeout = 1e3 * 5 * retryCount;
            logger.info(`Crawler[${index}] ${url} fail, continue after ${timeout}ms`);
            await sleep(timeout);
            crawlQueue.add(crawlUrlJob({ url, retryCount, index }), {
              priority: retryCount
              // Operations with greater priority will be scheduled first.
            });
          } else {
            logger.info(`Crawler[${index}] ${url} fail reach 3 times, skip it`);
          }
        } else {
          logger.info(`Crawler[${index}] ${url} skip`);
        }
        if (global.gc) {
          global.gc();
        }
      } catch (error) {
        logger.info(`Crawler[${index}] ${url} abort by error, skip it: ${error?.message || error?.reason || error}`);
      }
    };
  };
  for (const [index, url] of urls.entries()) {
    await crawlQueue.add(crawlUrlJob({ url, index: index + 1 }));
  }
};
const crawlBlockletRunningMap = /* @__PURE__ */ new Map();
export const crawlBlocklet = async () => {
  const { mountPoint, did } = getComponentInfo();
  if (crawlBlockletRunningMap.has(did) && crawlBlockletRunningMap.get(did)) {
    logger.info(`Crawler blocklet ${did} is running, skip it`);
    return;
  }
  try {
    const browser = await getBrowser();
    if (!browser) {
      throw new Error("No Browser can use");
    }
    logger.info("Crawler blocklet existing can use browser");
  } catch (error) {
    logger.info(`Crawler blocklet abort by error: ${error?.message || error?.reason || error}`);
    return;
  }
  const { appUrl } = env;
  if (!appUrl) {
    throw new Error("appUrl not found");
  }
  const sitemapList = await getSitemapList(appUrl);
  const matchMountPoint = joinURL(appUrl, !mountPoint || mountPoint === "/" ? "" : mountPoint);
  const otherMountPointList = components.filter((item) => item.mountPoint && item.mountPoint !== mountPoint).map((item) => item.mountPoint);
  const blockletLocList = sitemapList.filter((item) => {
    if (mountPoint !== "/") {
      return item?.url?.indexOf(matchMountPoint) > -1;
    }
    return otherMountPointList.every((mountPoint2) => item?.url?.indexOf(mountPoint2) === -1);
  });
  const canUseBlockletLocList = [];
  const lastmodMap = /* @__PURE__ */ new Map();
  let skipBlockletLocTotal = 0;
  let blockletLocTotal = 0;
  await Promise.all(
    blockletLocList.map(async (item) => {
      let tempLocList = [];
      if (item.url) {
        tempLocList.push(item.url);
      }
      if (item?.links?.length > 0) {
        tempLocList.push(...item.links?.map((ytem) => ytem.url));
      }
      blockletLocTotal += tempLocList.length;
      tempLocList = (await Promise.all(
        tempLocList.map(async (loc) => {
          try {
            const { lastmod: cacheLastmod, nextDate } = await useCache.get(getRelativePath(loc));
            if (item.lastmod && new Date(cacheLastmod).getTime() === new Date(item.lastmod).getTime()) {
              skipBlockletLocTotal++;
              return false;
            }
            if (nextDate && new Date(nextDate).getTime() >= (/* @__PURE__ */ new Date()).getTime()) {
              skipBlockletLocTotal++;
              return false;
            }
            return loc;
          } catch (error) {
          }
          return loc;
        })
      )).filter(Boolean);
      tempLocList.forEach((loc) => {
        if (item.lastmod) lastmodMap.set(loc, item.lastmod);
      });
      canUseBlockletLocList.push(...tempLocList);
    })
  );
  const crawlerLogText = (step = "") => [
    `Crawler sitemap.xml about ${did} ${step}: `,
    {
      blockletLocTotal,
      canUseBlockletLocTotal: canUseBlockletLocList.length,
      skipBlockletLocTotal,
      lastmodMapTotal: lastmodMap.size
    }
  ];
  logger.info(...crawlerLogText("start"));
  try {
    crawlBlockletRunningMap.set(did, true);
    await crawlUrl({
      // @ts-ignore
      urls: canUseBlockletLocList,
      lastmodMap,
      formatPageContent: async ({ page, url }) => {
        const pageContent = await page.evaluate(() => {
          const removeElements = (tagName) => {
            const elements = document.querySelectorAll(tagName);
            for (let i = elements.length - 1; i >= 0; i--) {
              try {
                elements[i]?.parentNode?.removeChild(elements[i]);
              } catch (error) {
              }
            }
          };
          removeElements('[id="uploader-container"]');
          removeElements('[class^="uppy-"]');
          removeElements('[id="point-up-component"]');
          const meta = document.createElement("meta");
          meta.name = "blocklet-crawler";
          meta.content = "true";
          document.head.appendChild(meta);
          return document.documentElement.outerHTML;
        });
        return pageContent;
      }
    });
    logger.info(...crawlerLogText("success"));
    await closeBrowser({
      trimCache: true
    });
  } catch (error) {
    logger.info(`Crawler blocklet abort by error`, error);
  } finally {
    crawlBlockletRunningMap.delete(did);
  }
};
const CRON_CRAWL_BLOCKLET_KEY = "cron-crawl-blocklet";
let cronCrawlBlockletJob = null;
export const initCronCrawlBlocklet = ({
  time = "0 0 */12 * * *",
  // every 12 hours
  options
} = {}) => {
  if (!cronCrawlBlockletJob) {
    cronCrawlBlockletJob = Cron.init({
      context: {},
      jobs: [
        {
          name: CRON_CRAWL_BLOCKLET_KEY,
          time,
          fn: debounce(crawlBlocklet),
          options: { runOnInit: false, ...options }
        }
      ],
      onError: (err) => {
        console.error("run job failed", err);
      }
    });
  }
  return cronCrawlBlockletJob;
};
export const cancelCronCrawlBlocklet = () => {
  if (cronCrawlBlockletJob) {
    cronCrawlBlockletJob.jobs[CRON_CRAWL_BLOCKLET_KEY].stop();
    cronCrawlBlockletJob = null;
    crawlQueue.clear();
    logger.info("Cron crawl blocklet stop, clear crawl queue");
  }
};