@blocklet/crawler/lib/es/middlewares/utils.js

blocklet crawler lib

import puppeteer from "@blocklet/puppeteer";
import { joinURL } from "ufo";
import { join } from "path";
import axios from "axios";
import { Readable } from "stream";
import fs from "fs-extra";
import robotsParser from "robots-parser";
import { createClient } from "redis";
import { createPool } from "generic-pool";
import { parseSitemap } from "sitemap";
import flattenDeep from "lodash/flattenDeep";
import uniq from "lodash/uniq";
import config from "@blocklet/sdk/lib/config";
import { promisify } from "util";
import { exec } from "child_process";
import { getComponentWebEndpoint } from "@blocklet/sdk/lib/component";
export * from "@blocklet/sdk/lib/config";
const { logger } = config;
const execAsync = promisify(exec);
let browser;
let checkBrowserTimer;
const BROWSER_WS_ENDPOINT_KEY = `browserWSEndpoint-${process.env.BLOCKLET_DID || "unknown"}`;
let cachedRedisUrl = null;
export const api = axios.create({
  timeout: 1e3 * 10,
  // 10s
  headers: {
    "Content-Type": "application/json"
  }
});
export const sleep = (ms) => {
  return new Promise((resolve) => {
    setTimeout(resolve, ms);
  });
};
export const clearCheckBrowserTimer = () => {
  if (checkBrowserTimer) {
    clearInterval(checkBrowserTimer);
    checkBrowserTimer = null;
  }
};
export const closeBrowser = async ({ trimCache = true } = {}) => {
  try {
    if (browser) {
      const pages = await browser.pages().catch(() => []);
      await Promise.all(pages.map((page) => page.close().catch(() => {
      })));
      await browser.close().catch((err) => {
        logger.warn("Browser close failed with error:", err);
      });
      browser = null;
      await useCache.remove(BROWSER_WS_ENDPOINT_KEY);
    }
    clearCheckBrowserTimer();
    if (trimCache) {
      await puppeteer.trimCache();
      const { cacheDirectory, temporaryDirectory } = getPuppeteerrc();
      if (cacheDirectory) {
        fs.emptyDirSync(cacheDirectory);
      }
      if (temporaryDirectory) {
        fs.emptyDirSync(temporaryDirectory);
      }
      logger.info("Trim cache success");
    }
    logger.info("Close browser success");
    if (global.gc) {
      global.gc();
    }
  } catch (error) {
    logger.error("Failed to close browser:", error);
  }
};
export const getBrowser = async () => {
  if (!browser) {
    await sleep(Math.floor(Math.random() * 1e3 * 5));
    await checkBrowserDownloaded();
    try {
      await withCache(async (client) => {
        let browserWSEndpoint = await client.get(BROWSER_WS_ENDPOINT_KEY);
        if (!browserWSEndpoint) {
          await sleep(Math.floor(Math.random() * 1e3 * 5));
          browserWSEndpoint = await client.get(BROWSER_WS_ENDPOINT_KEY);
        }
        if (browserWSEndpoint) {
          browser = await puppeteer.connect({
            browserWSEndpoint
          });
          logger.info("Connect browser success");
        } else {
          throw new Error("no browserWSEndpoint to connect");
        }
      });
    } catch (error) {
      try {
        browser = await puppeteer.launch({
          headless: true,
          args: [
            // docs: https://peter.sh/experiments/chromium-command-line-switches/
            "--no-first-run",
            "--hide-scrollbars",
            "--no-sandbox",
            "--no-zygote",
            "--disable-setuid-sandbox",
            "--disable-gpu",
            "--disable-dev-shm-usage",
            "--disable-site-isolation-trials",
            "--disable-accelerated-2d-canvas",
            "--disable-extensions",
            "--js-flags=--max_old_space_size=512",
            // 限制V8内存
            "--disable-background-networking",
            "--disable-default-apps",
            // '--disable-web-security', // 允许跨域请求
            "--disable-software-rasterizer",
            "--disable-crash-reporter",
            "--disable-service-workers",
            "--disable-notifications",
            "--disable-infobars",
            "--font-render-hinting=none"
          ]
        });
        logger.info("Launch browser success");
        const browserWSEndpoint = await browser.wsEndpoint();
        await withCache(async (client) => {
          await client.set(BROWSER_WS_ENDPOINT_KEY, browserWSEndpoint);
        });
      } catch (error2) {
        logger.error("launch browser failed: ", error2);
      }
    } finally {
      if (browser) {
        let count = 0;
        checkBrowserTimer = setInterval(async () => {
          if (browser) {
            const pages = await browser.pages().catch(() => []);
            if (pages.length === 1 && pages[0].url() === "about:blank") {
              count++;
              logger.debug(`Browser inactive count: ${count}/3`);
            } else {
              count = 0;
            }
            if (count >= 3) {
              logger.info("Browser inactive for 3 minutes, closing...");
              await closeBrowser({
                trimCache: true
              });
            }
          }
        }, 1e3 * 60);
      }
    }
  }
  if (!browser) {
    throw new Error("No browser to use, should install redis or browser");
  }
  return browser;
};
export const CRAWLER_FLAG = "x-crawler";
export const isSelfCrawler = (req) => {
  const ua = req.get("user-agent") || "";
  return req.get(CRAWLER_FLAG) === "true" || `${ua}`.toLowerCase().indexOf("headless") !== -1;
};
export const initPage = async ({
  abortResourceTypes = [
    "image",
    // 图片
    "media",
    // 媒体文件
    "font",
    // 字体
    "websocket",
    // websocket 连接
    "manifest",
    // manifest 文件
    "other"
    // 其他资源
  ]
} = {}) => {
  const browser2 = await getBrowser();
  const page = await browser2.newPage();
  await page.setViewport({ width: 1440, height: 900 });
  await page.setExtraHTTPHeaders({
    [CRAWLER_FLAG]: "true"
  });
  if (abortResourceTypes.length > 0) {
    await page.setRequestInterception(true);
    page.on("request", (req) => {
      if (abortResourceTypes.includes(req.resourceType())) {
        return req.abort();
      }
      return req.continue();
    });
  }
  return page;
};
const botUserAgents = [
  /bot/i,
  /spider/i,
  /facebookexternalhit/i,
  /simplepie/i,
  /yahooseeker/i,
  /embedly/i,
  /quora link preview/i,
  /outbrain/i,
  /vkshare/i,
  /monit/i,
  /Pingability/i,
  /Monitoring/i,
  /WinHttpRequest/i,
  /Apache-HttpClient/i,
  /getprismatic.com/i,
  /python-requests/i,
  /Twurly/i,
  /yandex/i,
  /browserproxy/i,
  /crawler/i,
  /Qwantify/i,
  /Yahoo/i,
  /pinterest/i,
  /Tumblr/i,
  /Tumblr Agent/i,
  /WhatsApp/i,
  /Google-Structured-Data-Testing-Tool/i,
  /Google-InspectionTool/i,
  /Googlebot/i,
  /GPTBot/i,
  /Applebot/i,
  // AI bots
  /Anthropic-ai/i,
  /Claude-Web/i,
  /anthropic-ai-scraper/i,
  /Google-Extended/i,
  /GoogleOther/i,
  /CCBot\/\d/i,
  /Bytespider/i,
  /BingBot/i,
  /Baiduspider/i,
  /Sogou/i,
  /Perplexity/i,
  /Cohere-ai/i,
  /xlts-bot/i,
  /THAAS/i,
  /YisouSpider/i,
  /AlibabaGroup/i,
  /adaptive-edge-crawler/i
];
const isSpider = (ua) => botUserAgents.some((spider) => {
  return spider.test(ua);
});
const staticFileExtensions = [
  "ai",
  "avi",
  "css",
  "dat",
  "dmg",
  "doc",
  "doc",
  "exe",
  "flv",
  "gif",
  "ico",
  "iso",
  "jpeg",
  "jpg",
  "js",
  "less",
  "m4a",
  "m4v",
  "mov",
  "mp3",
  "mp4",
  "mpeg",
  "mpg",
  "pdf",
  "png",
  "ppt",
  "psd",
  "rar",
  "rss",
  "svg",
  "swf",
  "tif",
  "torrent",
  "ttf",
  "txt",
  "wav",
  "wmv",
  "woff",
  "xls",
  "xml",
  "zip"
];
export const getDefaultRobotsUrl = (req) => {
  const { origin } = new URL(getFullUrl(req));
  return joinURL(origin, "robots.txt?nocache=1");
};
export const getDefaultSitemapUrl = (req) => {
  const { origin } = new URL(getFullUrl(req));
  return joinURL(origin, "sitemap.xml?nocache=1");
};
export const isAcceptCrawler = async (req) => {
  let acceptCrawler = true;
  const robotsUrl = getDefaultRobotsUrl(req);
  const { data: robotsTxt } = await api.get(robotsUrl).catch(() => ({
    data: ""
  }));
  if (robotsTxt) {
    const robots = robotsParser(robotsUrl, robotsTxt);
    acceptCrawler = !!await robots.isAllowed(getFullUrl(req));
  }
  return acceptCrawler;
};
export const getSitemapList = async (req) => {
  const robotsUrl = getDefaultRobotsUrl(req);
  let sitemapUrlList = [getDefaultSitemapUrl(req)];
  const { data: robotsTxt } = await api.get(robotsUrl).catch(() => ({
    data: ""
  }));
  if (robotsTxt) {
    const robots = robotsParser(robotsUrl, robotsTxt);
    let robotsTxtSitemapUrlList = await robots.getSitemaps() || [];
    if (robotsTxtSitemapUrlList.length > 0) {
      sitemapUrlList = robotsTxtSitemapUrlList;
    }
  }
  const sitemapList = await Promise.all(
    sitemapUrlList.map(async (sitemapUrl) => {
      const newUrl = new URL(sitemapUrl);
      newUrl.searchParams.set("nocache", "1");
      sitemapUrl = newUrl.toString();
      const { data: sitemapTxt } = await api.get(sitemapUrl).catch(() => ({
        data: ""
      }));
      if (sitemapTxt) {
        const stream = Readable.from([sitemapTxt]);
        const sitemapJson = await parseSitemap(stream);
        return sitemapJson;
      }
      return [];
    })
  );
  return uniq(flattenDeep(sitemapList.filter(Boolean)));
};
export const isBotUserAgent = (req) => {
  const ua = req.get("user-agent");
  const excludeUrlPattern = new RegExp(`\\.(${staticFileExtensions.join("|")})$`, "i");
  if (ua === void 0 || !isSpider(ua) || excludeUrlPattern.test(req.path)) {
    return false;
  }
  return true;
};
export const getBlockletPathname = (req) => {
  return req.headers["x-path-prefix"] ? joinURL(req.headers["x-path-prefix"], req.originalUrl) : req.originalUrl;
};
export const getComponentInfo = () => {
  const { env, components } = config;
  return components.find((item) => item.did === env.componentDid) || {};
};
export const getFullUrl = (req) => {
  if (typeof req === "string") {
    return req;
  }
  return joinURL(config.env.appUrl, getBlockletPathname(req));
};
export const getRelativePath = (url) => {
  try {
    return new URL(url).pathname;
  } catch (error) {
  }
  return url;
};
let ttl = 1e3 * 60 * 60 * 24 * 7;
const MAX_REDIS_RETRY = 3;
async function getDefaultGateway() {
  try {
    const { stdout } = await execAsync("cat /proc/net/route | grep '^eth0' | head -1 | awk '{print $3}'");
    if (stdout.trim()) {
      const hex = stdout.trim();
      const ip = [
        parseInt(hex.slice(6, 8), 16),
        parseInt(hex.slice(4, 6), 16),
        parseInt(hex.slice(2, 4), 16),
        parseInt(hex.slice(0, 2), 16)
      ].join(".");
      return ip;
    }
  } catch (err) {
  }
  return null;
}
async function detectBestRedisUrl() {
  if (cachedRedisUrl) {
    logger.info(`Using cached Redis connection: ${cachedRedisUrl}`);
    return cachedRedisUrl;
  }
  logger.info("Starting to detect best Redis connection...");
  const defaultGateway = await getDefaultGateway();
  const possibleUrls = [
    // environment variable priority
    process.env.REDIS_URL,
    // by blocklet host
    process.env.BLOCKLET_HOST ? `redis://${process.env.BLOCKLET_HOST}:6379` : null,
    // default gateway
    defaultGateway ? `redis://${defaultGateway}:6379` : null,
    // by web component endpoint
    (() => {
      try {
        const endpoint = getComponentWebEndpoint(config.env.componentDid);
        if (endpoint) {
          const url = new URL(endpoint);
          return `redis://${url.hostname}:6379`;
        }
      } catch (err) {
        logger.warn(`Failed to get component endpoint: ${err.message}`);
      }
      return null;
    })(),
    // common Docker gateway
    "redis://192.168.0.1:6379",
    // Docker special DNS
    "redis://host.docker.internal:6379",
    // local Redis (last try)
    "redis://localhost:6379"
  ].filter(Boolean);
  for (const url of possibleUrls) {
    try {
      logger.debug(`Testing Redis connection: ${url}`);
      const testClient = createClient({ url });
      testClient.on("error", () => {
      });
      await Promise.race([
        testClient.connect(),
        new Promise((_, reject) => setTimeout(() => reject(new Error("Connection timeout")), 2e3))
      ]);
      await testClient.ping();
      await testClient.disconnect();
      logger.info(`\u2705 Found available Redis connection: ${url}`);
      cachedRedisUrl = url;
      return url;
    } catch (err) {
      logger.debug(`Redis connection failed ${url}: ${err.message}`);
    }
  }
  logger.warn("\u26A0\uFE0F No available Redis connection found, using default value");
  return "redis://localhost:6379";
}
export const cachePool = createPool(
  {
    create: async () => {
      let cache = null;
      try {
        const redisUrl = await detectBestRedisUrl();
        const redisClient = createClient({
          url: redisUrl,
          socket: {
            // @ts-ignore
            reconnectStrategy: (retries) => {
              if (retries >= MAX_REDIS_RETRY) {
                return new Error("Retry Time Exhausted");
              }
              return Math.min(retries * 500, 1e3 * 3);
            }
          }
        });
        redisClient.on("error", (err) => logger.warn("Redis Client Error:", err));
        await redisClient.connect();
        logger.info(`Successfully connected to Redis: ${redisUrl}`);
        cache = redisClient;
      } catch (error) {
        logger.warn("Redis connection failed", error);
      }
      return cache;
    },
    destroy: async (client) => {
      if (client.isReady) {
        await client.quit();
      }
    }
  },
  {
    max: 2,
    // 2 clients
    min: 0
    // evictionRunIntervalMillis: 0,
  }
);
export const withCache = async (cb) => {
  const client = await cachePool.acquire();
  if (client) {
    try {
      return await cb(client);
    } finally {
      await cachePool.release(client);
    }
  }
};
const cacheKeyPrefix = process.env?.BLOCKLET_REAL_DID ? process.env.BLOCKLET_REAL_DID + ":" : "";
export const formatKey = (key) => {
  return `${cacheKeyPrefix}${key}`;
};
export const useCache = {
  get: async (key) => {
    return withCache(async (client) => {
      const value = await client.get(formatKey(key));
      try {
        return JSON.parse(value);
      } catch (error) {
      }
      return value;
    });
  },
  set: async (key, value, options) => {
    return withCache((client) => {
      const formatValue = typeof value === "string" ? value : JSON.stringify(value);
      return client.set(formatKey(key), formatValue, { PX: ttl, ...options });
    });
  },
  remove: async (key) => {
    return withCache((client) => {
      return client.del(formatKey(key));
    });
  },
  list: async (key = "*") => {
    return withCache((client) => {
      return client.keys(formatKey(key));
    });
  }
};
export const checkBrowserDownloaded = async () => {
  try {
    const executablePath = process.env.PUPPETEER_EXECUTABLE_PATH || "/usr/bin/chromium";
    if (fs.existsSync(executablePath)) {
      try {
        logger.info(`System Chromium found and tested successfully: ${executablePath}`);
        return;
      } catch (err) {
        logger.warn(`System Chromium exists but test failed, will try to download`);
      }
    }
    const { downloadBrowser } = await (async () => {
      try {
        return await import("@blocklet/puppeteer/internal/node/install.js");
      } catch (err) {
        logger.warn(
          "Skipping browser installation because the Puppeteer build is not available. Run `npm install` again after you have re-built Puppeteer."
        );
      }
    })();
    if (downloadBrowser) {
      await downloadBrowser();
      logger.info("Browser download completed successfully");
    }
  } catch (error) {
    logger.warn("Browser download failed", error);
  }
};
export const getPuppeteerrc = () => {
  const cacheDir = process.env.BLOCKLET_CACHE_DIR;
  if (fs.existsSync(cacheDir) === false) {
    fs.mkdirSync(cacheDir, { recursive: true });
  }
  const cacheDirectory = join(cacheDir, "puppeteer", "cache");
  const temporaryDirectory = join(cacheDir, "puppeteer", "tmp");
  fs.mkdirSync(cacheDirectory, { recursive: true });
  fs.mkdirSync(temporaryDirectory, { recursive: true });
  let config2 = {
    // Changes the cache location for Puppeteer.
    cacheDirectory,
    temporaryDirectory
  };
  try {
    fs.ensureDirSync(cacheDirectory);
    fs.ensureDirSync(temporaryDirectory);
  } catch (error) {
    config2 = {};
  }
  const fileContent = `module.exports = ${JSON.stringify(config2, null, 2)}`;
  const puppeteerrcPath = join(process.env.BLOCKLET_APP_DIR, ".puppeteerrc.js");
  try {
    const puppeteerrcContent = fs.readFileSync(puppeteerrcPath, "utf-8");
    if (puppeteerrcContent !== fileContent) {
      throw new Error(".puppeteerrc.js not right");
    }
  } catch (error) {
    logger.info("Resolve the .puppeteerrc.js error bug");
    fs.writeFileSync(puppeteerrcPath, fileContent);
  }
  return config2;
};
export { puppeteer };