@blocklet/crawler/lib/cjs/middlewares/utils.js

blocklet crawler lib

"use strict";

Object.defineProperty(exports, "__esModule", {
  value: true
});
var _exportNames = {
  api: true,
  sleep: true,
  clearCheckBrowserTimer: true,
  closeBrowser: true,
  getBrowser: true,
  CRAWLER_FLAG: true,
  isSelfCrawler: true,
  initPage: true,
  getDefaultRobotsUrl: true,
  getDefaultSitemapUrl: true,
  isAcceptCrawler: true,
  getSitemapList: true,
  isBotUserAgent: true,
  getBlockletPathname: true,
  getComponentInfo: true,
  getFullUrl: true,
  getRelativePath: true,
  cachePool: true,
  withCache: true,
  formatKey: true,
  useCache: true,
  checkBrowserDownloaded: true,
  getPuppeteerrc: true,
  puppeteer: true
};
exports.isSelfCrawler = exports.isBotUserAgent = exports.isAcceptCrawler = exports.initPage = exports.getSitemapList = exports.getRelativePath = exports.getPuppeteerrc = exports.getFullUrl = exports.getDefaultSitemapUrl = exports.getDefaultRobotsUrl = exports.getComponentInfo = exports.getBrowser = exports.getBlockletPathname = exports.formatKey = exports.closeBrowser = exports.clearCheckBrowserTimer = exports.checkBrowserDownloaded = exports.cachePool = exports.api = exports.CRAWLER_FLAG = void 0;
Object.defineProperty(exports, "puppeteer", {
  enumerable: true,
  get: function () {
    return _puppeteer.default;
  }
});
exports.withCache = exports.useCache = exports.sleep = void 0;
var _puppeteer = _interopRequireDefault(require("@blocklet/puppeteer"));
var _ufo = require("ufo");
var _path = require("path");
var _axios = _interopRequireDefault(require("axios"));
var _stream = require("stream");
var _fsExtra = _interopRequireDefault(require("fs-extra"));
var _robotsParser = _interopRequireDefault(require("robots-parser"));
var _redis = require("redis");
var _genericPool = require("generic-pool");
var _sitemap = require("sitemap");
var _flattenDeep = _interopRequireDefault(require("lodash/flattenDeep"));
var _uniq = _interopRequireDefault(require("lodash/uniq"));
var _config = _interopRequireWildcard(require("@blocklet/sdk/lib/config"));
Object.keys(_config).forEach(function (key) {
  if (key === "default" || key === "__esModule") return;
  if (Object.prototype.hasOwnProperty.call(_exportNames, key)) return;
  if (key in exports && exports[key] === _config[key]) return;
  Object.defineProperty(exports, key, {
    enumerable: true,
    get: function () {
      return _config[key];
    }
  });
});
var _util = require("util");
var _child_process = require("child_process");
var _component = require("@blocklet/sdk/lib/component");
function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function (e) { return e ? t : r; })(e); }
function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != typeof e && "function" != typeof e) return { default: e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && {}.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n.default = e, t && t.set(e, n), n; }
function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
const {
  logger
} = _config.default;
const execAsync = (0, _util.promisify)(_child_process.exec);
let browser;
let checkBrowserTimer;
const BROWSER_WS_ENDPOINT_KEY = `browserWSEndpoint-${process.env.BLOCKLET_DID || "unknown"}`;
let cachedRedisUrl = null;
const api = exports.api = _axios.default.create({
  timeout: 1e3 * 10,
  // 10s
  headers: {
    "Content-Type": "application/json"
  }
});
const sleep = ms => {
  return new Promise(resolve => {
    setTimeout(resolve, ms);
  });
};
exports.sleep = sleep;
const clearCheckBrowserTimer = () => {
  if (checkBrowserTimer) {
    clearInterval(checkBrowserTimer);
    checkBrowserTimer = null;
  }
};
exports.clearCheckBrowserTimer = clearCheckBrowserTimer;
const closeBrowser = async ({
  trimCache = true
} = {}) => {
  try {
    if (browser) {
      const pages = await browser.pages().catch(() => []);
      await Promise.all(pages.map(page => page.close().catch(() => {})));
      await browser.close().catch(err => {
        logger.warn("Browser close failed with error:", err);
      });
      browser = null;
      await useCache.remove(BROWSER_WS_ENDPOINT_KEY);
    }
    clearCheckBrowserTimer();
    if (trimCache) {
      await _puppeteer.default.trimCache();
      const {
        cacheDirectory,
        temporaryDirectory
      } = getPuppeteerrc();
      if (cacheDirectory) {
        _fsExtra.default.emptyDirSync(cacheDirectory);
      }
      if (temporaryDirectory) {
        _fsExtra.default.emptyDirSync(temporaryDirectory);
      }
      logger.info("Trim cache success");
    }
    logger.info("Close browser success");
    if (global.gc) {
      global.gc();
    }
  } catch (error) {
    logger.error("Failed to close browser:", error);
  }
};
exports.closeBrowser = closeBrowser;
const getBrowser = async () => {
  if (!browser) {
    await sleep(Math.floor(Math.random() * 1e3 * 5));
    await checkBrowserDownloaded();
    try {
      await withCache(async client => {
        let browserWSEndpoint = await client.get(BROWSER_WS_ENDPOINT_KEY);
        if (!browserWSEndpoint) {
          await sleep(Math.floor(Math.random() * 1e3 * 5));
          browserWSEndpoint = await client.get(BROWSER_WS_ENDPOINT_KEY);
        }
        if (browserWSEndpoint) {
          browser = await _puppeteer.default.connect({
            browserWSEndpoint
          });
          logger.info("Connect browser success");
        } else {
          throw new Error("no browserWSEndpoint to connect");
        }
      });
    } catch (error) {
      try {
        browser = await _puppeteer.default.launch({
          headless: true,
          args: [
          // docs: https://peter.sh/experiments/chromium-command-line-switches/
          "--no-first-run", "--hide-scrollbars", "--no-sandbox", "--no-zygote", "--disable-setuid-sandbox", "--disable-gpu", "--disable-dev-shm-usage", "--disable-site-isolation-trials", "--disable-accelerated-2d-canvas", "--disable-extensions", "--js-flags=--max_old_space_size=512",
          // 限制V8内存
          "--disable-background-networking", "--disable-default-apps",
          // '--disable-web-security', // 允许跨域请求
          "--disable-software-rasterizer", "--disable-crash-reporter", "--disable-service-workers", "--disable-notifications", "--disable-infobars", "--font-render-hinting=none"]
        });
        logger.info("Launch browser success");
        const browserWSEndpoint = await browser.wsEndpoint();
        await withCache(async client => {
          await client.set(BROWSER_WS_ENDPOINT_KEY, browserWSEndpoint);
        });
      } catch (error2) {
        logger.error("launch browser failed: ", error2);
      }
    } finally {
      if (browser) {
        let count = 0;
        checkBrowserTimer = setInterval(async () => {
          if (browser) {
            const pages = await browser.pages().catch(() => []);
            if (pages.length === 1 && pages[0].url() === "about:blank") {
              count++;
              logger.debug(`Browser inactive count: ${count}/3`);
            } else {
              count = 0;
            }
            if (count >= 3) {
              logger.info("Browser inactive for 3 minutes, closing...");
              await closeBrowser({
                trimCache: true
              });
            }
          }
        }, 1e3 * 60);
      }
    }
  }
  if (!browser) {
    throw new Error("No browser to use, should install redis or browser");
  }
  return browser;
};
exports.getBrowser = getBrowser;
const CRAWLER_FLAG = exports.CRAWLER_FLAG = "x-crawler";
const isSelfCrawler = req => {
  const ua = req.get("user-agent") || "";
  return req.get(CRAWLER_FLAG) === "true" || `${ua}`.toLowerCase().indexOf("headless") !== -1;
};
exports.isSelfCrawler = isSelfCrawler;
const initPage = async ({
  abortResourceTypes = ["image",
  // 图片
  "media",
  // 媒体文件
  "font",
  // 字体
  "websocket",
  // websocket 连接
  "manifest",
  // manifest 文件
  "other"
  // 其他资源
  ]
} = {}) => {
  const browser2 = await getBrowser();
  const page = await browser2.newPage();
  await page.setViewport({
    width: 1440,
    height: 900
  });
  await page.setExtraHTTPHeaders({
    [CRAWLER_FLAG]: "true"
  });
  if (abortResourceTypes.length > 0) {
    await page.setRequestInterception(true);
    page.on("request", req => {
      if (abortResourceTypes.includes(req.resourceType())) {
        return req.abort();
      }
      return req.continue();
    });
  }
  return page;
};
exports.initPage = initPage;
const botUserAgents = [/bot/i, /spider/i, /facebookexternalhit/i, /simplepie/i, /yahooseeker/i, /embedly/i, /quora link preview/i, /outbrain/i, /vkshare/i, /monit/i, /Pingability/i, /Monitoring/i, /WinHttpRequest/i, /Apache-HttpClient/i, /getprismatic.com/i, /python-requests/i, /Twurly/i, /yandex/i, /browserproxy/i, /crawler/i, /Qwantify/i, /Yahoo/i, /pinterest/i, /Tumblr/i, /Tumblr Agent/i, /WhatsApp/i, /Google-Structured-Data-Testing-Tool/i, /Google-InspectionTool/i, /Googlebot/i, /GPTBot/i, /Applebot/i,
// AI bots
/Anthropic-ai/i, /Claude-Web/i, /anthropic-ai-scraper/i, /Google-Extended/i, /GoogleOther/i, /CCBot\/\d/i, /Bytespider/i, /BingBot/i, /Baiduspider/i, /Sogou/i, /Perplexity/i, /Cohere-ai/i, /xlts-bot/i, /THAAS/i, /YisouSpider/i, /AlibabaGroup/i, /adaptive-edge-crawler/i];
const isSpider = ua => botUserAgents.some(spider => {
  return spider.test(ua);
});
const staticFileExtensions = ["ai", "avi", "css", "dat", "dmg", "doc", "doc", "exe", "flv", "gif", "ico", "iso", "jpeg", "jpg", "js", "less", "m4a", "m4v", "mov", "mp3", "mp4", "mpeg", "mpg", "pdf", "png", "ppt", "psd", "rar", "rss", "svg", "swf", "tif", "torrent", "ttf", "txt", "wav", "wmv", "woff", "xls", "xml", "zip"];
const getDefaultRobotsUrl = req => {
  const {
    origin
  } = new URL(getFullUrl(req));
  return (0, _ufo.joinURL)(origin, "robots.txt?nocache=1");
};
exports.getDefaultRobotsUrl = getDefaultRobotsUrl;
const getDefaultSitemapUrl = req => {
  const {
    origin
  } = new URL(getFullUrl(req));
  return (0, _ufo.joinURL)(origin, "sitemap.xml?nocache=1");
};
exports.getDefaultSitemapUrl = getDefaultSitemapUrl;
const isAcceptCrawler = async req => {
  let acceptCrawler = true;
  const robotsUrl = getDefaultRobotsUrl(req);
  const {
    data: robotsTxt
  } = await api.get(robotsUrl).catch(() => ({
    data: ""
  }));
  if (robotsTxt) {
    const robots = (0, _robotsParser.default)(robotsUrl, robotsTxt);
    acceptCrawler = !!(await robots.isAllowed(getFullUrl(req)));
  }
  return acceptCrawler;
};
exports.isAcceptCrawler = isAcceptCrawler;
const getSitemapList = async req => {
  const robotsUrl = getDefaultRobotsUrl(req);
  let sitemapUrlList = [getDefaultSitemapUrl(req)];
  const {
    data: robotsTxt
  } = await api.get(robotsUrl).catch(() => ({
    data: ""
  }));
  if (robotsTxt) {
    const robots = (0, _robotsParser.default)(robotsUrl, robotsTxt);
    let robotsTxtSitemapUrlList = (await robots.getSitemaps()) || [];
    if (robotsTxtSitemapUrlList.length > 0) {
      sitemapUrlList = robotsTxtSitemapUrlList;
    }
  }
  const sitemapList = await Promise.all(sitemapUrlList.map(async sitemapUrl => {
    const newUrl = new URL(sitemapUrl);
    newUrl.searchParams.set("nocache", "1");
    sitemapUrl = newUrl.toString();
    const {
      data: sitemapTxt
    } = await api.get(sitemapUrl).catch(() => ({
      data: ""
    }));
    if (sitemapTxt) {
      const stream = _stream.Readable.from([sitemapTxt]);
      const sitemapJson = await (0, _sitemap.parseSitemap)(stream);
      return sitemapJson;
    }
    return [];
  }));
  return (0, _uniq.default)((0, _flattenDeep.default)(sitemapList.filter(Boolean)));
};
exports.getSitemapList = getSitemapList;
const isBotUserAgent = req => {
  const ua = req.get("user-agent");
  const excludeUrlPattern = new RegExp(`\\.(${staticFileExtensions.join("|")})$`, "i");
  if (ua === void 0 || !isSpider(ua) || excludeUrlPattern.test(req.path)) {
    return false;
  }
  return true;
};
exports.isBotUserAgent = isBotUserAgent;
const getBlockletPathname = req => {
  return req.headers["x-path-prefix"] ? (0, _ufo.joinURL)(req.headers["x-path-prefix"], req.originalUrl) : req.originalUrl;
};
exports.getBlockletPathname = getBlockletPathname;
const getComponentInfo = () => {
  const {
    env,
    components
  } = _config.default;
  return components.find(item => item.did === env.componentDid) || {};
};
exports.getComponentInfo = getComponentInfo;
const getFullUrl = req => {
  if (typeof req === "string") {
    return req;
  }
  return (0, _ufo.joinURL)(_config.default.env.appUrl, getBlockletPathname(req));
};
exports.getFullUrl = getFullUrl;
const getRelativePath = url => {
  try {
    return new URL(url).pathname;
  } catch (error) {}
  return url;
};
exports.getRelativePath = getRelativePath;
let ttl = 1e3 * 60 * 60 * 24 * 7;
const MAX_REDIS_RETRY = 3;
async function getDefaultGateway() {
  try {
    const {
      stdout
    } = await execAsync("cat /proc/net/route | grep '^eth0' | head -1 | awk '{print $3}'");
    if (stdout.trim()) {
      const hex = stdout.trim();
      const ip = [parseInt(hex.slice(6, 8), 16), parseInt(hex.slice(4, 6), 16), parseInt(hex.slice(2, 4), 16), parseInt(hex.slice(0, 2), 16)].join(".");
      return ip;
    }
  } catch (err) {}
  return null;
}
async function detectBestRedisUrl() {
  if (cachedRedisUrl) {
    logger.info(`Using cached Redis connection: ${cachedRedisUrl}`);
    return cachedRedisUrl;
  }
  logger.info("Starting to detect best Redis connection...");
  const defaultGateway = await getDefaultGateway();
  const possibleUrls = [
  // environment variable priority
  process.env.REDIS_URL,
  // by blocklet host
  process.env.BLOCKLET_HOST ? `redis://${process.env.BLOCKLET_HOST}:6379` : null,
  // default gateway
  defaultGateway ? `redis://${defaultGateway}:6379` : null,
  // by web component endpoint
  (() => {
    try {
      const endpoint = (0, _component.getComponentWebEndpoint)(_config.default.env.componentDid);
      if (endpoint) {
        const url = new URL(endpoint);
        return `redis://${url.hostname}:6379`;
      }
    } catch (err) {
      logger.warn(`Failed to get component endpoint: ${err.message}`);
    }
    return null;
  })(),
  // common Docker gateway
  "redis://192.168.0.1:6379",
  // Docker special DNS
  "redis://host.docker.internal:6379",
  // local Redis (last try)
  "redis://localhost:6379"].filter(Boolean);
  for (const url of possibleUrls) {
    try {
      logger.debug(`Testing Redis connection: ${url}`);
      const testClient = (0, _redis.createClient)({
        url
      });
      testClient.on("error", () => {});
      await Promise.race([testClient.connect(), new Promise((_, reject) => setTimeout(() => reject(new Error("Connection timeout")), 2e3))]);
      await testClient.ping();
      await testClient.disconnect();
      logger.info(`\u2705 Found available Redis connection: ${url}`);
      cachedRedisUrl = url;
      return url;
    } catch (err) {
      logger.debug(`Redis connection failed ${url}: ${err.message}`);
    }
  }
  logger.warn("\u26A0\uFE0F No available Redis connection found, using default value");
  return "redis://localhost:6379";
}
const cachePool = exports.cachePool = (0, _genericPool.createPool)({
  create: async () => {
    let cache = null;
    try {
      const redisUrl = await detectBestRedisUrl();
      const redisClient = (0, _redis.createClient)({
        url: redisUrl,
        socket: {
          // @ts-ignore
          reconnectStrategy: retries => {
            if (retries >= MAX_REDIS_RETRY) {
              return new Error("Retry Time Exhausted");
            }
            return Math.min(retries * 500, 1e3 * 3);
          }
        }
      });
      redisClient.on("error", err => logger.warn("Redis Client Error:", err));
      await redisClient.connect();
      logger.info(`Successfully connected to Redis: ${redisUrl}`);
      cache = redisClient;
    } catch (error) {
      logger.warn("Redis connection failed", error);
    }
    return cache;
  },
  destroy: async client => {
    if (client.isReady) {
      await client.quit();
    }
  }
}, {
  max: 2,
  // 2 clients
  min: 0
  // evictionRunIntervalMillis: 0,
});
const withCache = async cb => {
  const client = await cachePool.acquire();
  if (client) {
    try {
      return await cb(client);
    } finally {
      await cachePool.release(client);
    }
  }
};
exports.withCache = withCache;
const cacheKeyPrefix = process.env?.BLOCKLET_REAL_DID ? process.env.BLOCKLET_REAL_DID + ":" : "";
const formatKey = key => {
  return `${cacheKeyPrefix}${key}`;
};
exports.formatKey = formatKey;
const useCache = exports.useCache = {
  get: async key => {
    return withCache(async client => {
      const value = await client.get(formatKey(key));
      try {
        return JSON.parse(value);
      } catch (error) {}
      return value;
    });
  },
  set: async (key, value, options) => {
    return withCache(client => {
      const formatValue = typeof value === "string" ? value : JSON.stringify(value);
      return client.set(formatKey(key), formatValue, {
        PX: ttl,
        ...options
      });
    });
  },
  remove: async key => {
    return withCache(client => {
      return client.del(formatKey(key));
    });
  },
  list: async (key = "*") => {
    return withCache(client => {
      return client.keys(formatKey(key));
    });
  }
};
const checkBrowserDownloaded = async () => {
  try {
    const executablePath = process.env.PUPPETEER_EXECUTABLE_PATH || "/usr/bin/chromium";
    if (_fsExtra.default.existsSync(executablePath)) {
      try {
        logger.info(`System Chromium found and tested successfully: ${executablePath}`);
        return;
      } catch (err) {
        logger.warn(`System Chromium exists but test failed, will try to download`);
      }
    }
    const {
      downloadBrowser
    } = await (async () => {
      try {
        return await Promise.resolve().then(() => require("@blocklet/puppeteer/internal/node/install.js"));
      } catch (err) {
        logger.warn("Skipping browser installation because the Puppeteer build is not available. Run `npm install` again after you have re-built Puppeteer.");
      }
    })();
    if (downloadBrowser) {
      await downloadBrowser();
      logger.info("Browser download completed successfully");
    }
  } catch (error) {
    logger.warn("Browser download failed", error);
  }
};
exports.checkBrowserDownloaded = checkBrowserDownloaded;
const getPuppeteerrc = () => {
  const cacheDir = process.env.BLOCKLET_CACHE_DIR;
  if (_fsExtra.default.existsSync(cacheDir) === false) {
    _fsExtra.default.mkdirSync(cacheDir, {
      recursive: true
    });
  }
  const cacheDirectory = (0, _path.join)(cacheDir, "puppeteer", "cache");
  const temporaryDirectory = (0, _path.join)(cacheDir, "puppeteer", "tmp");
  _fsExtra.default.mkdirSync(cacheDirectory, {
    recursive: true
  });
  _fsExtra.default.mkdirSync(temporaryDirectory, {
    recursive: true
  });
  let config2 = {
    // Changes the cache location for Puppeteer.
    cacheDirectory,
    temporaryDirectory
  };
  try {
    _fsExtra.default.ensureDirSync(cacheDirectory);
    _fsExtra.default.ensureDirSync(temporaryDirectory);
  } catch (error) {
    config2 = {};
  }
  const fileContent = `module.exports = ${JSON.stringify(config2, null, 2)}`;
  const puppeteerrcPath = (0, _path.join)(process.env.BLOCKLET_APP_DIR, ".puppeteerrc.js");
  try {
    const puppeteerrcContent = _fsExtra.default.readFileSync(puppeteerrcPath, "utf-8");
    if (puppeteerrcContent !== fileContent) {
      throw new Error(".puppeteerrc.js not right");
    }
  } catch (error) {
    logger.info("Resolve the .puppeteerrc.js error bug");
    _fsExtra.default.writeFileSync(puppeteerrcPath, fileContent);
  }
  return config2;
};
exports.getPuppeteerrc = getPuppeteerrc;