@blocklet/crawler/lib/es/middlewares/seo.js

blocklet crawler lib

import { isBotUserAgent, getFullUrl, useCache, isAcceptCrawler, isSelfCrawler, getRelativePath } from "./utils.js";
export const initSEOMiddleware = ({ autoReturnHtml = true, allowCrawler = true } = {}) => {
  return async (req, res, next) => {
    const isBot = isBotUserAgent(req);
    const isSelf = isSelfCrawler(req);
    if (!isBot || isSelf) {
      return next();
    }
    const fullUrl = getFullUrl(req);
    const canCrawl = await isAcceptCrawler(fullUrl);
    const allowCrawlerResult = typeof allowCrawler === "function" ? allowCrawler(req) : allowCrawler;
    if (!canCrawl || !allowCrawlerResult) {
      return next();
    }
    const cacheData = await useCache.get(getRelativePath(fullUrl));
    req.cachedHtml = cacheData?.content || cacheData || null;
    req.cachedLastmod = cacheData?.lastmod ? new Date(cacheData?.lastmod).toUTCString() : null;
    if (req.cachedLastmod) {
      res.setHeader("Last-Modified", req.cachedLastmod);
    }
    if (autoReturnHtml && req.cachedHtml) {
      res.send(req.cachedHtml);
      return;
    }
    next();
  };
};