@blocklet/crawler
Version:
blocklet crawler lib
37 lines (36 loc) • 1.22 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", {
value: true
});
exports.initSEOMiddleware = void 0;
var _utils = require("./utils");
const initSEOMiddleware = ({
autoReturnHtml = true,
allowCrawler = true
} = {}) => {
return async (req, res, next) => {
const isBot = (0, _utils.isBotUserAgent)(req);
const isSelf = (0, _utils.isSelfCrawler)(req);
if (!isBot || isSelf) {
return next();
}
const fullUrl = (0, _utils.getFullUrl)(req);
const canCrawl = await (0, _utils.isAcceptCrawler)(fullUrl);
const allowCrawlerResult = typeof allowCrawler === "function" ? allowCrawler(req) : allowCrawler;
if (!canCrawl || !allowCrawlerResult) {
return next();
}
const cacheData = await _utils.useCache.get((0, _utils.getRelativePath)(fullUrl));
req.cachedHtml = cacheData?.content || cacheData || null;
req.cachedLastmod = cacheData?.lastmod ? new Date(cacheData?.lastmod).toUTCString() : null;
if (req.cachedLastmod) {
res.setHeader("Last-Modified", req.cachedLastmod);
}
if (autoReturnHtml && req.cachedHtml) {
res.send(req.cachedHtml);
return;
}
next();
};
};
exports.initSEOMiddleware = initSEOMiddleware;