@just-every/mcp-read-website-fast/dist/crawler/robots.js

Markdown Content Preprocessor - Fetch web pages, extract content, convert to clean Markdown

import { fetchStream } from './fetch.js';
const robotsCache = new Map();
export async function getRobotsChecker(origin, userAgent = '*') {
    const cached = robotsCache.get(origin);
    if (cached)
        return cached;
    try {
        const robotsUrl = new URL('/robots.txt', origin).href;
        const robotsTxt = await fetchStream(robotsUrl, {
            timeout: 5000,
            userAgent,
        });
        const robotsParserModule = (await import('robots-parser'));
        const robotsParser = robotsParserModule.default || robotsParserModule;
        const robots = robotsParser(robotsUrl, robotsTxt);
        robotsCache.set(origin, robots);
        return robots;
    }
    catch {
        const permissive = {
            isAllowed: () => true,
            getCrawlDelay: () => undefined,
        };
        robotsCache.set(origin, permissive);
        return permissive;
    }
}
export async function isAllowedByRobots(url, userAgent = '*') {
    try {
        const { origin } = new URL(url);
        const checker = await getRobotsChecker(origin, userAgent);
        return checker.isAllowed(url, userAgent);
    }
    catch {
        return true;
    }
}
export async function getCrawlDelay(url, userAgent = '*') {
    try {
        const { origin } = new URL(url);
        const checker = await getRobotsChecker(origin, userAgent);
        return checker.getCrawlDelay(userAgent) || 0;
    }
    catch {
        return 0;
    }
}