@just-every/mcp-read-website-fast/dist/crawler/robots.d.ts

Markdown Content Preprocessor - Fetch web pages, extract content, convert to clean Markdown

interface RobotsChecker {
    isAllowed(url: string, userAgent?: string): boolean;
    getCrawlDelay(userAgent?: string): number | undefined;
}
export declare function getRobotsChecker(origin: string, userAgent?: string): Promise<RobotsChecker>;
export declare function isAllowedByRobots(url: string, userAgent?: string): Promise<boolean>;
export declare function getCrawlDelay(url: string, userAgent?: string): Promise<number>;
export {};