@just-every/mcp-read-website-fast/dist/internal/fetchMarkdown.js

Markdown Content Preprocessor - Fetch web pages, extract content, convert to clean Markdown

import { CrawlQueue } from '../crawler/queue.js';
export async function fetchMarkdown(url, options = {}) {
    try {
        const crawlOptions = {
            depth: options.depth ?? 0,
            maxConcurrency: options.maxConcurrency ?? 3,
            respectRobots: options.respectRobots ?? true,
            sameOriginOnly: options.sameOriginOnly ?? true,
            userAgent: options.userAgent,
            cacheDir: options.cacheDir ?? '.cache',
            timeout: options.timeout ?? 30000,
        };
        const queue = new CrawlQueue(crawlOptions);
        await queue.init();
        const results = await queue.crawl(url);
        const mainResult = results[0];
        if (!mainResult) {
            return {
                markdown: '',
                error: 'No results returned',
            };
        }
        return {
            markdown: mainResult.markdown,
            title: mainResult.title,
            links: mainResult.links,
            error: mainResult.error,
        };
    }
    catch (error) {
        return {
            markdown: '',
            error: error instanceof Error ? error.message : 'Unknown error',
        };
    }
}