@just-every/mcp-read-website-fast/dist/index.js

Markdown Content Preprocessor - Fetch web pages, extract content, convert to clean Markdown

#!/usr/bin/env node
import { Command } from 'commander';
import { fetchMarkdown } from './internal/fetchMarkdown.js';
import { secureCrawl } from './internal/secureCrawl.js';
import { readFileSync } from 'fs';
import { fileURLToPath } from 'url';
import { dirname, join } from 'path';
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const packageJson = JSON.parse(readFileSync(join(__dirname, '../package.json'), 'utf-8'));
const program = new Command();
program
    .name('mcp')
    .description('Markdown Content Preprocessor - Extract and convert web content to clean Markdown')
    .version(packageJson.version);
program
    .command('fetch <url>')
    .description('Fetch a URL and convert to Markdown')
    .option('-p, --pages <number>', 'Maximum number of pages to crawl', '1')
    .option('-c, --concurrency <number>', 'Max concurrent requests', '3')
    .option('--no-robots', 'Ignore robots.txt')
    .option('--all-origins', 'Allow cross-origin crawling')
    .option('-u, --user-agent <string>', 'Custom user agent')
    .option('--cache-dir <path>', 'Cache directory', '.cache')
    .option('-t, --timeout <ms>', 'Request timeout in milliseconds', '30000')
    .option('--cookies-file <path>', 'Path to Netscape cookie file for authenticated pages')
    .option('-o, --output <format>', 'Output format: json, markdown, or both', 'markdown')
    .action(async (url, options) => {
    try {
        const pages = parseInt(options.pages, 10);
        const depth = pages > 1 ? 1 : 0;
        const crawlOptions = {
            depth: depth,
            maxConcurrency: parseInt(options.concurrency, 10),
            respectRobots: options.robots,
            sameOriginOnly: !options.allOrigins,
            userAgent: options.userAgent,
            cacheDir: options.cacheDir,
            timeout: parseInt(options.timeout, 10),
        };
        if (options.cookiesFile) {
            crawlOptions.cookiesFile = options.cookiesFile;
        }
        console.error(`Fetching ${url}...`);
        if (options.output === 'json') {
            const results = await secureCrawl(url, crawlOptions);
            console.log(JSON.stringify(results, null, 2));
        }
        else if (options.output === 'markdown') {
            const result = await fetchMarkdown(url, {
                ...crawlOptions,
                maxPages: pages,
            });
            if (result.markdown) {
                console.log(result.markdown);
            }
            if (result.error) {
                console.error(`Error: ${result.error}`);
                if (!result.markdown) {
                    process.exit(1);
                }
            }
        }
        else if (options.output === 'both') {
            const results = await secureCrawl(url, crawlOptions);
            results.forEach((result) => {
                console.log(`\n## URL: ${result.url}\n`);
                if (result.markdown) {
                    console.log(result.markdown);
                }
                if (result.error) {
                    console.error(`${result.markdown ? 'Warning' : 'Error'}: ${result.error}`);
                }
            });
        }
    }
    catch (error) {
        console.error('Error:', error instanceof Error ? error.message : error);
        process.exit(1);
    }
});
program
    .command('clear-cache')
    .description('Clear the cache directory')
    .option('--cache-dir <path>', 'Cache directory', '.cache')
    .action(async (options) => {
    try {
        const { rm } = await import('fs/promises');
        await rm(options.cacheDir, { recursive: true, force: true });
        console.log(`Cache cleared: ${options.cacheDir}`);
    }
    catch (error) {
        console.error('Error clearing cache:', error);
        process.exit(1);
    }
});
program
    .command('serve')
    .description('Run as an MCP server')
    .action(async () => {
    await import('./serve.js');
});
program.parse();