mcp-search-tools/dist/tools/web-page.js

MCP server and client for web search and page viewing tools - DuckDuckGo search and web scraping

import { Website } from '@spider-rs/spider-rs';
import * as cheerio from 'cheerio';
import { z } from 'zod';
import { withRetry } from '../utils/retry.js';
import pLimit from 'p-limit';
export const webPageSchema = z.object({
    urls: z.array(z.string()).describe('URLs of the web pages to fetch'),
    includeImages: z
        .boolean()
        .optional()
        .default(false)
        .describe('Include images in the response'),
    includeLinks: z
        .boolean()
        .optional()
        .default(false)
        .describe('Include links in the response'),
    maxLength: z
        .number()
        .optional()
        .default(8000)
        .describe('Maximum length of content to return'),
    maxRetries: z
        .number()
        .optional()
        .default(3)
        .describe('Maximum number of retry attempts for failed requests'),
    retryDelay: z
        .number()
        .optional()
        .default(1000)
        .describe('Base delay in milliseconds between retry attempts'),
    concurrency: z
        .number()
        .optional()
        .default(5)
        .describe('Maximum number of parallel requests'),
});
export async function webPageTool(input) {
    const limit = pLimit(input.concurrency);
    const tasks = input.urls.map((url) => limit(async () => {
        try {
            return await withRetry(() => fetchWebPage({ ...input, url }), input.maxRetries, input.retryDelay);
        }
        catch (error) {
            return {
                url,
                title: 'Error',
                content: '',
                metadata: {},
                error: error instanceof Error
                    ? error.message
                    : String(error) || 'Unknown error occurred',
            };
        }
    }));
    const settled = await Promise.allSettled(tasks);
    const results = settled.map((r, i) => r.status === 'fulfilled'
        ? r.value
        : {
            url: input.urls[i],
            title: 'Error',
            content: '',
            metadata: {},
            error: r.reason instanceof Error
                ? r.reason.message
                : String(r.reason) || 'Unknown error occurred',
        });
    return {
        content: [
            {
                type: 'text',
                text: JSON.stringify(results, null, 2),
            },
        ],
    };
}
async function fetchWebPage(input) {
    const { url, includeImages, includeLinks, maxLength } = input;
    const w = new Website(url)
        .withChromeIntercept(true, true)
        .withBudget({ '*': 1 })
        .build();
    w.withHeaders({
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
        Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
        'Accept-Language': 'en-US,en;q=0.9',
    });
    await w.scrape();
    const page = w.getPages()[0];
    if (!page?.content) {
        throw new Error(`Failed to fetch content from ${url}`);
    }
    const $ = cheerio.load(page.content);
    // Extract title
    const title = $('title').text().trim() || 'No title found';
    // Remove script and style elements
    $('script, style, nav, footer, aside, .advertisement, .ads').remove();
    // Extract main content
    const contentSelectors = [
        'main',
        'article',
        '[role="main"]',
        '.content',
        '.main-content',
        '.post-content',
        '.entry-content',
        'body',
    ];
    let content = '';
    for (const selector of contentSelectors) {
        const element = $(selector);
        if (element.length > 0) {
            content = element.text().trim();
            break;
        }
    }
    if (!content) {
        content = $('body').text().trim();
    }
    // Truncate content if needed
    if (content.length > maxLength) {
        content = content.substring(0, maxLength) + '...';
    }
    // Extract metadata
    const metadata = {
        description: $('meta[name="description"]').attr('content'),
        keywords: $('meta[name="keywords"]').attr('content'),
        author: $('meta[name="author"]').attr('content'),
        publishDate: $('meta[property="article:published_time"]').attr('content') ||
            $('meta[name="publishdate"]').attr('content'),
    };
    // Extract images if requested
    let images;
    if (includeImages) {
        images = $('img')
            .map((_, img) => {
            const src = $(img).attr('src');
            if (src && !src.startsWith('data:')) {
                try {
                    return new URL(src, url).toString();
                }
                catch {
                    return null;
                }
            }
            return null;
        })
            .get()
            .filter((src) => src !== null);
    }
    // Extract links if requested
    let links;
    if (includeLinks) {
        links = $('a[href]')
            .map((_, a) => {
            const href = $(a).attr('href');
            if (href && !href.startsWith('#') && !href.startsWith('mailto:')) {
                try {
                    return new URL(href, url).toString();
                }
                catch {
                    return null;
                }
            }
            return null;
        })
            .get()
            .filter((href) => href !== null);
    }
    return {
        url,
        title,
        content,
        images,
        links,
        metadata,
    };
}
//# sourceMappingURL=web-page.js.map