puppeteer-vision-mcp-server/build/scrapers/content-processor.js

MCP Server for scraping webpages and converting to markdown

import { JSDOM } from 'jsdom';
import { Readability } from '@mozilla/readability';
import sanitizeHtml from 'sanitize-html';
import { configureTurndownService } from '../utils/markdown-formatters.js';
import { markCodeParents } from '../utils/html-helpers.js';
/**
 * Processes HTML content to extract the main content and convert it to Markdown
 * @param htmlContent The raw HTML content to process
 * @returns Markdown formatted content
 */
export async function processHtmlContent(htmlContent) {
    // Create DOM
    const dom = new JSDOM(htmlContent);
    const document = dom.window.document;
    // Mark code blocks to influence Readability scoring
    const preElements = document.querySelectorAll('pre');
    preElements.forEach(pre => {
        // Add classes that Readability considers as content
        pre.classList.add('article-content');
        // Set a very high readable score
        pre.setAttribute('data-readable-content-score', '100');
        // Make parent more likely to be kept
        if (pre.parentElement) {
            pre.parentElement.classList.add('article-content');
        }
    });
    document.querySelectorAll('pre, code').forEach(pre => {
        markCodeParents(pre.parentElement);
    });
    // Modify Readability options to be more lenient
    const readerOptions = {
        charThreshold: 20,
        classesToPreserve: ['article-content'],
    };
    // Now run Readability
    const reader = new Readability(document, readerOptions);
    const article = reader.parse();
    // Continue with sanitization and markdown conversion
    if (!article) {
        throw new Error('Failed to parse the article content.');
    }
    const cleanHtml = sanitizeHtml(article.content, {
        allowedTags: [
            'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'a', 'ul', 'ol',
            'li', 'b', 'i', 'strong', 'em', 'code', 'pre',
            'div', 'span', 'table', 'thead', 'tbody', 'tr', 'th', 'td'
        ],
        allowedAttributes: {
            'a': ['href'],
            'pre': ['class', 'data-language'],
            'code': ['class', 'data-language'],
            'div': ['class'],
            'span': ['class']
        }
    });
    const turndownService = configureTurndownService();
    const markdown = turndownService.turndown(cleanHtml);
    return markdown;
}