@just-every/mcp-read-website-fast/dist/parser/markdown.js

Markdown Content Preprocessor - Fetch web pages, extract content, convert to clean Markdown

import TurndownService from 'turndown';
import { gfm } from 'turndown-plugin-gfm';
import { JSDOM } from 'jsdom';
function convertRelativeUrls(html, baseUrl) {
    try {
        const dom = new JSDOM(html, { url: baseUrl });
        const document = dom.window.document;
        document.querySelectorAll('a[href]').forEach(link => {
            const href = link.getAttribute('href');
            if (href &&
                !href.startsWith('http://') &&
                !href.startsWith('https://') &&
                !href.startsWith('//') &&
                !href.startsWith('mailto:') &&
                !href.startsWith('tel:') &&
                !href.startsWith('javascript:') &&
                !href.startsWith('#')) {
                try {
                    const absoluteUrl = new URL(href, baseUrl).href;
                    link.setAttribute('href', absoluteUrl);
                }
                catch {
                }
            }
        });
        document.querySelectorAll('img[src]').forEach(img => {
            const src = img.getAttribute('src');
            if (src &&
                !src.startsWith('http://') &&
                !src.startsWith('https://') &&
                !src.startsWith('//') &&
                !src.startsWith('data:')) {
                try {
                    const absoluteUrl = new URL(src, baseUrl).href;
                    img.setAttribute('src', absoluteUrl);
                }
                catch {
                }
            }
        });
        const bodyElement = document.body || document.documentElement;
        return bodyElement ? bodyElement.innerHTML : html;
    }
    catch {
        return html;
    }
}
export function createTurndownService() {
    const turndown = new TurndownService({
        headingStyle: 'atx',
        codeBlockStyle: 'fenced',
        linkStyle: 'inlined',
        emDelimiter: '_',
        bulletListMarker: '-',
        strongDelimiter: '**',
        hr: '---',
        blankReplacement: (_content, node) => {
            return node.isBlock ? '\n\n' : '';
        },
        keepReplacement: (content, node) => {
            return node.isBlock ? '\n\n' + content + '\n\n' : content;
        },
        defaultReplacement: (content, node) => {
            return node.isBlock ? '\n\n' + content + '\n\n' : content;
        },
    });
    turndown.use(gfm);
    turndown.addRule('media', {
        filter: ['iframe', 'video', 'audio', 'embed'],
        replacement: (_content, node) => {
            const element = node;
            const src = element.getAttribute('src') || element.getAttribute('data-src');
            const title = element.getAttribute('title') ||
                element.getAttribute('alt') ||
                'media';
            if (src) {
                return `\n\n[${title}](${src})\n\n`;
            }
            return '';
        },
    });
    turndown.addRule('figure', {
        filter: 'figure',
        replacement: (content, node) => {
            const figure = node;
            const caption = figure.querySelector('figcaption');
            if (caption) {
                const captionText = caption.textContent || '';
                return `\n\n${content.trim()}\n*${captionText}*\n\n`;
            }
            return `\n\n${content.trim()}\n\n`;
        },
    });
    return turndown;
}
export function htmlToMarkdown(html) {
    const turndown = createTurndownService();
    let markdown = turndown.turndown(html);
    markdown = markdown
        .replace(/\n{3,}/g, '\n\n')
        .replace(/\s+$/gm, '')
        .trim();
    return markdown;
}
export function formatArticleMarkdown(article) {
    try {
        const turndown = createTurndownService();
        let markdown = '';
        if (article.title && article.title.trim()) {
            markdown = `# ${article.title}\n\n`;
        }
        if (article.byline) {
            markdown += `*By ${article.byline}*\n\n---\n\n`;
        }
        try {
            const processedContent = article.baseUrl
                ? convertRelativeUrls(article.content, article.baseUrl)
                : article.content;
            markdown += turndown.turndown(processedContent);
        }
        catch (conversionError) {
            console.error('Error converting HTML to markdown:', conversionError);
            const tempDiv = typeof document !== 'undefined'
                ? document.createElement('div')
                : null;
            if (tempDiv) {
                tempDiv.innerHTML = article.content;
                markdown += tempDiv.textContent || article.content;
            }
            else {
                markdown += article.content
                    .replace(/<[^>]*>/g, ' ')
                    .replace(/\s+/g, ' ');
            }
        }
        return markdown
            .replace(/\n{3,}/g, '\n\n')
            .replace(/\s+$/gm, '')
            .trim();
    }
    catch (error) {
        console.error('Fatal error in formatArticleMarkdown:', error);
        return article.title
            ? `# ${article.title}\n\n[Content extraction failed]`
            : '[Content extraction failed]';
    }
}