@just-every/mcp-read-website-fast/dist/parser/article.js

Markdown Content Preprocessor - Fetch web pages, extract content, convert to clean Markdown

import { Readability } from '@mozilla/readability';
export function extractArticle(dom) {
    const document = dom.window.document;
    const baseUrl = dom.window.location.href;
    const articleParagraph = document.querySelector('article p');
    const hasStrongArticleIndicators = (document.querySelector('article') !== null &&
        articleParagraph?.textContent &&
        articleParagraph.textContent.length > 200) ||
        document.querySelector('[itemtype*="BlogPosting"]') !== null ||
        document.querySelector('[itemtype*="NewsArticle"]') !== null ||
        document.querySelector('meta[property="article:published_time"]') !==
            null;
    if (hasStrongArticleIndicators) {
        const documentClone = document.cloneNode(true);
        const reader = new Readability(documentClone);
        const article = reader.parse();
        if (article && article.content && article.content.trim().length > 500) {
            return {
                title: article.title || 'Untitled',
                content: article.content || '',
                textContent: article.textContent || '',
                length: article.length || 0,
                excerpt: article.excerpt || '',
                byline: article.byline || null,
                dir: article.dir || null,
                lang: article.lang || null,
                siteName: article.siteName || null,
                publishedTime: article.publishedTime || null,
                baseUrl,
            };
        }
    }
    return extractContentManually(dom);
}
function extractContentManually(dom) {
    try {
        const document = dom.window.document;
        const baseUrl = dom.window.location.href;
        const title = document.querySelector('title')?.textContent ||
            document.querySelector('h1')?.textContent ||
            document
                .querySelector('meta[property="og:title"]')
                ?.getAttribute('content') ||
            document
                .querySelector('meta[name="title"]')
                ?.getAttribute('content') ||
            'Untitled Page';
        const byline = document
            .querySelector('meta[name="author"]')
            ?.getAttribute('content') ||
            document.querySelector('[rel="author"]')?.textContent ||
            document.querySelector('.author')?.textContent ||
            null;
        if (!document.body) {
            const html = document.documentElement?.innerHTML || '';
            return {
                title: title.trim(),
                content: html,
                byline,
                excerpt: '',
                dir: null,
                lang: document.documentElement?.lang || null,
                length: html.length,
                siteName: null,
                textContent: document.documentElement?.textContent || '',
                publishedTime: null,
                baseUrl,
            };
        }
        const contentClone = document.body.cloneNode(true);
        const selectorsToRemove = ['script', 'style', 'noscript', 'template'];
        selectorsToRemove.forEach(selector => {
            try {
                contentClone
                    .querySelectorAll(selector)
                    .forEach(el => el.remove());
            }
            catch {
            }
        });
        const mainContent = contentClone;
        const content = mainContent.innerHTML || mainContent.textContent || '';
        return {
            title: title.trim(),
            content,
            byline,
            excerpt: '',
            dir: null,
            lang: document.documentElement?.lang || null,
            length: content.length,
            siteName: null,
            textContent: mainContent.textContent || '',
            publishedTime: null,
            baseUrl,
        };
    }
    catch (error) {
        console.error('Error in manual extraction:', error);
        return {
            title: 'Error extracting content',
            content: dom.window.document.body?.innerHTML ||
                dom.window.document.documentElement?.innerHTML ||
                '',
            byline: null,
            excerpt: '',
            dir: null,
            lang: null,
            length: 0,
            siteName: null,
            textContent: dom.window.document.body?.textContent || '',
            publishedTime: null,
            baseUrl: dom.window.location.href,
        };
    }
}
export function hasContent(html) {
    const lowerHtml = html.toLowerCase();
    if (lowerHtml.includes('<noscript>') &&
        !lowerHtml.includes('<article') &&
        !lowerHtml.includes('<main')) {
        return false;
    }
    const textContent = html.replace(/<[^>]*>/g, '').trim();
    return textContent.length > 100;
}