@just-every/mcp-read-website-fast
Version:
Markdown Content Preprocessor - Fetch web pages, extract content, convert to clean Markdown
61 lines (60 loc) • 1.85 kB
JavaScript
import { JSDOM, VirtualConsole } from 'jsdom';
export function htmlToDom(html, url) {
try {
return new JSDOM(html, {
url,
contentType: 'text/html',
includeNodeLocations: false,
runScripts: undefined,
resources: undefined,
pretendToBeVisual: true,
virtualConsole: new VirtualConsole().sendTo(console, {
omitJSDOMErrors: true,
}),
});
}
catch {
try {
return new JSDOM(html, {
url,
contentType: 'text/html',
virtualConsole: new VirtualConsole().sendTo(console, {
omitJSDOMErrors: true,
}),
});
}
catch {
return new JSDOM(`<!DOCTYPE html><html><body>${html}</body></html>`, {
url,
contentType: 'text/html',
virtualConsole: new VirtualConsole().sendTo(console, {
omitJSDOMErrors: true,
}),
});
}
}
}
export function extractLinks(dom) {
const document = dom.window.document;
const links = [];
const baseUrl = dom.window.location.href;
const anchorElements = document.querySelectorAll('a[href]');
anchorElements.forEach(element => {
try {
const href = element.getAttribute('href');
if (!href)
return;
if (href.startsWith('mailto:') ||
href.startsWith('tel:') ||
href.startsWith('javascript:') ||
href.startsWith('#')) {
return;
}
const absoluteUrl = new URL(href, baseUrl).href;
links.push(absoluteUrl);
}
catch {
}
});
return [...new Set(links)];
}