@just-every/mcp-read-website-fast/dist/utils/urlPolicy.js

Markdown Content Preprocessor - Fetch web pages, extract content, convert to clean Markdown

import { lookup } from 'node:dns/promises';
import { isIP } from 'node:net';
const IPV4_BLOCKED_RANGES = [
    [0x00000000n, 0xff000000n],
    [0x0a000000n, 0xff000000n],
    [0x64400000n, 0xffc00000n],
    [0x7f000000n, 0xff000000n],
    [0xa9fe0000n, 0xffff0000n],
    [0xac100000n, 0xfff00000n],
    [0xc0000000n, 0xffffff00n],
    [0xc0000200n, 0xffffff00n],
    [0xc0586300n, 0xffffff00n],
    [0xc0a80000n, 0xffff0000n],
    [0xc6120000n, 0xfffe0000n],
    [0xc6336400n, 0xffffff00n],
    [0xcb007100n, 0xffffff00n],
    [0xe0000000n, 0xf0000000n],
    [0xf0000000n, 0xf0000000n],
];
const IPV6_BLOCKED_RANGES = [
    [0n, (1n << 128n) - 1n],
    [1n, (1n << 128n) - 1n],
    [0xffffn << 32n, ((1n << 96n) - 1n) << 32n],
    [0x0064ff9b000000000000000000000000n, prefixMask(96)],
    [0x01000000000000000000000000000000n, prefixMask(64)],
    [0x20010000000000000000000000000000n, prefixMask(23)],
    [0x20020000000000000000000000000000n, prefixMask(16)],
    [0xfc000000000000000000000000000000n, prefixMask(7)],
    [0xfe800000000000000000000000000000n, prefixMask(10)],
    [0xff000000000000000000000000000000n, prefixMask(8)],
];
export class UrlPolicyError extends Error {
    constructor(message) {
        super(message);
        this.name = 'UrlPolicyError';
    }
}
export async function assertPublicHttpUrl(url) {
    return (await resolvePublicHttpUrl(url)).url;
}
export async function resolvePublicHttpUrl(url) {
    let parsed;
    try {
        parsed = new URL(url);
    }
    catch {
        throw new UrlPolicyError(`Invalid URL: ${url}`);
    }
    if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
        throw new UrlPolicyError(`URL scheme is not allowed: ${parsed.protocol}`);
    }
    if (parsed.username || parsed.password) {
        throw new UrlPolicyError('URL credentials are not allowed');
    }
    const hostname = normalizeHostname(parsed.hostname);
    const hostIpVersion = isIP(hostname);
    if (hostIpVersion !== 0) {
        const family = ipVersionToFamily(hostIpVersion);
        assertPublicIp(hostname, family);
        return {
            url: parsed,
            hostname,
            addresses: [{ address: hostname, family }],
        };
    }
    let addresses;
    try {
        addresses = await lookup(hostname, { all: true, verbatim: true });
    }
    catch (error) {
        throw new UrlPolicyError(`Unable to resolve hostname "${hostname}": ${error instanceof Error ? error.message : 'Unknown DNS error'}`);
    }
    if (addresses.length === 0) {
        throw new UrlPolicyError(`Hostname "${hostname}" did not resolve`);
    }
    const publicAddresses = addresses.map(address => {
        const family = ipVersionToFamily(address.family);
        assertPublicIp(address.address, family);
        return { address: address.address, family };
    });
    return { url: parsed, hostname, addresses: publicAddresses };
}
function normalizeHostname(hostname) {
    if (hostname.startsWith('[') && hostname.endsWith(']')) {
        return hostname.slice(1, -1);
    }
    return hostname;
}
function assertPublicIp(address, family) {
    if (family === 4) {
        if (isBlockedIpv4(address)) {
            throw new UrlPolicyError(`IP address is not allowed: ${address}`);
        }
        return;
    }
    if (family === 6) {
        const ipv4Mapped = address
            .toLowerCase()
            .match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/);
        if (ipv4Mapped) {
            assertPublicIp(ipv4Mapped[1], 4);
            return;
        }
        if (isBlockedIpv6(address)) {
            throw new UrlPolicyError(`IP address is not allowed: ${address}`);
        }
        return;
    }
    throw new UrlPolicyError(`Unknown IP address family for ${address}`);
}
function ipVersionToFamily(family) {
    if (family === 4 || family === 6) {
        return family;
    }
    throw new UrlPolicyError(`Unknown IP address family: ${family}`);
}
function isBlockedIpv4(address) {
    const numeric = ipv4ToNumber(address);
    return IPV4_BLOCKED_RANGES.some(([range, mask]) => (numeric & mask) === range);
}
function ipv4ToNumber(address) {
    const parts = address.split('.').map(part => Number(part));
    if (parts.length !== 4 ||
        parts.some(part => !Number.isInteger(part) || part < 0 || part > 255)) {
        throw new UrlPolicyError(`Invalid IPv4 address: ${address}`);
    }
    return ((BigInt(parts[0]) << 24n) +
        (BigInt(parts[1]) << 16n) +
        (BigInt(parts[2]) << 8n) +
        BigInt(parts[3]));
}
function isBlockedIpv6(address) {
    const numeric = ipv6ToBigInt(address);
    return IPV6_BLOCKED_RANGES.some(([range, mask]) => (numeric & mask) === range);
}
function ipv6ToBigInt(address) {
    const lower = address.toLowerCase();
    const [head = '', tail = ''] = lower.split('::');
    const headParts = parseIpv6Parts(head);
    const tailParts = parseIpv6Parts(tail);
    const missing = 8 - headParts.length - tailParts.length;
    if (lower.includes('::')) {
        if (missing < 1) {
            throw new UrlPolicyError(`Invalid IPv6 address: ${address}`);
        }
    }
    else if (headParts.length !== 8) {
        throw new UrlPolicyError(`Invalid IPv6 address: ${address}`);
    }
    const parts = lower.includes('::')
        ? [...headParts, ...Array(missing).fill(0), ...tailParts]
        : headParts;
    return parts.reduce((acc, part) => (acc << 16n) + BigInt(part), 0n);
}
function parseIpv6Parts(value) {
    if (!value) {
        return [];
    }
    return value.split(':').map(part => {
        const parsed = Number.parseInt(part, 16);
        if (!part ||
            !Number.isInteger(parsed) ||
            parsed < 0 ||
            parsed > 0xffff) {
            throw new UrlPolicyError(`Invalid IPv6 address part: ${part}`);
        }
        return parsed;
    });
}
function prefixMask(prefixLength) {
    return ((1n << BigInt(prefixLength)) - 1n) << BigInt(128 - prefixLength);
}