link-preview-js
Version:
Javascript module to extract and fetch HTTP link information from blocks of text.
341 lines (340 loc) • 16 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.getPreviewFromContent = void 0;
exports.getLinkPreview = getLinkPreview;
const undici_1 = require("undici");
const constants_1 = require("./constants");
const shared_1 = require("./shared");
var shared_2 = require("./shared");
Object.defineProperty(exports, "getPreviewFromContent", { enumerable: true, get: function () { return shared_2.getPreviewFromContent; } });
function parseIPv4Address(address) {
const parts = address.split(".");
if (parts.length !== 4) {
return undefined;
}
const octets = parts.map((part) => Number(part));
if (octets.some((octet) => !Number.isInteger(octet) || octet < 0 || octet > 255)) {
return undefined;
}
return octets;
}
function parseIPv6Hextets(address) {
const addressWithoutZone = address.split("%")[0];
const ipv4Match = addressWithoutZone.match(/(.+:)(\d{1,3}(?:\.\d{1,3}){3})$/);
let normalizedAddress = addressWithoutZone;
if (ipv4Match) {
const octets = parseIPv4Address(ipv4Match[2]);
if (!octets) {
return undefined;
}
normalizedAddress =
ipv4Match[1] +
[
((octets[0] << 8) | octets[1]).toString(16),
((octets[2] << 8) | octets[3]).toString(16),
].join(":");
}
const compressedParts = normalizedAddress.toLowerCase().split("::");
if (compressedParts.length > 2) {
return undefined;
}
const left = compressedParts[0] ? compressedParts[0].split(":") : [];
const right = compressedParts[1] ? compressedParts[1].split(":") : [];
const missing = 8 - left.length - right.length;
if (missing < 0 || (compressedParts.length === 1 && missing !== 0)) {
return undefined;
}
const hextets = [...left, ...Array(missing).fill("0"), ...right].map((part) => {
if (!/^[\da-f]{1,4}$/i.test(part)) {
return undefined;
}
return Number.parseInt(part, 16);
});
if (hextets.some((part) => part === undefined)) {
return undefined;
}
return hextets;
}
function ipv4FromHextets(high, low) {
return `${high >> 8}.${high & 255}.${low >> 8}.${low & 255}`;
}
function getEmbeddedIPv4Address(address) {
const hextets = parseIPv6Hextets(address);
if (!hextets) {
return undefined;
}
const firstFiveAreZero = hextets.slice(0, 5).every((part) => part === 0);
const firstSixAreZero = firstFiveAreZero && hextets[5] === 0;
const isIPv4Mapped = firstFiveAreZero && hextets[5] === 0xffff;
const isNat64WellKnownPrefix = hextets[0] === 0x64 && hextets[1] === 0xff9b && hextets.slice(2, 6).every((part) => part === 0);
if (firstSixAreZero || isIPv4Mapped || isNat64WellKnownPrefix) {
return ipv4FromHextets(hextets[6], hextets[7]);
}
if (hextets[0] === 0x2002) {
return ipv4FromHextets(hextets[1], hextets[2]);
}
return undefined;
}
function isIPv4AddressBlocked(address) {
const octets = parseIPv4Address(address);
if (!octets) {
return false;
}
const [first, second, third] = octets;
return (first === 0 ||
first === 10 ||
first === 127 ||
(first === 100 && second >= 64 && second <= 127) ||
(first === 169 && second === 254) ||
(first === 172 && second >= 16 && second <= 31) ||
(first === 192 && second === 168) ||
(first === 192 && second === 0 && third === 2) ||
(first === 198 && second === 51 && third === 100) ||
(first === 203 && second === 0 && third === 113) ||
first >= 224);
}
function isIPv6AddressBlocked(address) {
const hextets = parseIPv6Hextets(address);
if (!hextets) {
return false;
}
const embeddedIPv4Address = getEmbeddedIPv4Address(address);
return (hextets.every((part) => part === 0) ||
(hextets.slice(0, 7).every((part) => part === 0) && hextets[7] === 1) ||
(hextets[0] & 0xfe00) === 0xfc00 ||
(hextets[0] & 0xffc0) === 0xfe80 ||
Boolean(embeddedIPv4Address && isIPv4AddressBlocked(embeddedIPv4Address)));
}
function stripIPv6Brackets(address) {
if (address.startsWith("[") && address.endsWith("]")) {
return address.slice(1, -1);
}
return address;
}
function normalizeIPAddress(address) {
const normalizedAddress = stripIPv6Brackets(address.trim()).split("%")[0];
if (parseIPv4Address(normalizedAddress)) {
return normalizedAddress;
}
if (parseIPv6Hextets(normalizedAddress)) {
return normalizedAddress.toLowerCase();
}
return undefined;
}
function getResolvedAddress(addressOrUrl) {
const trimmedAddress = addressOrUrl.trim();
if (trimmedAddress.startsWith("http://") || trimmedAddress.startsWith("https://")) {
return normalizeIPAddress(new URL(trimmedAddress).hostname);
}
return normalizeIPAddress(trimmedAddress);
}
function throwOnLoopback(addressOrUrl) {
const normalizedAddress = getResolvedAddress(addressOrUrl);
if (normalizedAddress) {
if (isIPv4AddressBlocked(normalizedAddress) || isIPv6AddressBlocked(normalizedAddress)) {
throw new Error("SSRF request detected, trying to query host");
}
return;
}
if (constants_1.CONSTANTS.REGEX_LOOPBACK.test(addressOrUrl.trim())) {
throw new Error("SSRF request detected, trying to query host");
}
}
function formatHostnameForUrl(address) {
return parseIPv6Hextets(address) ? `[${address}]` : address;
}
function normalizeHostnameForComparison(hostname) {
return hostname.toLowerCase().replace(/\.$/, "");
}
/**
* Turns a detected URL plus its resolveDNSHost result into what to actually fetch.
* This is the single place that decides how a validated address gets used, so there
* is exactly one path through this logic - not two independently-maintained ones that
* could quietly drift apart and produce a hostname-preserving request with no pin.
*
* - No resolved address: nothing to enforce, fetch the URL as-is.
* - resolveDNSHost returned a URL: trust it verbatim, nothing to pin to.
* - Plain HTTP: rewriting the hostname to the resolved IP is itself the pin (the
* request can't land anywhere else), so no dispatcher is needed.
* - HTTPS: rewriting the hostname to a bare IP breaks TLS (the handshake would send
* the IP as the SNI server name, and certificate validation fails since
* certificates are issued for hostnames, not IPs). The hostname is kept for SNI and
* the Host header, and the connection itself is pinned to the validated address
* instead via a dispatcher, so a DNS resolver returning something else on the real
* request (DNS rebinding) can't route the request anywhere else.
*/
function prepareFetch(url, resolvedAddressOrUrl) {
if (!resolvedAddressOrUrl) {
return { url };
}
const trimmedResolvedAddressOrUrl = resolvedAddressOrUrl.trim();
if (trimmedResolvedAddressOrUrl.startsWith("http://") ||
trimmedResolvedAddressOrUrl.startsWith("https://")) {
// Trusted verbatim - there's no bare address here to validate or pin to.
return { url: trimmedResolvedAddressOrUrl };
}
const resolvedAddress = getResolvedAddress(trimmedResolvedAddressOrUrl);
if (!resolvedAddress) {
throw new Error("resolveDNSHost must resolve to an IP address or URL");
}
const parsedUrl = new URL(url);
if (parsedUrl.protocol !== "https:") {
parsedUrl.hostname = formatHostnameForUrl(resolvedAddress);
return { url: parsedUrl.href };
}
const family = parseIPv4Address(resolvedAddress) ? 4 : 6;
const expectedHostname = normalizeHostnameForComparison(parsedUrl.hostname);
const dispatcher = new undici_1.Agent({
connect: {
lookup(hostname, lookupOptions, callback) {
if (normalizeHostnameForComparison(hostname) !== expectedHostname) {
callback(new Error("SSRF request detected, hostname changed after DNS validation"));
return;
}
if (lookupOptions === null || lookupOptions === void 0 ? void 0 : lookupOptions.all) {
callback(null, [{ address: resolvedAddress, family }]);
return;
}
callback(null, resolvedAddress, family);
},
},
});
return { url, dispatcher };
}
/**
* Node's global `fetch` is powered by its own internal, vendored copy of undici. A
* `Dispatcher` built from the separately installed `undici` package is a different
* class instance and isn't recognized by that internal copy, so pinning a connection
* silently has no effect if it's handed to the global `fetch`. Requests that need a
* pinned dispatcher are therefore made with undici's own `fetch` instead.
*/
function fetchWithDispatcher(url, fetchOptions, dispatcher) {
if (!dispatcher) {
return fetch(url, fetchOptions);
}
return (0, undici_1.fetch)(url, Object.assign(Object.assign({}, fetchOptions), { dispatcher }));
}
function destroyDispatchers(dispatchers) {
return __awaiter(this, void 0, void 0, function* () {
yield Promise.all(dispatchers.map((dispatcher) => __awaiter(this, void 0, void 0, function* () {
try {
yield dispatcher.destroy();
}
catch (_a) {
// Ignore dispatcher disposal errors; the request result has already been handled.
}
})));
});
}
/**
* Parses the text, extracts the first link it finds and does a HTTP request
* to fetch the website content, afterwards it tries to parse the internal HTML
* and extract the information via meta tags
* @param text string, text to be parsed
* @param options ILinkPreviewOptions
*/
function getLinkPreview(text, options) {
return __awaiter(this, void 0, void 0, function* () {
var _a, _b, _c;
if (!text || typeof text !== `string`) {
throw new Error(`link-preview-js did not receive a valid url or text`);
}
const detectedUrl = text
.replace(/\n/g, ` `)
.split(` `)
.find((token) => constants_1.CONSTANTS.REGEX_VALID_URL.test(token));
if (!detectedUrl) {
throw new Error(`link-preview-js did not receive a valid a url or text`);
}
if ((options === null || options === void 0 ? void 0 : options.followRedirects) === `manual` && !(options === null || options === void 0 ? void 0 : options.handleRedirects)) {
throw new Error(`link-preview-js followRedirects is set to manual, but no handleRedirects function was provided`);
}
let resolvedUrl;
if (options === null || options === void 0 ? void 0 : options.resolveDNSHost) {
resolvedUrl = yield options.resolveDNSHost(detectedUrl);
throwOnLoopback(resolvedUrl);
}
else {
console.error("[link-preview-js] You are not resolving DNS addresses (resolveDNSHost option) before fetching a link. This can cause loopback attacks. Always try to resolve DNS addresses");
}
const timeout = (_a = options === null || options === void 0 ? void 0 : options.timeout) !== null && _a !== void 0 ? _a : 3000; // 3 second timeout default
const controller = new AbortController();
const timeoutCounter = setTimeout(() => controller.abort(), timeout);
const fetchOptions = {
headers: (_b = options === null || options === void 0 ? void 0 : options.headers) !== null && _b !== void 0 ? _b : {},
redirect: (_c = options === null || options === void 0 ? void 0 : options.followRedirects) !== null && _c !== void 0 ? _c : `error`,
signal: controller.signal,
};
// A proxy already changes where the request is physically sent, so pinning the
// connection to the resolved address would fight the proxy instead of the target -
// in that case there's nothing for prepareFetch to decide, the proxy URL is it.
const initialFetch = (options === null || options === void 0 ? void 0 : options.proxyUrl)
? { url: options.proxyUrl.concat(detectedUrl) }
: prepareFetch(detectedUrl, resolvedUrl);
const fetchUrl = initialFetch.url;
const dispatchers = [];
if (initialFetch.dispatcher) {
dispatchers.push(initialFetch.dispatcher);
}
try {
const fetchWithTimeout = (url, requestDispatcher) => __awaiter(this, void 0, void 0, function* () {
return fetchWithDispatcher(url, fetchOptions, requestDispatcher).catch((e) => {
if (e.name === `AbortError`) {
throw new Error(`Request timeout`);
}
clearTimeout(timeoutCounter);
throw e;
});
});
let response = yield fetchWithTimeout(fetchUrl, initialFetch.dispatcher);
if (response.status > 300 &&
response.status < 309 &&
fetchOptions.redirect === `manual` &&
(options === null || options === void 0 ? void 0 : options.handleRedirects)) {
const locationHeader = response.headers.get(`location`) || ``;
const isAbsoluteURI = locationHeader.startsWith("http://") || locationHeader.startsWith("https://");
// Resolve the URL, handling both absolute and relative URLs
const forwardedUrl = isAbsoluteURI ? locationHeader : new URL(locationHeader, fetchUrl).href;
if (!options.handleRedirects(fetchUrl, forwardedUrl)) {
throw new Error(`link-preview-js could not handle redirect`);
}
let forwardedResolvedUrl;
if (options === null || options === void 0 ? void 0 : options.resolveDNSHost) {
forwardedResolvedUrl = yield options.resolveDNSHost(forwardedUrl);
throwOnLoopback(forwardedResolvedUrl);
}
const forwardedFetch = (options === null || options === void 0 ? void 0 : options.proxyUrl)
? { url: forwardedUrl }
: prepareFetch(forwardedUrl, forwardedResolvedUrl);
if (forwardedFetch.dispatcher) {
dispatchers.push(forwardedFetch.dispatcher);
}
response = yield fetchWithTimeout(forwardedFetch.url, forwardedFetch.dispatcher);
}
clearTimeout(timeoutCounter);
const headers = {};
response.headers.forEach((header, key) => {
headers[key] = header;
});
const normalizedResponse = {
url: (options === null || options === void 0 ? void 0 : options.proxyUrl) ? response.url.replace(options.proxyUrl, ``) : response.url,
headers,
data: yield response.text(),
};
return (0, shared_1.parseResponse)(normalizedResponse, options);
}
finally {
clearTimeout(timeoutCounter);
yield destroyDispatchers(dispatchers);
}
});
}