@apidevtools/json-schema-ref-parser/lib/util/url.ts

Parse, Resolve, and Dereference JSON Schema $ref pointers

import convertPathToPosix from "./convert-path-to-posix.js";

const forwardSlashPattern = /\//g;
const protocolPattern = /^(\w{2,}):\/\//i;
const jsonPointerSlash = /~1/g;
const jsonPointerTilde = /~0/g;

import { isWindows } from "./is-windows.js";

const isAbsoluteWin32Path = /^[a-zA-Z]:[\\/]/;

// RegExp patterns to URL-encode special characters in local filesystem paths
const urlEncodePatterns = [
  [/\?/g, "%3F"],
  [/#/g, "%23"],
] as [RegExp, string][];

// RegExp patterns to URL-decode special characters for local filesystem paths
const urlDecodePatterns = [/%23/g, "#", /%24/g, "$", /%26/g, "&", /%2C/g, ",", /%40/g, "@"];

const unsafeDomainSuffixes = [".localhost", ".local", ".internal", ".intranet", ".corp", ".home", ".lan"];

export const parse = (u: string | URL) => new URL(u);

/**
 * Returns resolved target URL relative to a base URL in a manner similar to that of a Web browser resolving an anchor tag HREF.
 *
 * @returns
 */
export function resolve(from: string, to: string) {
  // we use a non-existent URL to check if its a relative URL
  const fromUrl = new URL(convertPathToPosix(from), "https://aaa.nonexistanturl.com");
  const resolvedUrl = new URL(convertPathToPosix(to), fromUrl);
  const endSpaces = to.match(/(\s*)$/)?.[1] || "";
  if (resolvedUrl.hostname === "aaa.nonexistanturl.com") {
    // `from` is a relative URL.
    const { pathname, search, hash } = resolvedUrl;
    return pathname + search + safeDecodeURIComponent(hash) + endSpaces;
  }
  const resolved = resolvedUrl.toString() + endSpaces;
  // if there is a #, we want to split on the first one only, and decode the part after
  if (resolved.includes("#")) {
    const [base, hash] = resolved.split("#", 2);
    return base + "#" + safeDecodeURIComponent(hash || "");
  }
  return resolved;
}

function safeDecodeURIComponent(value: string): string {
  try {
    return decodeURIComponent(value);
  } catch {
    try {
      // Preserve literal or malformed percent signs while continuing to decode
      // any valid percent-encoded characters in the same fragment.
      return decodeURIComponent(value.replace(/%(?![\dA-F]{2})/gi, "%25"));
    } catch {
      return value;
    }
  }
}

/**
 * Returns the current working directory (in Node) or the current page URL (in browsers).
 *
 * @returns
 */
export function cwd() {
  if (typeof window !== "undefined" && window.location && window.location.href) {
    const href = window.location.href;
    if (!href || !href.startsWith("http")) {
      // try parsing as url, and if it fails, return root url /
      try {
        new URL(href);
        return href;
      } catch {
        return "/";
      }
    }
    return href;
  }

  if (typeof process !== "undefined" && process.cwd) {
    const path = process.cwd();

    const lastChar = path.slice(-1);
    if (lastChar === "/" || lastChar === "\\") {
      return path;
    } else {
      return path + "/";
    }
  }
  return "/";
}

/**
 * Returns the protocol of the given URL, or `undefined` if it has no protocol.
 *
 * @param path
 * @returns
 */
export function getProtocol(path: string | undefined) {
  const match = protocolPattern.exec(path || "");
  if (match) {
    return match[1].toLowerCase();
  }
  return undefined;
}

/**
 * Returns the lowercased file extension of the given URL,
 * or an empty string if it has no extension.
 *
 * @param path
 * @returns
 */
export function getExtension(path: string) {
  const pathEnd = path.search(/[?#]/);
  const pathname = pathEnd >= 0 ? path.substring(0, pathEnd) : path;
  const lastSlash = Math.max(pathname.lastIndexOf("/"), pathname.lastIndexOf("\\"));
  const lastDot = pathname.lastIndexOf(".");
  if (lastDot > lastSlash) {
    return pathname.substring(lastDot).toLowerCase();
  }
  return "";
}

/**
 * Removes the query, if any, from the given path.
 *
 * @param path
 * @returns
 */
export function stripQuery(path: string) {
  const queryIndex = path.indexOf("?");
  if (queryIndex >= 0) {
    path = path.substring(0, queryIndex);
  }
  return path;
}

/**
 * Returns the hash (URL fragment), of the given path.
 * If there is no hash, then the root hash ("#") is returned.
 *
 * @param path
 * @returns
 */
export function getHash(path: undefined | string) {
  if (!path) {
    return "#";
  }
  const hashIndex = path.indexOf("#");
  if (hashIndex >= 0) {
    return path.substring(hashIndex);
  }
  return "#";
}

/**
 * Removes the hash (URL fragment), if any, from the given path.
 *
 * @param path
 * @returns
 */
export function stripHash(path: string | undefined) {
  if (!path) {
    return "";
  }
  const hashIndex = path.indexOf("#");
  if (hashIndex >= 0) {
    path = path.substring(0, hashIndex);
  }
  return path;
}

/**
 * Determines whether the given path is an HTTP(S) URL.
 *
 * @param path
 * @returns
 */
export function isHttp(path: string) {
  const protocol = getProtocol(path);
  if (protocol === "http" || protocol === "https") {
    return true;
  } else if (protocol === undefined) {
    // There is no protocol.  If we're running in a browser, then assume it's HTTP.
    return typeof window !== "undefined";
  } else {
    // It's some other protocol, such as "ftp://", "mongodb://", etc.
    return false;
  }
}
/**
 * Determines whether the given url is an unsafe or internal url.
 *
 * @param path - The URL or path to check
 * @returns true if the URL is unsafe/internal, false otherwise
 */
export function isUnsafeUrl(path: string | unknown): boolean {
  if (!path || typeof path !== "string") {
    return true;
  }

  // Trim whitespace and convert to lowercase for comparison
  const normalizedPath = path.trim().toLowerCase();

  // Empty or just whitespace
  if (!normalizedPath) {
    return true;
  }

  // JavaScript protocols
  if (
    normalizedPath.startsWith("javascript:") ||
    normalizedPath.startsWith("vbscript:") ||
    normalizedPath.startsWith("data:")
  ) {
    return true;
  }

  // File protocol
  if (normalizedPath.startsWith("file:")) {
    return true;
  }

  // if we're in the browser, we assume that it is safe
  if (typeof window !== "undefined" && window.location && window.location.href) {
    return false;
  }

  try {
    // Try to parse as URL
    const url = new URL(normalizedPath.startsWith("//") ? "http:" + normalizedPath : normalizedPath);

    if (isUnsafeHostname(url.hostname)) {
      return true;
    }

    // Check for non-standard ports that might indicate internal services
    const port = url.port;
    if (port && isInternalPort(parseInt(port))) {
      return true;
    }
  } catch {
    // If URL parsing fails, check if it's a relative path or contains suspicious patterns

    // Relative paths starting with / are generally safe for same-origin
    if (normalizedPath.startsWith("/") && !normalizedPath.startsWith("//")) {
      return false;
    }

    if (containsUnsafeHostname(normalizedPath)) {
      return true;
    }
  }

  return false;
}

/**
 * Determines whether an HTTP(S) URL is unsafe, including hostnames that resolve
 * to a non-public IP address. DNS resolution is only available in Node.js; in a
 * browser, {@link isUnsafeUrl} remains the source of truth and the browser's
 * own network security model applies.
 *
 * DNS lookup failures are allowed to propagate so callers fail closed rather
 * than fetching a hostname whose addresses could not be validated.
 */
export interface ResolvedUrlAddress {
  address: string;
  family: number;
}

export interface UrlSafetyResult {
  unsafe: boolean;
  addresses?: ResolvedUrlAddress[];
}

/**
 * Checks a URL and, in Node.js, returns the exact public addresses that were
 * validated. Callers that perform the request must pin their connection to
 * these addresses; resolving the hostname again would reintroduce a DNS
 * rebinding race between validation and connection establishment.
 */
export async function resolveUrlSafety(path: string | unknown): Promise<UrlSafetyResult> {
  if (isUnsafeUrl(path)) {
    return { unsafe: true };
  }

  if (
    typeof path !== "string" ||
    typeof process === "undefined" ||
    !process.versions?.node ||
    typeof window !== "undefined"
  ) {
    return { unsafe: false };
  }

  const parsedUrl = new URL(path.startsWith("//") ? `http:${path}` : path);
  if (parsedUrl.protocol !== "http:" && parsedUrl.protocol !== "https:") {
    return { unsafe: false };
  }

  // Keep the Node-only dependency out of browser module graphs. The import is
  // never evaluated outside Node.js.
  const dnsModuleName = "node:dns/promises";
  const dns = (await import(dnsModuleName)) as {
    lookup(
      hostname: string,
      options: { all: true; verbatim: true },
    ): Promise<Array<{ address: string; family: number }>>;
  };
  const addresses: ResolvedUrlAddress[] = await dns.lookup(normalizeHostname(parsedUrl.hostname), {
    all: true,
    verbatim: true,
  });

  if (addresses.length === 0 || addresses.some(({ address }) => isUnsafeHostname(address))) {
    return { unsafe: true };
  }

  return { unsafe: false, addresses };
}

export async function isUnsafeUrlWithDns(path: string | unknown): Promise<boolean> {
  return (await resolveUrlSafety(path)).unsafe;
}

/**
 * Helper function to check if a hostname is local or resolves to a non-public literal address.
 */
function isUnsafeHostname(hostname: string): boolean {
  const normalizedHostname = normalizeHostname(hostname);

  if (!normalizedHostname) {
    return true;
  }

  if (
    normalizedHostname === "localhost" ||
    unsafeDomainSuffixes.some((suffix) => normalizedHostname.endsWith(suffix))
  ) {
    return true;
  }

  const ipv4 = parseIPv4Address(normalizedHostname);
  if (ipv4) {
    return isUnsafeIPv4Address(ipv4);
  }

  const ipv6 = parseIPv6Address(normalizedHostname);
  if (ipv6) {
    return isUnsafeIPv6Address(ipv6);
  }

  return false;
}

function normalizeHostname(hostname: string): string {
  let normalizedHostname = hostname.trim().toLowerCase();

  if (normalizedHostname.startsWith("[") && normalizedHostname.endsWith("]")) {
    normalizedHostname = normalizedHostname.slice(1, -1);
  }

  while (normalizedHostname.endsWith(".")) {
    normalizedHostname = normalizedHostname.slice(0, -1);
  }

  return normalizedHostname;
}

function parseIPv4Address(ip: string): number[] | undefined {
  const parts = ip.split(".");

  if (parts.length !== 4) {
    return undefined;
  }

  const octets = parts.map((part) => {
    if (!/^\d+$/.test(part)) {
      return Number.NaN;
    }

    return Number(part);
  });

  if (octets.some((octet) => !Number.isInteger(octet) || octet < 0 || octet > 255)) {
    return undefined;
  }

  return octets;
}

/**
 * Helper function to check if an IPv4 address is in a non-public range.
 */
function isUnsafeIPv4Address([a, b, c, d]: number[]): boolean {
  return (
    a === 0 ||
    a === 10 ||
    a === 127 ||
    (a === 100 && b >= 64 && b <= 127) ||
    (a === 169 && b === 254) ||
    (a === 172 && b >= 16 && b <= 31) ||
    (a === 192 && b === 0 && c === 0) ||
    (a === 192 && b === 168) ||
    (a === 198 && (b === 18 || b === 19)) ||
    a >= 224 ||
    (a === 255 && b === 255 && c === 255 && d === 255)
  );
}

function parseIPv6Address(ip: string): number[] | undefined {
  if (!ip.includes(":")) {
    return undefined;
  }

  let normalizedIP = ip;
  const lastSeparator = normalizedIP.lastIndexOf(":");
  const possibleIPv4 = normalizedIP.slice(lastSeparator + 1);

  if (possibleIPv4.includes(".")) {
    const ipv4 = parseIPv4Address(possibleIPv4);

    if (!ipv4) {
      return undefined;
    }

    const firstGroup = ipv4[0] * 256 + ipv4[1];
    const secondGroup = ipv4[2] * 256 + ipv4[3];
    normalizedIP = `${normalizedIP.slice(0, lastSeparator + 1)}${firstGroup.toString(16)}:${secondGroup.toString(16)}`;
  }

  const halves = normalizedIP.split("::");

  if (halves.length > 2) {
    return undefined;
  }

  const head = parseIPv6Groups(halves[0]);
  const tail = halves.length === 2 ? parseIPv6Groups(halves[1]) : [];

  if (!head || !tail) {
    return undefined;
  }

  if (halves.length === 1) {
    return head.length === 8 ? head : undefined;
  }

  const missingGroups = 8 - head.length - tail.length;

  if (missingGroups < 1) {
    return undefined;
  }

  return [...head, ...Array<number>(missingGroups).fill(0), ...tail];
}

function parseIPv6Groups(groups: string): number[] | undefined {
  if (!groups) {
    return [];
  }

  const parsedGroups = groups.split(":").map((group) => {
    if (!/^[\da-f]{1,4}$/i.test(group)) {
      return Number.NaN;
    }

    return Number.parseInt(group, 16);
  });

  if (parsedGroups.some((group) => !Number.isInteger(group) || group < 0 || group > 0xffff)) {
    return undefined;
  }

  return parsedGroups;
}

/**
 * Helper function to check if an IPv6 address is in a non-public range.
 */
function isUnsafeIPv6Address(groups: number[]): boolean {
  if (groups.length !== 8) {
    return false;
  }

  const isUnspecified = groups.every((group) => group === 0);
  const isLoopback = groups.slice(0, 7).every((group) => group === 0) && groups[7] === 1;
  const isUniqueLocal = (groups[0] & 0xfe00) === 0xfc00;
  const isLinkLocal = (groups[0] & 0xffc0) === 0xfe80;
  const isMulticast = (groups[0] & 0xff00) === 0xff00;

  if (isUnspecified || isLoopback || isUniqueLocal || isLinkLocal || isMulticast) {
    return true;
  }

  const mappedIPv4 = getMappedIPv4Address(groups);

  return mappedIPv4 ? isUnsafeIPv4Address(mappedIPv4) : false;
}

function getMappedIPv4Address(groups: number[]): number[] | undefined {
  const firstFiveGroupsAreZero = groups.slice(0, 5).every((group) => group === 0);
  const firstSixGroupsAreZero = firstFiveGroupsAreZero && groups[5] === 0;
  const isIPv4Mapped = firstFiveGroupsAreZero && groups[5] === 0xffff;

  if (!firstSixGroupsAreZero && !isIPv4Mapped) {
    return undefined;
  }

  return [Math.floor(groups[6] / 256), groups[6] % 256, Math.floor(groups[7] / 256), groups[7] % 256];
}

function containsUnsafeHostname(value: string): boolean {
  const candidates = value
    .split(/[\s/?#]+/)
    .map((candidate) => candidate.replace(/^[a-z][\d+.a-z-]*:\/\//i, "").replace(/:\d+$/, ""))
    .filter(Boolean);

  return candidates.some((candidate) => isUnsafeHostname(candidate));
}

/**
 * Helper function to check if a port is typically used for internal services
 */
function isInternalPort(port: number): boolean {
  const internalPorts = [
    22, // SSH
    23, // Telnet
    25, // SMTP
    53, // DNS
    135, // RPC
    139, // NetBIOS
    445, // SMB
    993, // IMAPS
    995, // POP3S
    1433, // SQL Server
    1521, // Oracle
    3306, // MySQL
    3389, // RDP
    5432, // PostgreSQL
    5900, // VNC
    6379, // Redis
    8080, // Common internal web
    8443, // Common internal HTTPS
    9200, // Elasticsearch
    27017, // MongoDB
  ];

  return internalPorts.includes(port);
}
/**
 * Determines whether the given path is a filesystem path.
 * This includes "file://" URLs.
 *
 * @param path
 * @returns
 */
export function isFileSystemPath(path: string | undefined) {
  // @ts-ignore
  if (typeof window !== "undefined" || (typeof process !== "undefined" && process.browser)) {
    // We're running in a browser, so assume that all paths are URLs.
    // This way, even relative paths will be treated as URLs rather than as filesystem paths
    return false;
  }

  const protocol = getProtocol(path);
  return protocol === undefined || protocol === "file";
}

/**
 * Converts a filesystem path to a properly-encoded URL.
 *
 * This is intended to handle situations where JSON Schema $Ref Parser is called
 * with a filesystem path that contains characters which are not allowed in URLs.
 *
 * @example
 * The following filesystem paths would be converted to the following URLs:
 *
 *    <"!@#$%^&*+=?'>.json              ==>   %3C%22!@%23$%25%5E&*+=%3F\'%3E.json
 *    C:\\My Documents\\File (1).json   ==>   C:/My%20Documents/File%20(1).json
 *    file://Project #42/file.json      ==>   file://Project%20%2342/file.json
 *
 * @param path
 * @returns
 */
export function fromFileSystemPath(path: string) {
  // Step 1: On Windows, replace backslashes with forward slashes,
  // rather than encoding them as "%5C"
  if (isWindows()) {
    const projectDir = cwd();
    const upperPath = path.toUpperCase();
    const projectDirPosixPath = convertPathToPosix(projectDir);
    const posixUpper = projectDirPosixPath.toUpperCase();
    const hasProjectDir = upperPath.includes(posixUpper);
    const hasProjectUri = upperPath.includes(posixUpper);
    const isAbsolutePath =
      isAbsoluteWin32Path.test(path) ||
      path.startsWith("http://") ||
      path.startsWith("https://") ||
      path.startsWith("file://");

    if (!(hasProjectDir || hasProjectUri || isAbsolutePath) && !projectDir.startsWith("http")) {
      const join = (a: string, b: string) => {
        if (a.endsWith("/") || a.endsWith("\\")) {
          return a + b;
        } else {
          return a + "/" + b;
        }
      };
      path = join(projectDir, path);
    }
    path = convertPathToPosix(path);
  }

  // Step 2: `encodeURI` will take care of MOST characters
  path = encodeURI(path);

  // Step 3: Manually encode characters that are not encoded by `encodeURI`.
  // This includes characters such as "#" and "?", which have special meaning in URLs,
  // but are just normal characters in a filesystem path.
  for (const pattern of urlEncodePatterns) {
    path = path.replace(pattern[0], pattern[1]);
  }

  return path;
}

/**
 * Converts a URL to a local filesystem path.
 */
export function toFileSystemPath(path: string | undefined, keepFileProtocol?: boolean): string {
  // Bare "%" characters are valid in filesystem paths, but they make `decodeURI` throw.
  // Escape only the non-encoded ones so percent-encoded sequences still decode normally.
  path = path!.replace(/%(?![0-9A-Fa-f]{2})/g, "%25");

  // Step 1: `decodeURI` will decode characters such as Cyrillic characters, spaces, etc.
  path = decodeURI(path!);

  // Step 2: Manually decode characters that are not decoded by `decodeURI`.
  // This includes characters such as "#" and "?", which have special meaning in URLs,
  // but are just normal characters in a filesystem path.
  for (let i = 0; i < urlDecodePatterns.length; i += 2) {
    path = path.replace(urlDecodePatterns[i], urlDecodePatterns[i + 1] as string);
  }

  // Step 3: If it's a "file://" URL, then format it consistently
  // or convert it to a local filesystem path
  let isFileUrl = path.toLowerCase().startsWith("file://");
  if (isFileUrl) {
    // Strip-off the protocol, and the initial "/", if there is one
    path = path.replace(/^file:\/\//i, "").replace(/^\//, "");

    // insert a colon (":") after the drive letter on Windows
    if (isWindows() && path[1] === "/") {
      path = `${path[0]}:${path.substring(1)}`;
    }

    if (keepFileProtocol) {
      // Return the consistently-formatted "file://" URL
      path = "file:///" + path;
    } else {
      // Convert the "file://" URL to a local filesystem path.
      // On Windows, it will start with something like "C:/".
      // On Posix, it will start with "/"
      isFileUrl = false;
      path = isWindows() ? path : "/" + path;
    }
  }

  // Step 4: Normalize Windows paths (unless it's a "file://" URL)
  if (isWindows() && !isFileUrl) {
    // Replace forward slashes with backslashes
    path = path.replace(forwardSlashPattern, "\\");

    // Capitalize the drive letter
    if (path.match(/^[a-z]:\\/i)) {
      path = path[0].toUpperCase() + path.substring(1);
    }
  }

  return path;
}

/**
 * Converts a $ref pointer to a valid JSON Path.
 *
 * @param pointer
 * @returns
 */
export function safePointerToPath(pointer: string) {
  if (pointer.length <= 1 || pointer[0] !== "#" || pointer[1] !== "/") {
    return [];
  }

  return pointer
    .slice(2)
    .split("/")
    .map((value: string) => {
      return value.replace(jsonPointerSlash, "/").replace(jsonPointerTilde, "~");
    });
}