uniqhtt
Version:
A sophisticated, enterprise-grade HTTP client for Node.js, Web, and edge environments featuring intelligent cookie management, advanced web crawling capabilities, comprehensive automation tools, and a new TypeScript-first Pro API with HTTP/2, HTTP/3, stre
1,304 lines (1,299 loc) • 275 kB
JavaScript
"use strict";
var __create = Object.create;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __getProtoOf = Object.getPrototypeOf;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
// If the importer is in node compatibility mode or this is not an ESM
// file that has been converted to a CommonJS file using a Babel-
// compatible transform (i.e. "__esModule" has not been set), then set
// "default" to the CommonJS "module.exports" for node compatibility.
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
mod
));
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
// src/entry/nodejs.ts
var nodejs_exports = {};
__export(nodejs_exports, {
Cookie: () => Cookie,
CookieJar: () => CookieJar,
FormData: () => Form,
Uniqhtt: () => Uniqhtt,
default: () => nodejs_default,
uniqhtt: () => uniqhtt
});
module.exports = __toCommonJS(nodejs_exports);
// src/core/adapters/base.ts
var import_node_buffer = require("node:buffer");
var import_node_path = __toESM(require("node:path"), 1);
var import_p_queue = __toESM(require("p-queue"), 1);
// src/core/util/cookies.ts
var import_tough_cookie = require("tough-cookie");
var Cookie = class extends import_tough_cookie.Cookie {
constructor(options3) {
super(options3);
}
getExpires() {
let expires = 0;
if (this.expires && typeof this.expires !== "string") {
expires = Math.round(this.expires.getTime() / 1e3);
} else if (this.maxAge) {
if (this.maxAge === "Infinity") {
expires = 2147483647;
} else if (this.maxAge === "-Infinity") {
expires = 0;
} else if (typeof this.maxAge === "number") {
expires = Math.round(Date.now() / 1e3 + this.maxAge);
}
}
return expires;
}
toNetscapeFormat() {
const domain = !this.hostOnly ? this.domain.startsWith(".") ? this.domain : "." + this.domain : this.domain;
const secure = this.secure ? "TRUE" : "FALSE";
const expires = this.getExpires();
return `${domain} TRUE ${this.path} ${secure} ${expires} ${this.key} ${this.value}`;
}
toSetCookieString() {
let str = this.cookieString();
if (this.expires instanceof Date) str += `; Expires=${this.expires.toUTCString()}`;
if (this.maxAge != null) str += `; Max-Age=${this.maxAge}`;
if (this.domain) str += `; Domain=${this.domain}`;
if (this.path) str += `; Path=${this.path}`;
if (this.secure) str += "; Secure";
if (this.httpOnly) str += "; HttpOnly";
if (this.sameSite) str += `; SameSite=${this.sameSite}`;
if (this.extensions) str += `; ${this.extensions.join("; ")}`;
return str;
}
/**
* Retrieves the complete URL from the cookie object
* @returns {string | undefined} The complete URL including protocol, domain and path. Returns undefined if domain is not set
* @example
* const cookie = new Cookie({
* domain: "example.com",
* path: "/path",
* secure: true
* });
* cookie.getURL(); // Returns: "https://example.com/path"
*/
getURL() {
if (!this.domain) return void 0;
const protocol = this.secure ? "https://" : "http://";
const domain = this.domain.startsWith(".") ? this.domain.substring(1) : this.domain;
const path5 = this.path || "/";
return `${protocol}${domain}${path5}`;
}
};
var CookieJar = class extends import_tough_cookie.CookieJar {
constructor(store, options3) {
super(store, options3);
}
generateCookies(data) {
const cookies = !data ? (this.toJSON()?.cookies || []).map((cookie) => new Cookie(cookie)) : data[0] instanceof Cookie ? data : data.map((cookie) => new Cookie(cookie instanceof import_tough_cookie.Cookie ? cookie : Cookie.fromJSON(cookie)));
const netscape = cookies.map((cookie) => cookie.toNetscapeFormat());
const cookieString = cookies.map((cookie) => cookie.cookieString());
const setCookiesString = cookies.map((cookie) => cookie.toSetCookieString());
let netscapeString = "# Netscape HTTP Cookie File\n";
netscapeString += "# This file was generated by uniqhtt npm package\n";
netscapeString += "# https://www.npmjs.com/package/uniqhtt\n";
netscapeString += netscape.join("\n") || "";
return {
array: cookies,
serialized: this.toJSON()?.cookies || [],
netscape: netscapeString,
string: cookieString.join("; "),
setCookiesString
};
}
cookies() {
return this.generateCookies();
}
parseResponseCookies(cookies) {
return this.generateCookies(cookies);
}
static toNetscapeCookie(cookies) {
cookies = cookies.map((cookie) => {
return cookie instanceof Cookie ? cookie : new Cookie(Cookie.fromJSON(cookie));
});
let netscapeString = "# Netscape HTTP Cookie File\n";
netscapeString += "# This file was generated by uniqhtt npm package\n";
netscapeString += "# https://www.npmjs.com/package/uniqhtt\n";
netscapeString += cookies.map((cookie) => cookie.toNetscapeFormat()).join("\n") || "";
return netscapeString;
}
static toCookieString(cookies) {
cookies = cookies.map((cookie) => {
return cookie instanceof Cookie ? cookie : new Cookie(Cookie.fromJSON(cookie));
});
return cookies.map((cookie) => cookie.toNetscapeFormat()).join("; ") || "";
}
toCookieString() {
return this.cookies().string;
}
toNetscapeCookie() {
return this.cookies().netscape;
}
toArray() {
return this.cookies().array;
}
toSetCookies() {
return this.cookies().setCookiesString;
}
toSerializedCookies() {
return this.cookies().serialized;
}
setCookiesSync(cookiesData, url) {
const cookies = [];
if (Array.isArray(cookiesData)) {
cookiesData.forEach((c) => {
const cookie = c instanceof Cookie ? c : typeof c === "string" ? new Cookie(Cookie.parse(c)) : new Cookie(Cookie.fromJSON(c));
let isFailed = 0;
while (isFailed < 2) {
try {
if (cookie) {
const _url = isFailed > 0 ? cookie.getURL() || url || this.getUrlFromCookie(cookie) : url || this.getUrlFromCookie(cookie);
if (_url) {
const __cookie = this.setCookieSync(cookie, _url);
if (__cookie) {
cookies.push(__cookie);
}
}
isFailed = 4;
break;
} else {
isFailed++;
}
} catch (error) {
isFailed++;
if (isFailed > 1) {
break;
}
}
}
});
} else if (typeof cookiesData === "string") {
if (cookiesData.includes(",") && (cookiesData.includes("=") && (cookiesData.includes(";") || cookiesData.includes("expires=") || cookiesData.includes("path=")))) {
const setCookieStrings = this.splitSetCookiesString(cookiesData);
setCookieStrings.forEach((cookieStr) => {
const cookie = new Cookie(Cookie.parse(cookieStr));
let isFailed = 0;
while (isFailed < 2) {
try {
if (cookie) {
const _url = isFailed > 0 ? cookie.getURL() || url || this.getUrlFromCookie(cookie) : url || this.getUrlFromCookie(cookie);
if (_url) {
const __cookie = this.setCookieSync(cookie, _url);
if (__cookie) {
cookies.push(__cookie);
}
}
isFailed = 4;
break;
} else {
isFailed++;
}
} catch (error) {
isFailed++;
if (isFailed > 1) {
break;
}
}
}
});
} else if (cookiesData.includes("=") && cookiesData.includes(";")) {
const cookiePairs = cookiesData.split(";");
cookiePairs.forEach((pair) => {
const trimmedPair = pair.trim();
if (trimmedPair) {
const cookie = new Cookie({ key: trimmedPair.split("=")[0].trim(), value: trimmedPair.split("=")[1]?.trim() || "" });
let isFailed = 0;
while (isFailed < 2) {
try {
if (cookie) {
const _url = isFailed > 0 ? cookie.getURL() || url || this.getUrlFromCookie(cookie) : url || this.getUrlFromCookie(cookie);
if (_url) {
const __cookie = this.setCookieSync(cookie, _url);
if (__cookie) {
cookies.push(__cookie);
}
}
isFailed = 4;
break;
} else {
isFailed++;
}
} catch (error) {
isFailed++;
if (isFailed > 1) {
break;
}
}
}
}
});
} else if (cookiesData.includes(" ") && /^\S+\t/.test(cookiesData)) {
const netscapeCookies = this.parseNetscapeCookies(cookiesData);
netscapeCookies.forEach((c) => {
const cookie = new Cookie(c);
let isFailed = 0;
while (isFailed < 2) {
try {
if (cookie) {
const _url = isFailed > 0 ? cookie.getURL() || url || this.getUrlFromCookie(cookie) : url || this.getUrlFromCookie(cookie);
if (_url) {
const __cookie = this.setCookieSync(cookie, _url);
if (__cookie) {
cookies.push(__cookie);
}
}
isFailed = 4;
break;
} else {
isFailed++;
}
} catch (error) {
isFailed++;
if (isFailed > 1) {
break;
}
}
}
});
} else if (cookiesData.includes("=")) {
const cookie = new Cookie(Cookie.parse(cookiesData));
let isFailed = 0;
while (isFailed < 2) {
try {
if (cookie) {
const _url = isFailed > 0 ? cookie.getURL() || url || this.getUrlFromCookie(cookie) : url || this.getUrlFromCookie(cookie);
if (_url) {
const __cookie = this.setCookieSync(cookie, _url);
if (__cookie) {
cookies.push(__cookie);
}
}
isFailed = 4;
break;
} else {
isFailed++;
}
} catch (error) {
isFailed++;
if (isFailed > 1) {
break;
}
}
}
}
}
return this.generateCookies(cookies);
}
// Helper method to split Set-Cookie strings that may contain commas within their values
splitSetCookiesString(cookiesString) {
const result = [];
let currentCookie = "";
let withinValue = false;
for (let i = 0; i < cookiesString.length; i++) {
const char = cookiesString[i];
if (char === "," && !withinValue) {
result.push(currentCookie.trim());
currentCookie = "";
continue;
}
if (char === "=") {
withinValue = true;
} else if (char === ";") {
withinValue = false;
}
currentCookie += char;
}
if (currentCookie.trim()) {
result.push(currentCookie.trim());
}
return result;
}
getUrlFromCookie(cookie) {
if (!cookie.domain) return void 0;
const proto = cookie.secure ? "https://" : "http://";
const domain = cookie.domain.startsWith(".") ? cookie.domain.substring(1) : cookie.domain;
const pathname = cookie.path || "/";
return `${proto}${domain}${pathname}`;
}
parseNetscapeCookies = (cookieText) => {
const lines = cookieText.split("\n").filter((line) => line.trim() !== "" && !line.startsWith("#"));
return lines.map((line) => {
const parts = line.split(" ");
if (parts.length < 7) {
throw new Error(`Invalid Netscape cookie format: ${line}`);
}
const [domain, _, path5, secureStr, expiresStr, name, value] = parts;
let expires = null;
if (expiresStr !== "0" && expiresStr !== "") {
expires = new Date(parseInt(expiresStr, 10) * 1e3);
}
return {
domain,
path: path5,
secure: secureStr.toUpperCase() === "TRUE",
expires,
key: name,
value,
httpOnly: false,
// Not specified in Netscape format, default to false
sameSite: void 0
// Not specified in Netscape format
};
});
};
/**
* Converts Netscape cookie format to an array of Set-Cookie header strings
*
* @param netscapeCookieText - Netscape format cookie string
* @returns Array of Set-Cookie header strings
*/
static netscapeCookiesToSetCookieArray(netscapeCookieText) {
const parseNetscapeCookies = (cookieText) => {
const lines = cookieText.split("\n").filter((line) => line.trim() !== "" && !line.startsWith("#"));
return lines.map((line) => {
const parts = line.split(" ");
if (parts.length < 7) {
throw new Error(`Invalid Netscape cookie format: ${line}`);
}
const [domain, _, path5, secureStr, expiresStr, name, value] = parts;
let expires = null;
if (expiresStr !== "0" && expiresStr !== "") {
expires = new Date(parseInt(expiresStr, 10) * 1e3);
}
return {
domain,
path: path5,
secure: secureStr.toUpperCase() === "TRUE",
expires,
name,
value,
httpOnly: false,
// Not specified in Netscape format, default to false
sameSite: void 0
// Not specified in Netscape format
};
});
};
const cookieToSetCookieString = (cookie) => {
let setCookie = `${cookie.name}=${cookie.value}`;
if (cookie.domain) {
setCookie += `; Domain=${cookie.domain}`;
}
if (cookie.path) {
setCookie += `; Path=${cookie.path}`;
}
if (cookie.expires) {
setCookie += `; Expires=${cookie.expires.toUTCString()}`;
}
if (cookie.secure) {
setCookie += "; Secure";
}
if (cookie.httpOnly) {
setCookie += "; HttpOnly";
}
if (cookie.sameSite) {
setCookie += `; SameSite=${cookie.sameSite}`;
}
return setCookie;
};
try {
const cookies = parseNetscapeCookies(netscapeCookieText);
return cookies.map(cookieToSetCookieString);
} catch (error) {
return [];
}
}
};
// src/core/util/httpOptions.ts
var import_form_data = __toESM(require("form-data"), 1);
var ERROR_INFO = {
"ECONNREFUSED": {
"code": -111,
// Typical errno for Connection Refused (e.g., Linux)
"message": "Connection Refused: The target server actively refused the TCP connection attempt."
},
"ECONNRESET": {
"code": -104,
// Typical errno for Connection Reset by Peer
"message": "Connection Reset: An existing TCP connection was forcibly closed by the peer (server or intermediary)."
},
"ETIMEDOUT": {
"code": -110,
// Typical errno for Connection Timed Out
"message": "Connection Timeout: Attempt to establish a TCP connection timed out (no response received within the timeout period)."
},
"ENOTFOUND": {
"code": -3008,
// Node.js specific code for DNS lookup failed (UV_EAI_NODATA or similar)
"message": "DNS Lookup Failed: DNS lookup for this hostname failed (domain name does not exist or DNS server error)."
},
"EAI_AGAIN": {
"code": -3001,
// Node.js specific code for temporary DNS failure (UV_EAI_AGAIN)
"message": "Temporary DNS Failure: Temporary failure in DNS name resolution; retrying might succeed."
},
"EPROTO": {
"code": -71,
// Typical errno for Protocol Error
"message": "Protocol Error: A protocol error occurred, often during the TLS/SSL handshake phase."
},
"ERR_INVALID_PROTOCOL": {
"code": -1001,
// Custom code for Node.js specific error
"message": "Invalid URL Protocol: The provided URL uses an unsupported or invalid protocol."
},
"ERR_TLS_CERT_ALTNAME_INVALID": {
"code": -1002,
// Custom code for Node.js specific error
"message": "Certificate Invalid Alt Name: The server's SSL/TLS certificate hostname does not match the requested domain (Subject Alternative Name mismatch)."
},
"ERR_TLS_HANDSHAKE_TIMEOUT": {
"code": -1003,
// Custom code for Node.js specific error
"message": "TLS Handshake Timeout: The TLS/SSL handshake timed out before completing."
},
"ERR_TLS_INVALID_PROTOCOL_VERSION": {
"code": -1004,
// Custom code for Node.js specific error
"message": "Invalid TLS Protocol Version: The client and server could not agree on a mutually supported TLS/SSL protocol version."
},
"ERR_TLS_RENEGOTIATION_DISABLED": {
"code": -1005,
// Custom code for Node.js specific error
"message": "TLS Renegotiation Disabled: An attempt at TLS/SSL renegotiation was made, but it is disabled or disallowed by the server/configuration."
},
"ERR_TLS_CERT_SIGNATURE_ALGORITHM_UNSUPPORTED": {
"code": -1006,
// Custom code for Node.js specific error
"message": "Unsupported Cert Signature Algorithm: The signature algorithm used in the server's SSL/TLS certificate is not supported or deemed insecure by the client."
},
"ERR_HTTP_HEADERS_SENT": {
"code": -1007,
// Custom code for Node.js specific error
"message": "Headers Already Sent: An attempt was made to send HTTP headers after they had already been sent (application logic error)."
},
"ERR_INVALID_ARG_TYPE": {
"code": -1008,
// Custom code for Node.js specific error
"message": "Invalid Argument Type: An argument of an incorrect type was passed to the underlying HTTP(S) request function."
},
"ERR_INVALID_URL": {
"code": -1009,
// Custom code for Node.js specific error
"message": "Invalid URL: The provided URL is syntactically invalid or cannot be parsed."
},
"ERR_STREAM_DESTROYED": {
"code": -1010,
// Custom code for Node.js specific error
"message": "Stream Destroyed: The readable/writable stream associated with the request/response was destroyed prematurely."
},
"ERR_STREAM_PREMATURE_CLOSE": {
"code": -1011,
// Custom code for Node.js specific error
"message": "Premature Stream Close: The server closed the connection before sending the complete response body (e.g., incomplete chunked encoding)."
},
"UND_ERR_CONNECT_TIMEOUT": {
"code": -1020,
// Custom code for undici specific error
"message": "Connect Timeout (uniqhtt): Timeout occurred specifically while waiting for the TCP socket connection to be established."
},
"UND_ERR_HEADERS_TIMEOUT": {
"code": -1021,
// Custom code for undici specific error
"message": "Headers Timeout (uniqhtt): Timeout occurred while waiting to receive the complete HTTP response headers from the server."
},
"UND_ERR_SOCKET": {
"code": -1022,
// Custom code for undici specific error (often wraps lower-level errors)
"message": "Socket Error (uniqhtt): An error occurred at the underlying socket level (may wrap other errors like ECONNRESET)."
},
"UND_ERR_INFO": {
"code": -1023,
// Custom code for undici specific error
"message": "Invalid Request Info (uniqhtt): Internal error related to invalid or missing metadata needed to process the request."
},
"UND_ERR_ABORTED": {
"code": -1024,
// Custom code for undici specific error
"message": "Request Aborted (uniqhtt): The request was explicitly aborted, often due to a timeout signal or user action."
},
"ABORT_ERR": {
"code": -1025,
// Custom code representing the standard DOM AbortError
"message": "Request Aborted: The request was explicitly aborted, often due to a timeout signal or user action."
},
"UND_ERR_REQUEST_TIMEOUT": {
"code": -1026,
// Custom code for undici specific error
"message": "Request Timeout (uniqhtt): The request timed out (check specific connect/headers/body timeouts if applicable)."
},
"UNQ_UNKOWN_ERROR": {
"code": -9999,
// Generic code for unknown errors
"message": "Unknown Error: An unspecified or unrecognized error occurred during the request."
},
"UNQ_FILE_PERMISSION_ERROR": {
"code": -1027,
// Custom code for file permission related errors
"message": "File Permission Error: Insufficient permissions to read or write a required file."
},
"UNQ_MISSING_REDIRECT_LOCATION": {
"code": -1028,
// Custom code for missing Location header on redirect
"message": "Redirect Location Not Found: Redirect header (Location:) missing in the server's response for a redirect status code (3xx)."
},
"UNQ_DECOMPRESSION_ERROR": {
"code": -1029,
// Custom code for content decompression errors
"message": "Decompression Error: Failed to decompress response body. Data may be corrupt or encoding incorrect."
},
"UNQ_DOWNLOAD_FAILED": {
"code": -1030,
// Custom code for generic download failure
"message": "Download Failed: The resource could not be fully downloaded due to an unspecified error during data transfer."
},
"UNQ_HTTP_ERROR": {
"code": -1031,
// Custom code for non-2xx/3xx HTTP responses
"message": "HTTP Error: The server responded with a non-successful HTTP status code."
},
"UNQ_REDIRECT_DENIED": {
"code": -1032,
// Custom code for when redirect following is disallowed
"message": "Redirect Denied: A redirect response was received, but following redirects is disabled or disallowed by configuration/user."
},
"UNQ_PROXY_INVALID_PROTOCOL": {
"code": -1033,
// Custom code for bad proxy protocol
"message": "Invalid Proxy Protocol: The specified proxy URL has an invalid or unsupported protocol scheme."
},
"UNQ_PROXY_INVALID_HOSTPORT": {
"code": -1034,
// Custom code for bad proxy host/port
"message": "Invalid Proxy Host/Port: The hostname or port number provided for the proxy server is invalid or malformed."
},
"UNQ_SOCKS_CONNECTION_FAILED": {
"code": -1040,
// General SOCKS connection error
"message": "SOCKS Proxy Connection Failed: Failed to establish connection with the SOCKS proxy server (check host, port, network)."
},
"UNQ_SOCKS_AUTHENTICATION_FAILED": {
"code": -1041,
// SOCKS auth error
"message": "SOCKS Proxy Authentication Failed: Authentication with the SOCKS5 proxy failed (invalid credentials or unsupported method)."
},
"UNQ_SOCKS_TARGET_CONNECTION_FAILED": {
"code": -1042,
// Error reported by SOCKS proxy for target connect
"message": "SOCKS Proxy Target Connection Failed: The SOCKS proxy reported failure connecting to the final destination (e.g., Connection refused, Host unreachable, Network unreachable)."
},
"UNQ_SOCKS_PROTOCOL_ERROR": {
"code": -1043,
// Malformed SOCKS reply/request
"message": "SOCKS Proxy Protocol Error: Invalid or malformed response received from the SOCKS proxy during handshake or connection."
},
"UNQ_PROXY_ERROR": {
"code": -1047,
// Generic proxy error code
"message": "Proxy Error: An unspecified error occurred while communicating with or through the proxy server."
}
};
function prepareHTTPOptions(type, runtime, options3, url, method, adapter, isCurl, maxRedirection, queueOptions, isRedirected, redirectedUrl, mainUrl, isRetrying, redirectCode, lastDomain) {
const validMethods = ["post", "put", "patch"];
const forContentType = validMethods.includes(method.toLowerCase());
let fetchOptions = { others: {} };
let headers = options3.headers instanceof Headers ? options3.headers : new Headers(options3.headers || {});
let requestCookies = [];
let useCookies = options3.enableCookieJar || typeof options3.enableCookieJar === "undefined";
let contentType = options3.contentType || headers.get("Content-Type") || (forContentType ? "application/json" : void 0);
if (options3.customHeaders) {
fetchOptions.headers = new Headers(options3.customHeaders);
} else if (options3.headers) {
fetchOptions.headers = headers instanceof Headers ? headers : new Headers(headers);
} else {
fetchOptions.headers = new Headers();
}
if (headers.has("Cookie")) {
const cookieString = headers.get("Cookie");
if (useCookies && !redirectedUrl && !isRedirected) {
runtime.setCookies(cookieString, url);
}
headers.delete("Cookie");
fetchOptions.headers.delete("Cookie");
}
if (useCookies) {
if (options3.cookies && !redirectedUrl && !isRedirected) {
runtime.setCookies(options3.cookies, url);
}
}
let cookiesString = "";
if (useCookies) {
requestCookies = runtime.jar.getCookiesSync(url).map((c) => new Cookie(c));
cookiesString = runtime.jar.getCookieStringSync(url);
}
if (requestCookies.length > 0) {
fetchOptions.headers.set("Cookie", cookiesString);
}
if (options3.body) {
fetchOptions.body = options3.body;
}
const isFormData = fetchOptions.body && (fetchOptions.body instanceof FormData || fetchOptions.body instanceof import_form_data.default);
if (!isFormData) {
if (options3.isFormData || options3.isJson || options3.isMultipart) {
if (options3.isFormData) {
fetchOptions.body = options3.body;
contentType = "application/x-www-form-urlencoded";
fetchOptions.headers.set("Content-Type", contentType);
} else if (options3.isJson) {
fetchOptions.body = options3.body;
contentType = "application/json";
fetchOptions.headers.set("Content-Type", contentType);
} else if (options3.isMultipart) {
fetchOptions.body = options3.body;
}
} else {
if (options3.json) {
fetchOptions.body = JSON.stringify(options3.json);
contentType = "application/json";
fetchOptions.headers.set("Content-Type", contentType);
} else if (options3.form_params) {
fetchOptions.body = new URLSearchParams(options3.form_params).toString();
contentType = "application/x-www-form-urlencoded";
fetchOptions.headers.set("Content-Type", contentType);
} else if (options3.multipart && !(options3.multipart instanceof FormData || options3.multipart instanceof import_form_data.default)) {
const formData = typeof FormData !== "undefined" ? new FormData() : new import_form_data.default();
Object.entries(options3.multipart).forEach(([key, value]) => {
formData.append(key, value);
});
fetchOptions.body = formData;
} else if (options3.requestType) {
switch (options3.requestType) {
case "json":
contentType = "application/json";
if (typeof fetchOptions.body === "object") {
fetchOptions.body = JSON.stringify(fetchOptions.body);
}
fetchOptions.headers.set("Content-Type", contentType);
break;
case "form":
contentType = "application/x-www-form-urlencoded";
if (typeof fetchOptions.body === "object") {
fetchOptions.body = new URLSearchParams(fetchOptions.body).toString();
}
fetchOptions.headers.set("Content-Type", contentType);
break;
case "formData":
if (typeof fetchOptions.body === "object") {
const formData = typeof FormData !== "undefined" ? new FormData() : new import_form_data.default();
Object.entries(fetchOptions.body).forEach(([key, value]) => {
formData.append(key, value ? typeof value === "object" ? JSON.stringify(value) : value.toString() : "");
});
fetchOptions.body = formData;
}
break;
case "text":
contentType = "text/plain";
fetchOptions.headers.set("Content-Type", contentType);
break;
}
} else if (contentType) {
const type2 = contentType.toLowerCase();
if (type2.includes("json")) {
fetchOptions.headers.set("Content-Type", "application/json");
if (fetchOptions.body && typeof fetchOptions.body === "object") {
fetchOptions.body = JSON.stringify(fetchOptions.body);
}
} else if (type2.includes("x-www-form-urlencoded")) {
fetchOptions.headers.set("Content-Type", "application/x-www-form-urlencoded");
if (fetchOptions.body && typeof fetchOptions.body === "object") {
fetchOptions.body = new URLSearchParams(fetchOptions.body).toString();
}
} else if (type2.includes("multipart")) {
if (fetchOptions.body && typeof fetchOptions.body === "object") {
const formData = new FormData();
Object.entries(fetchOptions.body).forEach(([key, value]) => {
formData.append(key, value);
});
fetchOptions.body = formData;
}
} else if (type2.includes("text/") || type2.includes("/javascript")) {
fetchOptions.body = fetchOptions.body ? typeof fetchOptions.body === "object" ? JSON.stringify(fetchOptions.body) : fetchOptions.body : "";
fetchOptions.headers.set("Content-Type", contentType);
}
} else if (contentType && !options3.withoutContentType) {
fetchOptions.headers.set("Content-Type", contentType);
}
}
}
if (options3.withoutContentType || isFormData) {
fetchOptions.headers.delete("Content-Type");
}
if (options3.withoutBodyOnRedirect && isRedirected) {
fetchOptions.body = void 0;
}
if (options3.printHeaders) {
console.log("Fetch headers:", fetchOptions.headers);
}
if ((typeof options3.autoSetReferer !== "boolean" || options3.autoSetReferer) && redirectedUrl) {
if (!options3.customHeaders) fetchOptions.headers.set("Referer", redirectedUrl);
}
for (const option of options3.innerFetchOption) {
if (Object.keys(options3).includes(option) && typeof options3[option] !== "undefined" && options3[option] !== null) {
fetchOptions.others[option] = options3[option];
}
}
if (!useCookies) {
fetchOptions.headers.delete("Cookie");
}
if (fetchOptions.body && (fetchOptions.body instanceof FormData || fetchOptions.body instanceof import_form_data.default)) {
fetchOptions.headers.delete("Content-Type");
}
delete fetchOptions.credentials;
const config = {
requestBody: fetchOptions.body,
requestOptions: { useCookies, config: null, useHTTP2: options3.useHTTP2 || runtime?.useHTTP2 || false }
};
let uniqhttConfig;
let redirectOptions = null;
if (!options3.uniqhttConfig) {
uniqhttConfig = buildConfig(
{},
fetchOptions.body ?? null,
method.toUpperCase(),
null,
url,
maxRedirection || 0,
options3.mimicBrowser || false,
options3.proxy || options3.thisProxy || null,
options3.timeout || 0,
options3.retry || null,
queueOptions || null,
// queueOptions
options3.signal || null,
// signal
isCurl,
// iscurl
null,
// redirectedOptions
adapter,
// adapter
requestCookies,
useCookies
);
} else {
uniqhttConfig = options3.uniqhttConfig;
if (!uniqhttConfig.redirectOptions) uniqhttConfig.redirectOptions = [];
if (!isRetrying) {
redirectOptions = {
method: method.toUpperCase(),
url: new URL(url),
requestBody: fetchOptions.body || null,
requestHeader: {}
};
}
}
if (method.toLowerCase() === "get" && fetchOptions.headers.has("Content-Type")) {
fetchOptions.headers.delete("Content-Type");
}
if (options3.mimicBrowser) {
if (!fetchOptions.headers.has("user-agent")) {
fetchOptions.headers.set("user-agent", options3.defaultUserAgent);
}
if (!fetchOptions.headers.has("accept")) {
fetchOptions.headers.set(
"accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8"
);
}
if (!fetchOptions.headers.has("accept-language")) {
fetchOptions.headers.set("accept-language", "en-US,en;q=0.9");
}
fetchOptions.headers.set("host", new URL(url).host);
if ([`POST`, `PUT`, `PATCH`, `DELETE`, `OPTIONS`].includes(method.toUpperCase()))
fetchOptions.headers.set("origin", new URL(mainUrl || url).origin);
if (mainUrl && fetchOptions.headers.has("origin")) {
const r = [`POST`, `PUT`, `PATCH`, `DELETE`, `OPTIONS`].includes(method.toUpperCase());
if (!r) fetchOptions.headers.delete("origin");
}
if (mainUrl && !fetchOptions.headers.has("referer")) {
fetchOptions.headers.set("referer", mainUrl);
}
} else if (mainUrl || redirectedUrl) {
fetchOptions.headers.set("host", new URL(url).host);
if ([`POST`, `PUT`, `PATCH`, `DELETE`, `OPTIONS`].includes(method.toUpperCase()))
fetchOptions.headers.set("origin", new URL(mainUrl || url).origin);
} else {
if (!fetchOptions.headers.has("origin") && options3.autoSetOrigin) {
const r = [`POST`, `PUT`, `PATCH`, `DELETE`, `OPTIONS`].includes(method.toUpperCase());
if (r) fetchOptions.headers.set("origin", new URL(mainUrl || url).origin);
}
if (mainUrl && !fetchOptions.headers.has("referer") && options3.autoSetReferer) {
fetchOptions.headers.set("referer", mainUrl);
}
}
if (redirectCode && lastDomain) {
fetchOptions.headers.set("referer", lastDomain);
}
const _headers = {};
for (const [key, value] of fetchOptions.headers.entries()) {
_headers[key.toLowerCase()] = value;
}
config.requestOptions.headers = _headers;
if (type === "node") {
config.requestOptions.proxy = options3.proxy ?? options3.thisProxy;
config.requestOptions.filename = options3.fileName;
}
config.requestOptions.method = method;
if (redirectOptions) {
redirectOptions.requestHeader = _headers;
uniqhttConfig.redirectOptions?.push(redirectOptions);
} else if (!isRetrying) {
uniqhttConfig.requestHeader = _headers;
}
if (options3.auth) {
config.auth = options3.auth;
}
fetchOptions = void 0;
config.requestOptions.useCookies = useCookies;
config.requestOptions.config = uniqhttConfig;
return config;
}
function buildConfig(requestHeader, requestBody, method, httpAgent, url, maxRedirection, mimicBrowser, proxy, timeout, retry, queueOptions, signal, isCurl, redirectOptions, adapter, requestCookies, cookiesEnabled) {
return {
requestHeader,
requestBody,
method,
httpAgent,
url: url instanceof URL ? url : new URL(url),
maxRedirection,
mimicBrowser,
proxy,
timeout,
retry,
queueOptions,
signal,
isCurl,
redirectOptions: redirectOptions ? Array.isArray(redirectOptions) ? redirectOptions : [redirectOptions] : null,
adapter,
requestCookies,
cookiesEnabled
};
}
function getCode(code) {
const error = ERROR_INFO[code];
if (error) {
return {
code,
errno: error.code,
message: error.message
};
} else {
const error2 = ERROR_INFO["UNQ_UNKOWN_ERROR"];
return {
code: "UNQ_UNKOWN_ERROR",
errno: error2.code,
message: error2.message
};
}
}
// src/core/adapters/base.ts
var process2 = __toESM(require("node:process"), 1);
var UniqhttError2 = class _UniqhttError extends Error {
response = {};
#method;
#url;
#finalUrl;
#statusText;
#urls;
#headers;
code;
#status;
config;
constructor(message, response, data, code, headers, config, urls) {
super(message);
this.name = "UniqhttError";
this.#headers = headers;
this.#method = config.method;
this.code = code;
this.#status = response.status;
if (response instanceof Response) {
const length = response.headers.get("Content-Length");
this.response = {
urls: urls || [response.url],
data,
status: response.status,
statusText: response.statusText,
finalUrl: response.url,
cookies: {
array: [],
string: "",
netscape: ""
},
headers,
contentType: response.headers.get("Content-Type"),
contentLength: length ? parseInt(length) : void 0,
config
};
} else {
this.response = {
data,
urls: urls || [response.url],
status: response?.status,
statusText: response?.statusText,
finalUrl: response?.url,
cookies: response?.cookies ?? {
array: [],
string: "",
netscape: ""
},
headers: Object.keys(typeof response.headers === "object" ? response.headers : {}).length > 1 ? response.headers : headers,
contentType: response.contentType,
config,
contentLength: response.contentLength ? response.contentLength : void 0
};
}
this.#url = this.response.finalUrl;
this.#finalUrl = this.response.finalUrl;
this.#statusText = this.response.statusText;
this.config = config;
this.#urls = urls && urls.length > 0 ? urls : [this.#url];
this.#clearStack();
Object.setPrototypeOf(this, _UniqhttError.prototype);
}
toJSON() {
return {
name: this.name,
message: this.message,
method: this.#method,
url: this.#url,
headers: this.#headers,
status: this.#status,
config: this.config,
code: this.code,
cause: this.cause ? typeof this.cause === "string" ? this.cause : this.cause?.message || null : null,
finalUrl: this.#finalUrl,
statusText: this.#statusText,
urls: this.#urls
};
}
#clearStack() {
this.stack = this.stack?.split("\n").filter((line) => !line.includes("node_modules") && !line.includes("toNetscapeFormat")).join("\n");
}
};
var Base = class {
queue = null;
isQueueEnabled = false;
jar = null;
// protected fetch: typeof fetch;
innerFetchOption = ["headers", "mode", "cache", "referrerPolicy", "integrity", "keepalive", "compress"];
environment = "node";
defaultUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
baseURL = null;
defaultHeaders = null;
defaultDebug = void 0;
mimicBrowser;
debug;
timeout;
retry;
queueOptions;
isCurl = { status: false, message: "Initializing" };
tempPath;
useCurl;
RETRYABLE_STATUS_CODES = /* @__PURE__ */ new Set([408, 429, 500, 502, 503, 504]);
rejectUnauthorized;
useSecureContext;
httpAgent;
httpsAgent;
enableCookieJar = true;
constructor(init) {
if (init && init.queueOptions?.enable) {
this.queue = new import_p_queue.default(init.queueOptions.options);
this.isQueueEnabled = true;
}
this.queueOptions = init?.queueOptions || null;
}
shouldRetry(status, isForbidden, isUnauthorized) {
if (status === 401 && isUnauthorized || status === 403 && isForbidden) return true;
return this.RETRYABLE_STATUS_CODES.has(status);
}
/**
* queueEnabled = true to enable PQueue instance for further http request, otherwise pass false to turn it off
*/
set queueEnabled(value) {
if (value && !this.isQueueEnabled) {
this.queue = new import_p_queue.default();
this.isQueueEnabled = true;
this.queueOptions = { ...this.queueOptions || {}, enable: value };
} else if (!value && this.isQueueEnabled) {
this.isQueueEnabled = false;
this.queueOptions = { ...this.queueOptions || {}, enable: value };
}
}
setQueueOptions(queueOptions) {
if (queueOptions.enable && !this.isQueueEnabled) {
this.queue = new import_p_queue.default(queueOptions.options);
this.isQueueEnabled = true;
} else if (!queueOptions.enable && this.isQueueEnabled) {
this.isQueueEnabled = false;
}
}
/**
* get the state of pQueue if its running or not.
*/
get queueEnabled() {
return this.isQueueEnabled;
}
/**
* Checks if the provided error is an instance of UniqhttError.
*
* @param error - The error object to check.
* @returns A boolean indicating whether the error is an instance of UniqhttError.
*/
isUniqhttError(error) {
return error?.name === "UniqhttError";
}
setCookies(cookies, url, startNew) {
if (startNew) this.jar.removeAllCookiesSync();
this.jar.setCookiesSync(cookies, url);
}
getCookies() {
return this.jar.cookies();
}
async get(input, config) {
return this.request(input, "GET", void 0, config);
}
// Clear all cookies for the current agent
clearCookies() {
this.jar?.removeAllCookiesSync();
}
async post(input, data, config) {
return this.request(input, "POST", data, config);
}
async postForm(input, data, config) {
let tempData = "";
if (data instanceof URLSearchParams) tempData = data.toString();
else if (data instanceof FormData) {
const keys = [];
for (const [key, value] of data.entries()) {
keys.push(`${key}=${value}`);
}
tempData = keys.join("&");
} else if (typeof data === "string") tempData = data;
else if (typeof data === "object") {
const params = new URLSearchParams();
for (const [key, value] of Object.entries(data)) {
params.append(key, value ? typeof value === "object" ? JSON.stringify(value) : value.toString() : "");
}
tempData = params.toString();
}
const headers = config?.headers ? config.headers instanceof Headers ? config.headers : new Headers(config.headers) : new Headers();
headers.set("Content-Type", "application/x-www-form-urlencoded");
return this.request(input, "POST", tempData, { ...config, headers, isFormData: true });
}
async postJson(input, data, config) {
let tempData = {};
if (typeof data === "string") {
tempData = this.parseJson(data);
if (!tempData) {
throw new Error("Invalid JSON string");
}
} else tempData = data;
const headers = config?.headers ? config.headers instanceof Headers ? config.headers : new Headers(config.headers) : new Headers();
headers.set("Content-Type", "application/json");
return this.request(input, "POST", JSON.stringify(tempData), { ...config, headers, isJson: true });
}
async put(input, data, config) {
return this.request(input, "PUT", data, config);
}
async putForm(input, data, config) {
let tempData = "";
if (data instanceof URLSearchParams) tempData = data.toString();
else if (data instanceof FormData) {
const keys = [];
for (const [key, value] of data.entries()) {
keys.push(`${key}=${value}`);
}
tempData = keys.join("&");
} else if (typeof data === "string") tempData = data;
else if (typeof data === "object") {
const params = new URLSearchParams();
for (const [key, value] of Object.entries(data)) {
params.append(key, value ? typeof value === "object" ? JSON.stringify(value) : value.toString() : "");
}
tempData = params.toString();
}
const headers = config?.headers ? config.headers instanceof Headers ? config.headers : new Headers(config.headers) : new Headers();
headers.set("Content-Type", "application/x-www-form-urlencoded");
return this.request(input, "PUT", tempData, { ...config, headers, isFormData: true });
}
async putJson(input, data, config) {
let tempData = {};
if (typeof data === "string") {
tempData = this.parseJson(data);
if (!tempData) {
throw new Error("Invalid JSON string");
}
} else tempData = data;
const headers = config?.headers ? config.headers instanceof Headers ? config.headers : new Headers(config.headers) : new Headers();
headers.set("Content-Type", "application/json");
return this.request(input, "PUT", JSON.stringify(tempData), { ...config, headers, isJson: true });
}
async patch(input, data, config) {
return this.request(input, "PATCH", data, config);
}
async patchForm(input, data, config) {
let tempData = "";
if (data instanceof URLSearchParams) tempData = data.toString();
else if (data instanceof FormData) {
const keys = [];
for (const [key, value] of data.entries()) {
keys.push(`${key}=${value}`);
}
tempData = keys.join("&");
} else if (typeof data === "string") tempData = data;
else if (typeof data === "object") {
const params = new URLSearchParams();
for (const [key, value] of Object.entries(data)) {
params.append(key, value ? typeof value === "object" ? JSON.stringify(value) : value.toString() : "");
}
tempData = params.toString();
}
const headers = config?.headers ? config.headers instanceof Headers ? config.headers : new Headers(config.headers) : new Headers();
headers.set("Content-Type", "application/x-www-form-urlencoded");
return this.request(input, "PATCH", tempData, { ...config, headers, isFormData: true });
}
async patchJson(input, data, config) {
let tempData = {};
if (typeof data === "string") {
tempData = this.parseJson(data);
if (!tempData) {
throw new Error("Invalid JSON string");
}
} else tempData = data;
const headers = config?.headers ? config.headers instanceof Headers ? config.headers : new Headers(config.headers) : new Headers();
headers.set("Content-Type", "application/json");
return this.request(input, "PATCH", JSON.stringify(tempData), { ...config, headers, isJson: true });
}
async delete(input, config) {
return this.request(input, "DELETE", void 0, config);
}
async head(input, config) {
return this.request(input, "HEAD", void 0, config);
}
async options(input, config) {
return this.request(input, "OPTIONS", void 0, config);
}
parseJson(data) {
try {
return JSON.parse(data);
} catch (e) {
return null;
}
}
async Error(response, message, config, urls, code) {
if (response instanceof Response) {
const contentType = response.headers.get("Content-Type");
const body = await this.parseResponseBody(response, contentType);
let headers = {};
if (response.headers instanceof Headers) {
response.headers.forEach((value, name) => {
if (name.toLowerCase() !== "set-cookie" && name.toLowerCase() !== "set-cookies") headers[name] = value;
});
}
return new UniqhttError2(message, response, body, code, headers, config || {}, urls);
} else if (response.headers) {
const body = response.body instanceof import_node_buffer.Buffer ? response?.body : response.body;
const headers = response.headers ?? {};
const contentType = response.contentType ?? null;
return new UniqhttError2(
message,
response,
body && contentType ? await this.parseResponseBody(response, contentType) : body ? body.toString("utf-8") : null,
code,
headers,
config || {},
urls
);
} else {
const res = {
status: response.status,
statusText: response.statusText,
url: response.url,
headers: response.headers ?? {},
body: null,
contentLength: 0,
cookies: {},
contentType: void 0
};
const headers = response.headers ?? {};
return new UniqhttError2(message, res, null, code, headers, config || {}, urls);
}
}
async formatResponse(response, finalUrl, isBuffer, config, downloadConfig, urls = [], cookies) {
const contentType = response instanceof Response ? response.headers.get("Content-Type") : response.contentType;
const body = !downloadConfig ? isBuffer ? response instanceof Response ? import_node_buffer.Buffer.from(await response.arrayBuffer()) : response.body : await this.parseResponseBody(response, contentType) : null;
let headers = {};
let length;
if (response instanceof Response) {
const _length = response.headers.get("content-length");
response.headers.forEach((value, name) => {
headers[name] = value;
});
if (_length) {
length = parseInt(_length);
}
} else {
const _length = response.headers["content-length"];
if (_length) {
length = parseInt(_length);
}
headers = response.headers;
}
return {
urls,
contentLength: length,
data: body,
status: response.status,
statusText: response.statusText,
headers,
finalUrl,
cookies: this.jar?.parseResponseCookies(cookies),
config,
contentType,
...downloadConfig
};
}
async parseResponseBody(response, contentType) {
if (contentType && contentType.includes("application/json") || contentType && contentType.includes("application/dns-json") || contentType && contentType.includes("application/jsonrequest")) {
return this.parseJsonData(response instanceof Response ? await response.text() : response.body ? response.body : "");
}
const textRelatedTypes = [
"text",
"xml",
"xhtml+xml",
"html",
"php",
"javascript",
"ecmascript",
"x-javascript",
"typescript",
"x-httpd-p