got
Version:
Human-friendly and powerful HTTP request library for Node.js
442 lines (441 loc) • 19.8 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
const url_1 = require("url");
const util_1 = require("util");
const CacheableRequest = require("cacheable-request");
const http = require("http");
const https = require("https");
const lowercaseKeys = require("lowercase-keys");
const toReadableStream = require("to-readable-stream");
const is_1 = require("@sindresorhus/is");
const cacheable_lookup_1 = require("cacheable-lookup");
const errors_1 = require("./errors");
const known_hook_events_1 = require("./known-hook-events");
const dynamic_require_1 = require("./utils/dynamic-require");
const get_body_size_1 = require("./utils/get-body-size");
const is_form_data_1 = require("./utils/is-form-data");
const merge_1 = require("./utils/merge");
const options_to_url_1 = require("./utils/options-to-url");
const supports_brotli_1 = require("./utils/supports-brotli");
const types_1 = require("./types");
const nonEnumerableProperties = [
'context',
'body',
'json',
'form'
];
const isAgentByProtocol = (agent) => is_1.default.object(agent);
// TODO: `preNormalizeArguments` should merge `options` & `defaults`
exports.preNormalizeArguments = (options, defaults) => {
var _a, _b, _c, _d, _e, _f;
// `options.headers`
if (is_1.default.undefined(options.headers)) {
options.headers = {};
}
else {
options.headers = lowercaseKeys(options.headers);
}
for (const [key, value] of Object.entries(options.headers)) {
if (is_1.default.null_(value)) {
throw new TypeError(`Use \`undefined\` instead of \`null\` to delete the \`${key}\` header`);
}
}
// `options.prefixUrl`
if (is_1.default.urlInstance(options.prefixUrl) || is_1.default.string(options.prefixUrl)) {
options.prefixUrl = options.prefixUrl.toString();
if (options.prefixUrl.length !== 0 && !options.prefixUrl.endsWith('/')) {
options.prefixUrl += '/';
}
}
else {
options.prefixUrl = defaults ? defaults.prefixUrl : '';
}
// `options.hooks`
if (is_1.default.undefined(options.hooks)) {
options.hooks = {};
}
if (is_1.default.object(options.hooks)) {
for (const event of known_hook_events_1.default) {
if (Reflect.has(options.hooks, event)) {
if (!is_1.default.array(options.hooks[event])) {
throw new TypeError(`Parameter \`${event}\` must be an Array, not ${is_1.default(options.hooks[event])}`);
}
}
else {
options.hooks[event] = [];
}
}
}
else {
throw new TypeError(`Parameter \`hooks\` must be an Object, not ${is_1.default(options.hooks)}`);
}
if (defaults) {
for (const event of known_hook_events_1.default) {
if (!(Reflect.has(options.hooks, event) && is_1.default.undefined(options.hooks[event]))) {
// @ts-ignore Union type array is not assignable to union array type
options.hooks[event] = [
...defaults.hooks[event],
...options.hooks[event]
];
}
}
}
// `options.timeout`
if (is_1.default.number(options.timeout)) {
options.timeout = { request: options.timeout };
}
else if (!is_1.default.object(options.timeout)) {
options.timeout = {};
}
// `options.retry`
const { retry } = options;
if (defaults) {
options.retry = { ...defaults.retry };
}
else {
options.retry = {
calculateDelay: retryObject => retryObject.computedValue,
limit: 0,
methods: [],
statusCodes: [],
errorCodes: [],
maxRetryAfter: undefined
};
}
if (is_1.default.object(retry)) {
options.retry = {
...options.retry,
...retry
};
}
else if (is_1.default.number(retry)) {
options.retry.limit = retry;
}
if (options.retry.maxRetryAfter === undefined) {
options.retry.maxRetryAfter = Math.min(...[options.timeout.request, options.timeout.connect].filter((n) => !is_1.default.nullOrUndefined(n)));
}
options.retry.methods = [...new Set(options.retry.methods.map(method => method.toUpperCase()))];
options.retry.statusCodes = [...new Set(options.retry.statusCodes)];
options.retry.errorCodes = [...new Set(options.retry.errorCodes)];
// `options.dnsCache`
if (options.dnsCache && !(options.dnsCache instanceof cacheable_lookup_1.default)) {
options.dnsCache = new cacheable_lookup_1.default({ cacheAdapter: options.dnsCache });
}
// `options.method`
if (is_1.default.string(options.method)) {
options.method = options.method.toUpperCase();
}
else {
options.method = (_b = (_a = defaults) === null || _a === void 0 ? void 0 : _a.method, (_b !== null && _b !== void 0 ? _b : 'GET'));
}
// Better memory management, so we don't have to generate a new object every time
if (options.cache) {
options.cacheableRequest = new CacheableRequest(
// @ts-ignore Cannot properly type a function with multiple definitions yet
(requestOptions, handler) => requestOptions[types_1.requestSymbol](requestOptions, handler), options.cache);
}
// `options.cookieJar`
if (is_1.default.object(options.cookieJar)) {
let { setCookie, getCookieString } = options.cookieJar;
// Horrible `tough-cookie` check
if (setCookie.length === 4 && getCookieString.length === 0) {
if (!Reflect.has(setCookie, util_1.promisify.custom)) {
// @ts-ignore TS is dumb - it says `setCookie` is `never`.
setCookie = util_1.promisify(setCookie.bind(options.cookieJar));
getCookieString = util_1.promisify(getCookieString.bind(options.cookieJar));
}
}
else if (setCookie.length !== 2) {
throw new TypeError('`options.cookieJar.setCookie` needs to be an async function with 2 arguments');
}
else if (getCookieString.length !== 1) {
throw new TypeError('`options.cookieJar.getCookieString` needs to be an async function with 1 argument');
}
options.cookieJar = { setCookie, getCookieString };
}
// `options.encoding`
if (is_1.default.null_(options.encoding)) {
throw new TypeError('To get a Buffer, set `options.responseType` to `buffer` instead');
}
// `options.maxRedirects`
if (!Reflect.has(options, 'maxRedirects') && !(defaults && Reflect.has(defaults, 'maxRedirects'))) {
options.maxRedirects = 0;
}
// Merge defaults
if (defaults) {
options = merge_1.default({}, defaults, options);
}
// `options._pagination`
if (is_1.default.object(options._pagination)) {
const { _pagination: pagination } = options;
if (!is_1.default.function_(pagination.transform)) {
throw new TypeError('`options._pagination.transform` must be implemented');
}
if (!is_1.default.function_(pagination.shouldContinue)) {
throw new TypeError('`options._pagination.shouldContinue` must be implemented');
}
if (!is_1.default.function_(pagination.filter)) {
throw new TypeError('`options._pagination.filter` must be implemented');
}
if (!is_1.default.function_(pagination.paginate)) {
throw new TypeError('`options._pagination.paginate` must be implemented');
}
}
// Other values
options.decompress = Boolean(options.decompress);
options.isStream = Boolean(options.isStream);
options.throwHttpErrors = Boolean(options.throwHttpErrors);
options.ignoreInvalidCookies = Boolean(options.ignoreInvalidCookies);
options.cache = (_c = options.cache, (_c !== null && _c !== void 0 ? _c : false));
options.responseType = (_d = options.responseType, (_d !== null && _d !== void 0 ? _d : 'text'));
options.resolveBodyOnly = Boolean(options.resolveBodyOnly);
options.followRedirect = Boolean(options.followRedirect);
options.dnsCache = (_e = options.dnsCache, (_e !== null && _e !== void 0 ? _e : false));
options.useElectronNet = Boolean(options.useElectronNet);
options.methodRewriting = Boolean(options.methodRewriting);
options.allowGetBody = Boolean(options.allowGetBody);
options.context = (_f = options.context, (_f !== null && _f !== void 0 ? _f : {}));
return options;
};
exports.mergeOptions = (...sources) => {
let mergedOptions = exports.preNormalizeArguments({});
// Non enumerable properties shall not be merged
const properties = {};
for (const source of sources) {
mergedOptions = exports.preNormalizeArguments(merge_1.default({}, source), mergedOptions);
for (const name of nonEnumerableProperties) {
if (!Reflect.has(source, name)) {
continue;
}
properties[name] = {
writable: true,
configurable: true,
enumerable: false,
value: source[name]
};
}
}
Object.defineProperties(mergedOptions, properties);
return mergedOptions;
};
exports.normalizeArguments = (url, options, defaults) => {
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k;
// Merge options
if (typeof url === 'undefined') {
throw new TypeError('Missing `url` argument');
}
const runInitHooks = (hooks, options) => {
if (hooks && options) {
for (const hook of hooks) {
const result = hook(options);
if (is_1.default.promise(result)) {
throw new TypeError('The `init` hook must be a synchronous function');
}
}
}
};
const hasUrl = is_1.default.urlInstance(url) || is_1.default.string(url);
if (hasUrl) {
if (options) {
if (Reflect.has(options, 'url')) {
throw new TypeError('The `url` option cannot be used if the input is a valid URL.');
}
}
else {
options = {};
}
// @ts-ignore URL is not URL
options.url = url;
runInitHooks((_a = defaults) === null || _a === void 0 ? void 0 : _a.options.hooks.init, options);
runInitHooks((_b = options.hooks) === null || _b === void 0 ? void 0 : _b.init, options);
}
else if (Reflect.has(url, 'resolve')) {
throw new Error('The legacy `url.Url` is deprecated. Use `URL` instead.');
}
else {
runInitHooks((_c = defaults) === null || _c === void 0 ? void 0 : _c.options.hooks.init, url);
runInitHooks((_d = url.hooks) === null || _d === void 0 ? void 0 : _d.init, url);
if (options) {
runInitHooks((_e = defaults) === null || _e === void 0 ? void 0 : _e.options.hooks.init, options);
runInitHooks((_f = options.hooks) === null || _f === void 0 ? void 0 : _f.init, options);
}
}
if (hasUrl) {
options = exports.mergeOptions((_h = (_g = defaults) === null || _g === void 0 ? void 0 : _g.options, (_h !== null && _h !== void 0 ? _h : {})), (options !== null && options !== void 0 ? options : {}));
}
else {
options = exports.mergeOptions((_k = (_j = defaults) === null || _j === void 0 ? void 0 : _j.options, (_k !== null && _k !== void 0 ? _k : {})), url, (options !== null && options !== void 0 ? options : {}));
}
// Normalize URL
// TODO: drop `optionsToUrl` in Got 12
if (is_1.default.string(options.url)) {
options.url = options.prefixUrl + options.url;
options.url = options.url.replace(/^unix:/, 'http://$&');
if (options.searchParams || options.search) {
options.url = options.url.split('?')[0];
}
// @ts-ignore URL is not URL
options.url = options_to_url_1.default({
origin: options.url,
...options
});
}
else if (!is_1.default.urlInstance(options.url)) {
// @ts-ignore URL is not URL
options.url = options_to_url_1.default({ origin: options.prefixUrl, ...options });
}
const normalizedOptions = options;
// Make it possible to change `options.prefixUrl`
let prefixUrl = options.prefixUrl;
Object.defineProperty(normalizedOptions, 'prefixUrl', {
set: (value) => {
if (!normalizedOptions.url.href.startsWith(value)) {
throw new Error(`Cannot change \`prefixUrl\` from ${prefixUrl} to ${value}: ${normalizedOptions.url.href}`);
}
normalizedOptions.url = new url_1.URL(value + normalizedOptions.url.href.slice(prefixUrl.length));
prefixUrl = value;
},
get: () => prefixUrl
});
// Make it possible to remove default headers
for (const [key, value] of Object.entries(normalizedOptions.headers)) {
if (is_1.default.undefined(value)) {
// eslint-disable-next-line @typescript-eslint/no-dynamic-delete
delete normalizedOptions.headers[key];
}
}
return normalizedOptions;
};
const withoutBody = new Set(['HEAD']);
const withoutBodyUnlessSpecified = 'GET';
exports.normalizeRequestArguments = async (options) => {
var _a, _b, _c;
options = exports.mergeOptions(options);
// Serialize body
const { headers } = options;
const hasNoContentType = is_1.default.undefined(headers['content-type']);
{
// TODO: these checks should be moved to `preNormalizeArguments`
const isForm = !is_1.default.undefined(options.form);
const isJson = !is_1.default.undefined(options.json);
const isBody = !is_1.default.undefined(options.body);
if ((isBody || isForm || isJson) && withoutBody.has(options.method)) {
throw new TypeError(`The \`${options.method}\` method cannot be used with a body`);
}
if (!options.allowGetBody && (isBody || isForm || isJson) && withoutBodyUnlessSpecified === options.method) {
throw new TypeError(`The \`${options.method}\` method cannot be used with a body`);
}
if ([isBody, isForm, isJson].filter(isTrue => isTrue).length > 1) {
throw new TypeError('The `body`, `json` and `form` options are mutually exclusive');
}
if (isBody &&
!is_1.default.nodeStream(options.body) &&
!is_1.default.string(options.body) &&
!is_1.default.buffer(options.body) &&
!(is_1.default.object(options.body) && is_form_data_1.default(options.body))) {
throw new TypeError('The `body` option must be a stream.Readable, string or Buffer');
}
if (isForm && !is_1.default.object(options.form)) {
throw new TypeError('The `form` option must be an Object');
}
}
if (options.body) {
// Special case for https://github.com/form-data/form-data
if (is_1.default.object(options.body) && is_form_data_1.default(options.body) && hasNoContentType) {
headers['content-type'] = `multipart/form-data; boundary=${options.body.getBoundary()}`;
}
}
else if (options.form) {
if (hasNoContentType) {
headers['content-type'] = 'application/x-www-form-urlencoded';
}
options.body = (new url_1.URLSearchParams(options.form)).toString();
}
else if (options.json) {
if (hasNoContentType) {
headers['content-type'] = 'application/json';
}
options.body = JSON.stringify(options.json);
}
const uploadBodySize = await get_body_size_1.default(options);
if (!is_1.default.nodeStream(options.body)) {
options.body = toReadableStream(options.body);
}
// See https://tools.ietf.org/html/rfc7230#section-3.3.2
// A user agent SHOULD send a Content-Length in a request message when
// no Transfer-Encoding is sent and the request method defines a meaning
// for an enclosed payload body. For example, a Content-Length header
// field is normally sent in a POST request even when the value is 0
// (indicating an empty payload body). A user agent SHOULD NOT send a
// Content-Length header field when the request message does not contain
// a payload body and the method semantics do not anticipate such a
// body.
if (is_1.default.undefined(headers['content-length']) && is_1.default.undefined(headers['transfer-encoding'])) {
if ((options.method === 'POST' || options.method === 'PUT' || options.method === 'PATCH' || options.method === 'DELETE' || (options.allowGetBody && options.method === 'GET')) &&
!is_1.default.undefined(uploadBodySize)) {
// @ts-ignore We assign if it is undefined, so this IS correct
headers['content-length'] = String(uploadBodySize);
}
}
if (!options.isStream && options.responseType === 'json' && is_1.default.undefined(headers.accept)) {
headers.accept = 'application/json';
}
if (options.decompress && is_1.default.undefined(headers['accept-encoding'])) {
headers['accept-encoding'] = supports_brotli_1.default ? 'gzip, deflate, br' : 'gzip, deflate';
}
// Validate URL
if (options.url.protocol !== 'http:' && options.url.protocol !== 'https:') {
throw new errors_1.UnsupportedProtocolError(options);
}
decodeURI(options.url.toString());
// Normalize request function
if (is_1.default.function_(options.request)) {
options[types_1.requestSymbol] = options.request;
delete options.request;
}
else {
options[types_1.requestSymbol] = options.url.protocol === 'https:' ? https.request : http.request;
}
// UNIX sockets
if (options.url.hostname === 'unix') {
const matches = /(?<socketPath>.+?):(?<path>.+)/.exec(options.url.pathname);
if ((_a = matches) === null || _a === void 0 ? void 0 : _a.groups) {
const { socketPath, path } = matches.groups;
options = {
...options,
socketPath,
path,
host: ''
};
}
}
if (isAgentByProtocol(options.agent)) {
options.agent = (_b = options.agent[options.url.protocol.slice(0, -1)], (_b !== null && _b !== void 0 ? _b : options.agent));
}
if (options.dnsCache) {
options.lookup = options.dnsCache.lookup;
}
/* istanbul ignore next: electron.net is broken */
// No point in typing process.versions correctly, as
// `process.version.electron` is used only once, right here.
if (options.useElectronNet && process.versions.electron) {
const electron = dynamic_require_1.default(module, 'electron'); // Trick webpack
options.request = util_1.deprecate((_c = electron.net.request, (_c !== null && _c !== void 0 ? _c : electron.remote.net.request)), 'Electron support has been deprecated and will be removed in Got 11.\n' +
'See https://github.com/sindresorhus/got/issues/899 for further information.', 'GOT_ELECTRON');
}
// Got's `timeout` is an object, http's `timeout` is a number, so they're not compatible.
delete options.timeout;
// Set cookies
if (options.cookieJar) {
const cookieString = await options.cookieJar.getCookieString(options.url.toString());
if (is_1.default.nonEmptyString(cookieString)) {
options.headers.cookie = cookieString;
}
else {
delete options.headers.cookie;
}
}
// `http-cache-semantics` checks this
delete options.url;
return options;
};