urllib
Version:
Help in opening URLs (mostly HTTP) in a complex world — basic and digest authentication, redirections, cookies and more. Base undici fetch API.
550 lines • 23.5 kB
JavaScript
"use strict";
var _a, _b, _c;
var _BlobFromStream_stream, _BlobFromStream_type, _HttpClient_instances, _HttpClient_defaultArgs, _HttpClient_dispatcher, _HttpClient_requestInternal, _HttpClient_updateSocketInfo;
Object.defineProperty(exports, "__esModule", { value: true });
exports.HttpClient = exports.HEADER_USER_AGENT = void 0;
const tslib_1 = require("tslib");
const events_1 = require("events");
const util_1 = require("util");
const zlib_1 = require("zlib");
const buffer_1 = require("buffer");
const stream_1 = require("stream");
const stream_2 = tslib_1.__importDefault(require("stream"));
const path_1 = require("path");
const fs_1 = require("fs");
const url_1 = require("url");
const perf_hooks_1 = require("perf_hooks");
const undici_1 = require("undici");
const formdata_node_1 = require("formdata-node");
const form_data_encoder_1 = require("form-data-encoder");
const default_user_agent_1 = tslib_1.__importDefault(require("default-user-agent"));
const mime_types_1 = tslib_1.__importDefault(require("mime-types"));
const pump_1 = tslib_1.__importDefault(require("pump"));
const HttpAgent_1 = require("./HttpAgent");
const utils_1 = require("./utils");
const symbols_1 = tslib_1.__importDefault(require("./symbols"));
const diagnosticsChannel_1 = require("./diagnosticsChannel");
const PROTO_RE = /^https?:\/\//i;
const FormData = undici_1.FormData !== null && undici_1.FormData !== void 0 ? undici_1.FormData : formdata_node_1.FormData;
// impl isReadable on Node.js 14
const isReadable = (_a = stream_2.default.isReadable) !== null && _a !== void 0 ? _a : function isReadable(stream) {
return stream && typeof stream.read === 'function';
};
// impl promise pipeline on Node.js 14
const pipelinePromise = (_c = (_b = stream_2.default.promises) === null || _b === void 0 ? void 0 : _b.pipeline) !== null && _c !== void 0 ? _c : function pipeline(...args) {
return new Promise((resolve, reject) => {
(0, pump_1.default)(...args, (err) => {
if (err)
return reject(err);
resolve();
});
});
};
function noop() {
// noop
}
const debug = (0, util_1.debuglog)('urllib:HttpClient');
// https://github.com/octet-stream/form-data
class BlobFromStream {
constructor(stream, type) {
_BlobFromStream_stream.set(this, void 0);
_BlobFromStream_type.set(this, void 0);
tslib_1.__classPrivateFieldSet(this, _BlobFromStream_stream, stream, "f");
tslib_1.__classPrivateFieldSet(this, _BlobFromStream_type, type, "f");
}
stream() {
return tslib_1.__classPrivateFieldGet(this, _BlobFromStream_stream, "f");
}
get type() {
return tslib_1.__classPrivateFieldGet(this, _BlobFromStream_type, "f");
}
get [(_BlobFromStream_stream = new WeakMap(), _BlobFromStream_type = new WeakMap(), Symbol.toStringTag)]() {
return 'Blob';
}
}
class HttpClientRequestTimeoutError extends Error {
constructor(timeout, options) {
const message = `Request timeout for ${timeout} ms`;
super(message, options);
this.name = this.constructor.name;
Error.captureStackTrace(this, this.constructor);
}
}
exports.HEADER_USER_AGENT = (0, default_user_agent_1.default)('node-urllib', '3.0.0');
function getFileName(stream) {
const filePath = stream.path;
if (filePath) {
return (0, path_1.basename)(filePath);
}
return '';
}
function defaultIsRetry(response) {
return response.status >= 500;
}
class HttpClient extends events_1.EventEmitter {
constructor(clientOptions) {
super();
_HttpClient_instances.add(this);
_HttpClient_defaultArgs.set(this, void 0);
_HttpClient_dispatcher.set(this, void 0);
tslib_1.__classPrivateFieldSet(this, _HttpClient_defaultArgs, clientOptions === null || clientOptions === void 0 ? void 0 : clientOptions.defaultArgs, "f");
if ((clientOptions === null || clientOptions === void 0 ? void 0 : clientOptions.lookup) || (clientOptions === null || clientOptions === void 0 ? void 0 : clientOptions.checkAddress) || (clientOptions === null || clientOptions === void 0 ? void 0 : clientOptions.connect)) {
tslib_1.__classPrivateFieldSet(this, _HttpClient_dispatcher, new HttpAgent_1.HttpAgent({
lookup: clientOptions.lookup,
checkAddress: clientOptions.checkAddress,
connect: clientOptions.connect,
}), "f");
}
(0, diagnosticsChannel_1.initDiagnosticsChannel)();
}
async request(url, options) {
return await tslib_1.__classPrivateFieldGet(this, _HttpClient_instances, "m", _HttpClient_requestInternal).call(this, url, options);
}
}
exports.HttpClient = HttpClient;
_HttpClient_defaultArgs = new WeakMap(), _HttpClient_dispatcher = new WeakMap(), _HttpClient_instances = new WeakSet(), _HttpClient_requestInternal = async function _HttpClient_requestInternal(url, options, requestContext) {
var _a, _b, _c, _d, _e, _f, _g;
const requestId = (0, utils_1.globalId)('HttpClientRequest');
let requestUrl;
if (typeof url === 'string') {
if (!PROTO_RE.test(url)) {
// Support `request('www.server.com')`
url = 'http://' + url;
}
requestUrl = new URL(url);
}
else {
if (!url.searchParams) {
// url maybe url.parse(url) object in urllib2
requestUrl = new URL((0, url_1.format)(url));
}
else {
requestUrl = url;
}
}
const method = ((_a = options === null || options === void 0 ? void 0 : options.method) !== null && _a !== void 0 ? _a : 'GET').toUpperCase();
const orginalHeaders = options === null || options === void 0 ? void 0 : options.headers;
const headers = {};
const args = {
retry: 0,
...tslib_1.__classPrivateFieldGet(this, _HttpClient_defaultArgs, "f"),
...options,
// keep method and headers exists on args for request event handler to easy use
method,
headers,
};
requestContext = {
retries: 0,
...requestContext,
};
const requestStartTime = perf_hooks_1.performance.now();
// https://developer.chrome.com/docs/devtools/network/reference/?utm_source=devtools#timing-explanation
const timing = {
// socket assigned
queuing: 0,
// dns lookup time
// dnslookup: 0,
// socket connected
connected: 0,
// request headers sent
requestHeadersSent: 0,
// request sent, including headers and body
requestSent: 0,
// Time to first byte (TTFB), the response headers have been received
waiting: 0,
// the response body and trailers have been received
contentDownload: 0,
};
const orginalOpaque = args.opaque;
// using opaque to diagnostics channel, binding request and socket
const internalOpaque = {
[symbols_1.default.kRequestId]: requestId,
[symbols_1.default.kRequestStartTime]: requestStartTime,
[symbols_1.default.kEnableRequestTiming]: !!args.timing,
[symbols_1.default.kRequestTiming]: timing,
[symbols_1.default.kRequestOrginalOpaque]: orginalOpaque,
};
const reqMeta = {
requestId,
url: requestUrl.href,
args,
ctx: args.ctx,
retries: requestContext.retries,
};
const socketInfo = {
id: 0,
localAddress: '',
localPort: 0,
remoteAddress: '',
remotePort: 0,
remoteFamily: '',
bytesWritten: 0,
bytesRead: 0,
handledRequests: 0,
handledResponses: 0,
};
// keep urllib createCallbackResponse style
const resHeaders = {};
const res = {
status: -1,
statusCode: -1,
headers: resHeaders,
size: 0,
aborted: false,
rt: 0,
keepAliveSocket: true,
requestUrls: [],
timing,
socket: socketInfo,
};
let headersTimeout = 5000;
let bodyTimeout = 5000;
if (args.timeout) {
if (Array.isArray(args.timeout)) {
headersTimeout = (_b = args.timeout[0]) !== null && _b !== void 0 ? _b : headersTimeout;
bodyTimeout = (_c = args.timeout[1]) !== null && _c !== void 0 ? _c : bodyTimeout;
}
else {
headersTimeout = bodyTimeout = args.timeout;
}
}
if (orginalHeaders) {
// convert headers to lower-case
for (const name in orginalHeaders) {
headers[name.toLowerCase()] = orginalHeaders[name];
}
}
// hidden user-agent
const hiddenUserAgent = 'user-agent' in headers && !headers['user-agent'];
if (hiddenUserAgent) {
delete headers['user-agent'];
}
else if (!headers['user-agent']) {
// need to set user-agent
headers['user-agent'] = exports.HEADER_USER_AGENT;
}
// Alias to dataType = 'stream'
if (args.streaming || args.customResponse) {
args.dataType = 'stream';
}
if (args.dataType === 'json' && !headers.accept) {
headers.accept = 'application/json';
}
// gzip alias to compressed
if (args.gzip && args.compressed !== false) {
args.compressed = true;
}
if (args.compressed && !headers['accept-encoding']) {
headers['accept-encoding'] = 'gzip, br';
}
if (requestContext.retries > 0) {
headers['x-urllib-retry'] = `${requestContext.retries}/${args.retry}`;
}
if (args.auth && !headers.authorization) {
headers.authorization = `Basic ${Buffer.from(args.auth).toString('base64')}`;
}
try {
const requestOptions = {
method,
keepalive: true,
maxRedirections: (_d = args.maxRedirects) !== null && _d !== void 0 ? _d : 10,
headersTimeout,
bodyTimeout,
opaque: internalOpaque,
dispatcher: (_e = args.dispatcher) !== null && _e !== void 0 ? _e : tslib_1.__classPrivateFieldGet(this, _HttpClient_dispatcher, "f"),
};
if (args.followRedirect === false) {
requestOptions.maxRedirections = 0;
}
const isGETOrHEAD = requestOptions.method === 'GET' || requestOptions.method === 'HEAD';
// alias to args.content
if (args.stream && !args.content) {
args.content = args.stream;
}
if (args.files) {
if (isGETOrHEAD) {
requestOptions.method = 'POST';
}
const formData = new FormData();
const uploadFiles = [];
if (Array.isArray(args.files)) {
for (const [index, file] of args.files.entries()) {
const field = index === 0 ? 'file' : `file${index}`;
uploadFiles.push([field, file]);
}
}
else if (args.files instanceof stream_1.Readable || isReadable(args.files)) {
uploadFiles.push(['file', args.files]);
}
else if (typeof args.files === 'string' || Buffer.isBuffer(args.files)) {
uploadFiles.push(['file', args.files]);
}
else if (typeof args.files === 'object') {
for (const field in args.files) {
uploadFiles.push([field, args.files[field]]);
}
}
// set normal fields first
if (args.data) {
for (const field in args.data) {
formData.append(field, args.data[field]);
}
}
for (const [index, [field, file]] of uploadFiles.entries()) {
if (typeof file === 'string') {
// FIXME: support non-ascii filename
// const fileName = encodeURIComponent(basename(file));
// formData.append(field, await fileFromPath(file, `utf-8''${fileName}`, { type: mime.lookup(fileName) || '' }));
const fileName = (0, path_1.basename)(file);
const fileReadable = (0, fs_1.createReadStream)(file);
formData.append(field, new BlobFromStream(fileReadable, mime_types_1.default.lookup(fileName) || ''), fileName);
}
else if (Buffer.isBuffer(file)) {
formData.append(field, new buffer_1.Blob([file]), `bufferfile${index}`);
}
else if (file instanceof stream_1.Readable || isReadable(file)) {
const fileName = getFileName(file) || `streamfile${index}`;
formData.append(field, new BlobFromStream(file, mime_types_1.default.lookup(fileName) || ''), fileName);
}
}
if (undici_1.FormData) {
requestOptions.body = formData;
}
else {
// Node.js 14 does not support spec-compliant FormData
// https://github.com/octet-stream/form-data#usage
const encoder = new form_data_encoder_1.FormDataEncoder(formData);
Object.assign(headers, encoder.headers);
// fix "Content-Length":"NaN"
delete headers['Content-Length'];
requestOptions.body = stream_1.Readable.from(encoder);
}
}
else if (args.content) {
if (!isGETOrHEAD) {
// handle content
requestOptions.body = args.content;
if (args.contentType) {
headers['content-type'] = args.contentType;
}
else if (typeof args.content === 'string' && !headers['content-type']) {
headers['content-type'] = 'text/plain;charset=UTF-8';
}
}
}
else if (args.data) {
const isStringOrBufferOrReadable = typeof args.data === 'string'
|| Buffer.isBuffer(args.data)
|| isReadable(args.data);
if (isGETOrHEAD) {
if (!isStringOrBufferOrReadable) {
for (const field in args.data) {
requestUrl.searchParams.append(field, args.data[field]);
}
}
}
else {
if (isStringOrBufferOrReadable) {
requestOptions.body = args.data;
}
else {
if (args.contentType === 'json'
|| args.contentType === 'application/json'
|| ((_f = headers['content-type']) === null || _f === void 0 ? void 0 : _f.startsWith('application/json'))) {
requestOptions.body = JSON.stringify(args.data);
if (!headers['content-type']) {
headers['content-type'] = 'application/json';
}
}
else {
headers['content-type'] = 'application/x-www-form-urlencoded;charset=UTF-8';
requestOptions.body = new URLSearchParams(args.data).toString();
}
}
}
}
debug('Request#%d %s %s, headers: %j, headersTimeout: %s, bodyTimeout: %s', requestId, requestOptions.method, requestUrl.href, headers, headersTimeout, bodyTimeout);
requestOptions.headers = headers;
if (this.listenerCount('request') > 0) {
this.emit('request', reqMeta);
}
let response = await (0, undici_1.request)(requestUrl, requestOptions);
// handle digest auth
if (response.statusCode === 401 && response.headers['www-authenticate'] &&
!requestOptions.headers.authorization && args.digestAuth) {
const authenticate = response.headers['www-authenticate'];
if (authenticate.startsWith('Digest ')) {
debug('Request#%d %s: got digest auth header WWW-Authenticate: %s', requestId, requestUrl.href, authenticate);
requestOptions.headers.authorization = (0, utils_1.digestAuthHeader)(requestOptions.method, `${requestUrl.pathname}${requestUrl.search}`, authenticate, args.digestAuth);
debug('Request#%d %s: auth with digest header: %s', requestId, url, requestOptions.headers.authorization);
if (response.headers['set-cookie']) {
// FIXME: merge exists cookie header
requestOptions.headers.cookie = response.headers['set-cookie'].join(';');
}
response = await (0, undici_1.request)(requestUrl, requestOptions);
}
}
const context = response.context;
let lastUrl = '';
if (context === null || context === void 0 ? void 0 : context.history) {
for (const urlObject of context === null || context === void 0 ? void 0 : context.history) {
res.requestUrls.push(urlObject.href);
lastUrl = urlObject.href;
}
}
else {
res.requestUrls.push(requestUrl.href);
lastUrl = requestUrl.href;
}
const contentEncoding = response.headers['content-encoding'];
const isCompressedContent = contentEncoding === 'gzip' || contentEncoding === 'br';
res.headers = response.headers;
res.status = res.statusCode = response.statusCode;
if (res.headers['content-length']) {
res.size = parseInt(res.headers['content-length']);
}
let data = null;
let responseBodyStream;
if (args.dataType === 'stream') {
// streaming mode will disable retry
args.retry = 0;
const meta = {
status: res.status,
statusCode: res.statusCode,
headers: res.headers,
timing,
socket: socketInfo,
};
if (isCompressedContent) {
// gzip or br
const decoder = contentEncoding === 'gzip' ? (0, zlib_1.createGunzip)() : (0, zlib_1.createBrotliDecompress)();
responseBodyStream = Object.assign((0, stream_1.pipeline)(response.body, decoder, noop), meta);
}
else {
responseBodyStream = Object.assign(response.body, meta);
}
}
else if (args.writeStream) {
// streaming mode will disable retry
args.retry = 0;
if (isCompressedContent) {
const decoder = contentEncoding === 'gzip' ? (0, zlib_1.createGunzip)() : (0, zlib_1.createBrotliDecompress)();
await pipelinePromise(response.body, decoder, args.writeStream);
}
else {
await pipelinePromise(response.body, args.writeStream);
}
}
else {
// buffer
data = Buffer.from(await response.body.arrayBuffer());
if (isCompressedContent && data.length > 0) {
try {
data = contentEncoding === 'gzip' ? (0, zlib_1.gunzipSync)(data) : (0, zlib_1.brotliDecompressSync)(data);
}
catch (err) {
if (err.name === 'Error') {
err.name = 'UnzipError';
}
throw err;
}
}
if (args.dataType === 'text') {
data = data.toString();
}
else if (args.dataType === 'json') {
if (data.length === 0) {
data = null;
}
else {
data = (0, utils_1.parseJSON)(data.toString(), args.fixJSONCtlChars);
}
}
}
res.rt = (0, utils_1.performanceTime)(requestStartTime);
// get real socket info from internalOpaque
tslib_1.__classPrivateFieldGet(this, _HttpClient_instances, "m", _HttpClient_updateSocketInfo).call(this, socketInfo, internalOpaque);
const clientResponse = {
opaque: orginalOpaque,
data,
status: res.status,
statusCode: res.status,
headers: res.headers,
url: lastUrl,
redirected: res.requestUrls.length > 1,
requestUrls: res.requestUrls,
res: responseBodyStream !== null && responseBodyStream !== void 0 ? responseBodyStream : res,
};
if (args.retry > 0 && requestContext.retries < args.retry) {
const isRetry = (_g = args.isRetry) !== null && _g !== void 0 ? _g : defaultIsRetry;
if (isRetry(clientResponse)) {
if (args.retryDelay) {
await (0, utils_1.sleep)(args.retryDelay);
}
requestContext.retries++;
return await tslib_1.__classPrivateFieldGet(this, _HttpClient_instances, "m", _HttpClient_requestInternal).call(this, url, options, requestContext);
}
}
if (this.listenerCount('response') > 0) {
this.emit('response', {
requestId,
error: null,
ctx: args.ctx,
req: {
...reqMeta,
options: args,
},
res,
});
}
return clientResponse;
}
catch (e) {
debug('Request#%d throw error: %s', requestId, e);
let err = e;
if (err.name === 'HeadersTimeoutError') {
err = new HttpClientRequestTimeoutError(headersTimeout, { cause: e });
}
else if (err.name === 'BodyTimeoutError') {
err = new HttpClientRequestTimeoutError(bodyTimeout, { cause: e });
}
err.opaque = orginalOpaque;
err.status = res.status;
err.headers = res.headers;
err.res = res;
// make sure requestUrls not empty
if (res.requestUrls.length === 0) {
res.requestUrls.push(requestUrl.href);
}
res.rt = (0, utils_1.performanceTime)(requestStartTime);
tslib_1.__classPrivateFieldGet(this, _HttpClient_instances, "m", _HttpClient_updateSocketInfo).call(this, socketInfo, internalOpaque);
if (this.listenerCount('response') > 0) {
this.emit('response', {
requestId,
error: err,
ctx: args.ctx,
req: {
...reqMeta,
options: args,
},
res,
});
}
throw err;
}
}, _HttpClient_updateSocketInfo = function _HttpClient_updateSocketInfo(socketInfo, internalOpaque) {
const socket = internalOpaque[symbols_1.default.kRequestSocket];
if (socket) {
socketInfo.id = socket[symbols_1.default.kSocketId];
socketInfo.handledRequests = socket[symbols_1.default.kHandledRequests];
socketInfo.handledResponses = socket[symbols_1.default.kHandledResponses];
socketInfo.localAddress = socket.localAddress;
socketInfo.localPort = socket.localPort;
socketInfo.remoteAddress = socket.remoteAddress;
socketInfo.remotePort = socket.remotePort;
socketInfo.remoteFamily = socket.remoteFamily;
socketInfo.bytesRead = socket.bytesRead;
socketInfo.bytesWritten = socket.bytesWritten;
}
};
//# sourceMappingURL=HttpClient.js.map