urllib
Version:
Help in opening URLs (mostly HTTP) in a complex world — basic and digest authentication, redirections, cookies and more. Base undici fetch API.
546 lines • 22.8 kB
JavaScript
var _a, _b, _c;
var _BlobFromStream_stream, _BlobFromStream_type, _HttpClient_instances, _HttpClient_defaultArgs, _HttpClient_dispatcher, _HttpClient_requestInternal, _HttpClient_updateSocketInfo;
import { __classPrivateFieldGet, __classPrivateFieldSet } from "tslib";
import { EventEmitter } from 'events';
import { debuglog } from 'util';
import { createGunzip, createBrotliDecompress, gunzipSync, brotliDecompressSync, } from 'zlib';
import { Blob } from 'buffer';
import { Readable, pipeline } from 'stream';
import stream from 'stream';
import { basename } from 'path';
import { createReadStream } from 'fs';
import { format as urlFormat } from 'url';
import { performance } from 'perf_hooks';
import { FormData as FormDataNative, request as undiciRequest, } from 'undici';
import { FormData as FormDataNode } from 'formdata-node';
import { FormDataEncoder } from 'form-data-encoder';
import createUserAgent from 'default-user-agent';
import mime from 'mime-types';
import pump from 'pump';
import { HttpAgent } from './HttpAgent.js';
import { parseJSON, sleep, digestAuthHeader, globalId, performanceTime } from './utils.js';
import symbols from './symbols.js';
import { initDiagnosticsChannel } from './diagnosticsChannel.js';
const PROTO_RE = /^https?:\/\//i;
const FormData = FormDataNative !== null && FormDataNative !== void 0 ? FormDataNative : FormDataNode;
// impl isReadable on Node.js 14
const isReadable = (_a = stream.isReadable) !== null && _a !== void 0 ? _a : function isReadable(stream) {
return stream && typeof stream.read === 'function';
};
// impl promise pipeline on Node.js 14
const pipelinePromise = (_c = (_b = stream.promises) === null || _b === void 0 ? void 0 : _b.pipeline) !== null && _c !== void 0 ? _c : function pipeline(...args) {
return new Promise((resolve, reject) => {
pump(...args, (err) => {
if (err)
return reject(err);
resolve();
});
});
};
function noop() {
// noop
}
const debug = debuglog('urllib:HttpClient');
// https://github.com/octet-stream/form-data
class BlobFromStream {
constructor(stream, type) {
_BlobFromStream_stream.set(this, void 0);
_BlobFromStream_type.set(this, void 0);
__classPrivateFieldSet(this, _BlobFromStream_stream, stream, "f");
__classPrivateFieldSet(this, _BlobFromStream_type, type, "f");
}
stream() {
return __classPrivateFieldGet(this, _BlobFromStream_stream, "f");
}
get type() {
return __classPrivateFieldGet(this, _BlobFromStream_type, "f");
}
get [(_BlobFromStream_stream = new WeakMap(), _BlobFromStream_type = new WeakMap(), Symbol.toStringTag)]() {
return 'Blob';
}
}
class HttpClientRequestTimeoutError extends Error {
constructor(timeout, options) {
const message = `Request timeout for ${timeout} ms`;
super(message, options);
this.name = this.constructor.name;
Error.captureStackTrace(this, this.constructor);
}
}
export const HEADER_USER_AGENT = createUserAgent('node-urllib', '3.0.0');
function getFileName(stream) {
const filePath = stream.path;
if (filePath) {
return basename(filePath);
}
return '';
}
function defaultIsRetry(response) {
return response.status >= 500;
}
export class HttpClient extends EventEmitter {
constructor(clientOptions) {
super();
_HttpClient_instances.add(this);
_HttpClient_defaultArgs.set(this, void 0);
_HttpClient_dispatcher.set(this, void 0);
__classPrivateFieldSet(this, _HttpClient_defaultArgs, clientOptions === null || clientOptions === void 0 ? void 0 : clientOptions.defaultArgs, "f");
if ((clientOptions === null || clientOptions === void 0 ? void 0 : clientOptions.lookup) || (clientOptions === null || clientOptions === void 0 ? void 0 : clientOptions.checkAddress) || (clientOptions === null || clientOptions === void 0 ? void 0 : clientOptions.connect)) {
__classPrivateFieldSet(this, _HttpClient_dispatcher, new HttpAgent({
lookup: clientOptions.lookup,
checkAddress: clientOptions.checkAddress,
connect: clientOptions.connect,
}), "f");
}
initDiagnosticsChannel();
}
async request(url, options) {
return await __classPrivateFieldGet(this, _HttpClient_instances, "m", _HttpClient_requestInternal).call(this, url, options);
}
}
_HttpClient_defaultArgs = new WeakMap(), _HttpClient_dispatcher = new WeakMap(), _HttpClient_instances = new WeakSet(), _HttpClient_requestInternal = async function _HttpClient_requestInternal(url, options, requestContext) {
var _a, _b, _c, _d, _e, _f, _g;
const requestId = globalId('HttpClientRequest');
let requestUrl;
if (typeof url === 'string') {
if (!PROTO_RE.test(url)) {
// Support `request('www.server.com')`
url = 'http://' + url;
}
requestUrl = new URL(url);
}
else {
if (!url.searchParams) {
// url maybe url.parse(url) object in urllib2
requestUrl = new URL(urlFormat(url));
}
else {
requestUrl = url;
}
}
const method = ((_a = options === null || options === void 0 ? void 0 : options.method) !== null && _a !== void 0 ? _a : 'GET').toUpperCase();
const orginalHeaders = options === null || options === void 0 ? void 0 : options.headers;
const headers = {};
const args = {
retry: 0,
...__classPrivateFieldGet(this, _HttpClient_defaultArgs, "f"),
...options,
// keep method and headers exists on args for request event handler to easy use
method,
headers,
};
requestContext = {
retries: 0,
...requestContext,
};
const requestStartTime = performance.now();
// https://developer.chrome.com/docs/devtools/network/reference/?utm_source=devtools#timing-explanation
const timing = {
// socket assigned
queuing: 0,
// dns lookup time
// dnslookup: 0,
// socket connected
connected: 0,
// request headers sent
requestHeadersSent: 0,
// request sent, including headers and body
requestSent: 0,
// Time to first byte (TTFB), the response headers have been received
waiting: 0,
// the response body and trailers have been received
contentDownload: 0,
};
const orginalOpaque = args.opaque;
// using opaque to diagnostics channel, binding request and socket
const internalOpaque = {
[symbols.kRequestId]: requestId,
[symbols.kRequestStartTime]: requestStartTime,
[symbols.kEnableRequestTiming]: !!args.timing,
[symbols.kRequestTiming]: timing,
[symbols.kRequestOrginalOpaque]: orginalOpaque,
};
const reqMeta = {
requestId,
url: requestUrl.href,
args,
ctx: args.ctx,
retries: requestContext.retries,
};
const socketInfo = {
id: 0,
localAddress: '',
localPort: 0,
remoteAddress: '',
remotePort: 0,
remoteFamily: '',
bytesWritten: 0,
bytesRead: 0,
handledRequests: 0,
handledResponses: 0,
};
// keep urllib createCallbackResponse style
const resHeaders = {};
const res = {
status: -1,
statusCode: -1,
headers: resHeaders,
size: 0,
aborted: false,
rt: 0,
keepAliveSocket: true,
requestUrls: [],
timing,
socket: socketInfo,
};
let headersTimeout = 5000;
let bodyTimeout = 5000;
if (args.timeout) {
if (Array.isArray(args.timeout)) {
headersTimeout = (_b = args.timeout[0]) !== null && _b !== void 0 ? _b : headersTimeout;
bodyTimeout = (_c = args.timeout[1]) !== null && _c !== void 0 ? _c : bodyTimeout;
}
else {
headersTimeout = bodyTimeout = args.timeout;
}
}
if (orginalHeaders) {
// convert headers to lower-case
for (const name in orginalHeaders) {
headers[name.toLowerCase()] = orginalHeaders[name];
}
}
// hidden user-agent
const hiddenUserAgent = 'user-agent' in headers && !headers['user-agent'];
if (hiddenUserAgent) {
delete headers['user-agent'];
}
else if (!headers['user-agent']) {
// need to set user-agent
headers['user-agent'] = HEADER_USER_AGENT;
}
// Alias to dataType = 'stream'
if (args.streaming || args.customResponse) {
args.dataType = 'stream';
}
if (args.dataType === 'json' && !headers.accept) {
headers.accept = 'application/json';
}
// gzip alias to compressed
if (args.gzip && args.compressed !== false) {
args.compressed = true;
}
if (args.compressed && !headers['accept-encoding']) {
headers['accept-encoding'] = 'gzip, br';
}
if (requestContext.retries > 0) {
headers['x-urllib-retry'] = `${requestContext.retries}/${args.retry}`;
}
if (args.auth && !headers.authorization) {
headers.authorization = `Basic ${Buffer.from(args.auth).toString('base64')}`;
}
try {
const requestOptions = {
method,
keepalive: true,
maxRedirections: (_d = args.maxRedirects) !== null && _d !== void 0 ? _d : 10,
headersTimeout,
bodyTimeout,
opaque: internalOpaque,
dispatcher: (_e = args.dispatcher) !== null && _e !== void 0 ? _e : __classPrivateFieldGet(this, _HttpClient_dispatcher, "f"),
};
if (args.followRedirect === false) {
requestOptions.maxRedirections = 0;
}
const isGETOrHEAD = requestOptions.method === 'GET' || requestOptions.method === 'HEAD';
// alias to args.content
if (args.stream && !args.content) {
args.content = args.stream;
}
if (args.files) {
if (isGETOrHEAD) {
requestOptions.method = 'POST';
}
const formData = new FormData();
const uploadFiles = [];
if (Array.isArray(args.files)) {
for (const [index, file] of args.files.entries()) {
const field = index === 0 ? 'file' : `file${index}`;
uploadFiles.push([field, file]);
}
}
else if (args.files instanceof Readable || isReadable(args.files)) {
uploadFiles.push(['file', args.files]);
}
else if (typeof args.files === 'string' || Buffer.isBuffer(args.files)) {
uploadFiles.push(['file', args.files]);
}
else if (typeof args.files === 'object') {
for (const field in args.files) {
uploadFiles.push([field, args.files[field]]);
}
}
// set normal fields first
if (args.data) {
for (const field in args.data) {
formData.append(field, args.data[field]);
}
}
for (const [index, [field, file]] of uploadFiles.entries()) {
if (typeof file === 'string') {
// FIXME: support non-ascii filename
// const fileName = encodeURIComponent(basename(file));
// formData.append(field, await fileFromPath(file, `utf-8''${fileName}`, { type: mime.lookup(fileName) || '' }));
const fileName = basename(file);
const fileReadable = createReadStream(file);
formData.append(field, new BlobFromStream(fileReadable, mime.lookup(fileName) || ''), fileName);
}
else if (Buffer.isBuffer(file)) {
formData.append(field, new Blob([file]), `bufferfile${index}`);
}
else if (file instanceof Readable || isReadable(file)) {
const fileName = getFileName(file) || `streamfile${index}`;
formData.append(field, new BlobFromStream(file, mime.lookup(fileName) || ''), fileName);
}
}
if (FormDataNative) {
requestOptions.body = formData;
}
else {
// Node.js 14 does not support spec-compliant FormData
// https://github.com/octet-stream/form-data#usage
const encoder = new FormDataEncoder(formData);
Object.assign(headers, encoder.headers);
// fix "Content-Length":"NaN"
delete headers['Content-Length'];
requestOptions.body = Readable.from(encoder);
}
}
else if (args.content) {
if (!isGETOrHEAD) {
// handle content
requestOptions.body = args.content;
if (args.contentType) {
headers['content-type'] = args.contentType;
}
else if (typeof args.content === 'string' && !headers['content-type']) {
headers['content-type'] = 'text/plain;charset=UTF-8';
}
}
}
else if (args.data) {
const isStringOrBufferOrReadable = typeof args.data === 'string'
|| Buffer.isBuffer(args.data)
|| isReadable(args.data);
if (isGETOrHEAD) {
if (!isStringOrBufferOrReadable) {
for (const field in args.data) {
requestUrl.searchParams.append(field, args.data[field]);
}
}
}
else {
if (isStringOrBufferOrReadable) {
requestOptions.body = args.data;
}
else {
if (args.contentType === 'json'
|| args.contentType === 'application/json'
|| ((_f = headers['content-type']) === null || _f === void 0 ? void 0 : _f.startsWith('application/json'))) {
requestOptions.body = JSON.stringify(args.data);
if (!headers['content-type']) {
headers['content-type'] = 'application/json';
}
}
else {
headers['content-type'] = 'application/x-www-form-urlencoded;charset=UTF-8';
requestOptions.body = new URLSearchParams(args.data).toString();
}
}
}
}
debug('Request#%d %s %s, headers: %j, headersTimeout: %s, bodyTimeout: %s', requestId, requestOptions.method, requestUrl.href, headers, headersTimeout, bodyTimeout);
requestOptions.headers = headers;
if (this.listenerCount('request') > 0) {
this.emit('request', reqMeta);
}
let response = await undiciRequest(requestUrl, requestOptions);
// handle digest auth
if (response.statusCode === 401 && response.headers['www-authenticate'] &&
!requestOptions.headers.authorization && args.digestAuth) {
const authenticate = response.headers['www-authenticate'];
if (authenticate.startsWith('Digest ')) {
debug('Request#%d %s: got digest auth header WWW-Authenticate: %s', requestId, requestUrl.href, authenticate);
requestOptions.headers.authorization = digestAuthHeader(requestOptions.method, `${requestUrl.pathname}${requestUrl.search}`, authenticate, args.digestAuth);
debug('Request#%d %s: auth with digest header: %s', requestId, url, requestOptions.headers.authorization);
if (response.headers['set-cookie']) {
// FIXME: merge exists cookie header
requestOptions.headers.cookie = response.headers['set-cookie'].join(';');
}
response = await undiciRequest(requestUrl, requestOptions);
}
}
const context = response.context;
let lastUrl = '';
if (context === null || context === void 0 ? void 0 : context.history) {
for (const urlObject of context === null || context === void 0 ? void 0 : context.history) {
res.requestUrls.push(urlObject.href);
lastUrl = urlObject.href;
}
}
else {
res.requestUrls.push(requestUrl.href);
lastUrl = requestUrl.href;
}
const contentEncoding = response.headers['content-encoding'];
const isCompressedContent = contentEncoding === 'gzip' || contentEncoding === 'br';
res.headers = response.headers;
res.status = res.statusCode = response.statusCode;
if (res.headers['content-length']) {
res.size = parseInt(res.headers['content-length']);
}
let data = null;
let responseBodyStream;
if (args.dataType === 'stream') {
// streaming mode will disable retry
args.retry = 0;
const meta = {
status: res.status,
statusCode: res.statusCode,
headers: res.headers,
timing,
socket: socketInfo,
};
if (isCompressedContent) {
// gzip or br
const decoder = contentEncoding === 'gzip' ? createGunzip() : createBrotliDecompress();
responseBodyStream = Object.assign(pipeline(response.body, decoder, noop), meta);
}
else {
responseBodyStream = Object.assign(response.body, meta);
}
}
else if (args.writeStream) {
// streaming mode will disable retry
args.retry = 0;
if (isCompressedContent) {
const decoder = contentEncoding === 'gzip' ? createGunzip() : createBrotliDecompress();
await pipelinePromise(response.body, decoder, args.writeStream);
}
else {
await pipelinePromise(response.body, args.writeStream);
}
}
else {
// buffer
data = Buffer.from(await response.body.arrayBuffer());
if (isCompressedContent && data.length > 0) {
try {
data = contentEncoding === 'gzip' ? gunzipSync(data) : brotliDecompressSync(data);
}
catch (err) {
if (err.name === 'Error') {
err.name = 'UnzipError';
}
throw err;
}
}
if (args.dataType === 'text') {
data = data.toString();
}
else if (args.dataType === 'json') {
if (data.length === 0) {
data = null;
}
else {
data = parseJSON(data.toString(), args.fixJSONCtlChars);
}
}
}
res.rt = performanceTime(requestStartTime);
// get real socket info from internalOpaque
__classPrivateFieldGet(this, _HttpClient_instances, "m", _HttpClient_updateSocketInfo).call(this, socketInfo, internalOpaque);
const clientResponse = {
opaque: orginalOpaque,
data,
status: res.status,
statusCode: res.status,
headers: res.headers,
url: lastUrl,
redirected: res.requestUrls.length > 1,
requestUrls: res.requestUrls,
res: responseBodyStream !== null && responseBodyStream !== void 0 ? responseBodyStream : res,
};
if (args.retry > 0 && requestContext.retries < args.retry) {
const isRetry = (_g = args.isRetry) !== null && _g !== void 0 ? _g : defaultIsRetry;
if (isRetry(clientResponse)) {
if (args.retryDelay) {
await sleep(args.retryDelay);
}
requestContext.retries++;
return await __classPrivateFieldGet(this, _HttpClient_instances, "m", _HttpClient_requestInternal).call(this, url, options, requestContext);
}
}
if (this.listenerCount('response') > 0) {
this.emit('response', {
requestId,
error: null,
ctx: args.ctx,
req: {
...reqMeta,
options: args,
},
res,
});
}
return clientResponse;
}
catch (e) {
debug('Request#%d throw error: %s', requestId, e);
let err = e;
if (err.name === 'HeadersTimeoutError') {
err = new HttpClientRequestTimeoutError(headersTimeout, { cause: e });
}
else if (err.name === 'BodyTimeoutError') {
err = new HttpClientRequestTimeoutError(bodyTimeout, { cause: e });
}
err.opaque = orginalOpaque;
err.status = res.status;
err.headers = res.headers;
err.res = res;
// make sure requestUrls not empty
if (res.requestUrls.length === 0) {
res.requestUrls.push(requestUrl.href);
}
res.rt = performanceTime(requestStartTime);
__classPrivateFieldGet(this, _HttpClient_instances, "m", _HttpClient_updateSocketInfo).call(this, socketInfo, internalOpaque);
if (this.listenerCount('response') > 0) {
this.emit('response', {
requestId,
error: err,
ctx: args.ctx,
req: {
...reqMeta,
options: args,
},
res,
});
}
throw err;
}
}, _HttpClient_updateSocketInfo = function _HttpClient_updateSocketInfo(socketInfo, internalOpaque) {
const socket = internalOpaque[symbols.kRequestSocket];
if (socket) {
socketInfo.id = socket[symbols.kSocketId];
socketInfo.handledRequests = socket[symbols.kHandledRequests];
socketInfo.handledResponses = socket[symbols.kHandledResponses];
socketInfo.localAddress = socket.localAddress;
socketInfo.localPort = socket.localPort;
socketInfo.remoteAddress = socket.remoteAddress;
socketInfo.remotePort = socket.remotePort;
socketInfo.remoteFamily = socket.remoteFamily;
socketInfo.bytesRead = socket.bytesRead;
socketInfo.bytesWritten = socket.bytesWritten;
}
};
//# sourceMappingURL=HttpClient.js.map