got-scraping
Version:
HTTP client made for scraping based on got.
89 lines • 3.96 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.defaultAgentCache = exports.proxyHook = void 0;
const tslib_1 = require("tslib");
const url_1 = require("url");
const http2_wrapper_1 = require("http2-wrapper");
const quick_lru_1 = tslib_1.__importDefault(require("quick-lru"));
const h1_proxy_agent_1 = require("../agent/h1-proxy-agent");
const transform_headers_agent_1 = require("../agent/transform-headers-agent");
const { HttpOverHttp2, HttpsOverHttp2, Http2OverHttp2, Http2OverHttps, Http2OverHttp, } = http2_wrapper_1.proxies;
async function proxyHook(options) {
const { context: { proxyUrl } } = options;
if (proxyUrl) {
const parsedProxy = new url_1.URL(proxyUrl);
validateProxyProtocol(parsedProxy.protocol);
options.agent = await getAgents(parsedProxy, options.https.rejectUnauthorized);
}
}
exports.proxyHook = proxyHook;
function validateProxyProtocol(protocol) {
const isSupported = protocol === 'http:' || protocol === 'https:';
if (!isSupported) {
throw new Error(`Proxy URL protocol "${protocol}" is not supported. Please use HTTP or HTTPS.`);
}
}
const createAgentCache = () => new quick_lru_1.default({ maxSize: 1000 });
exports.defaultAgentCache = createAgentCache();
async function getAgents(parsedProxyUrl, rejectUnauthorized, sessionData) {
var _a;
const key = `${rejectUnauthorized}:${parsedProxyUrl.href}`;
if (sessionData && !sessionData.agentCache) {
sessionData.agentCache = createAgentCache();
}
const agentCache = (_a = sessionData === null || sessionData === void 0 ? void 0 : sessionData.agentCache) !== null && _a !== void 0 ? _a : exports.defaultAgentCache;
let agent = agentCache.get(key);
if (agent) {
return agent;
}
const proxy = {
proxyOptions: {
url: parsedProxyUrl,
rejectUnauthorized, // based on the got https.rejectUnauthorized option.
},
};
const proxyUrl = proxy.proxyOptions.url;
if (proxyUrl.protocol === 'https:') {
let alpnProtocol = 'http/1.1';
try {
const result = await http2_wrapper_1.auto.resolveProtocol({
host: parsedProxyUrl.hostname,
port: parsedProxyUrl.port,
rejectUnauthorized,
ALPNProtocols: ['h2', 'http/1.1'],
servername: parsedProxyUrl.hostname,
});
alpnProtocol = result.alpnProtocol;
}
catch {
// Some proxies don't support CONNECT protocol, use http/1.1
}
const proxyIsHttp2 = alpnProtocol === 'h2';
if (proxyIsHttp2) {
agent = {
http: new transform_headers_agent_1.TransformHeadersAgent(new HttpOverHttp2(proxy)),
https: new transform_headers_agent_1.TransformHeadersAgent(new HttpsOverHttp2(proxy)),
http2: new Http2OverHttp2(proxy),
};
}
else {
// Upstream proxies hang up connections on CONNECT + unsecure HTTP
agent = {
http: new transform_headers_agent_1.TransformHeadersAgent(new h1_proxy_agent_1.HttpRegularProxyAgent({ proxy: proxyUrl })),
https: new transform_headers_agent_1.TransformHeadersAgent(new h1_proxy_agent_1.HttpsProxyAgent({ proxy: proxyUrl })),
http2: new Http2OverHttps(proxy),
};
}
}
else {
// Upstream proxies hang up connections on CONNECT + unsecure HTTP
agent = {
http: new transform_headers_agent_1.TransformHeadersAgent(new h1_proxy_agent_1.HttpRegularProxyAgent({ proxy: proxyUrl })),
https: new transform_headers_agent_1.TransformHeadersAgent(new h1_proxy_agent_1.HttpsProxyAgent({ proxy: proxyUrl })),
http2: new Http2OverHttp(proxy),
};
}
agentCache.set(key, agent);
return agent;
}
//# sourceMappingURL=proxy.js.map