got-scraping
Version:
HTTP client made for scraping based on got.
78 lines • 2.89 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.browserHeadersHook = exports.mergeHeaders = void 0;
const tslib_1 = require("tslib");
const http2_wrapper_1 = tslib_1.__importDefault(require("http2-wrapper"));
const resolve_protocol_1 = require("../resolve-protocol");
/**
* Merges original generated headers and user provided overrides.
* All header overrides will have the original header case, because of antiscraping.
*/
function mergeHeaders(original, overrides) {
const fixedHeaders = new Map();
for (const entry of Object.entries(original)) {
fixedHeaders.set(entry[0].toLowerCase(), entry);
}
for (const entry of Object.entries(overrides)) {
fixedHeaders.set(entry[0].toLowerCase(), entry);
}
return Object.fromEntries(fixedHeaders.values());
}
exports.mergeHeaders = mergeHeaders;
const getResolveProtocolFunction = (options, proxyUrl, sessionData) => {
const { resolveProtocol } = options;
if (resolveProtocol) {
return resolveProtocol;
}
if (proxyUrl) {
return resolve_protocol_1.createResolveProtocol(proxyUrl, sessionData);
}
return http2_wrapper_1.default.auto.resolveProtocol;
};
async function browserHeadersHook(options) {
const { context } = options;
const { headerGeneratorOptions, useHeaderGenerator, headerGenerator, proxyUrl, } = context;
const sessionData = context.sessionData;
if (!useHeaderGenerator || !headerGenerator)
return;
const createHeadersPair = () => ({
1: headerGenerator.getHeaders({
httpVersion: '1',
...headerGeneratorOptions,
}),
2: headerGenerator.getHeaders({
httpVersion: '2',
...headerGeneratorOptions,
}),
});
const url = options.url;
const resolveProtocol = getResolveProtocolFunction(options, proxyUrl, sessionData);
let alpnProtocol;
if (url.protocol === 'https:') {
alpnProtocol = (await resolveProtocol({
host: url.hostname,
port: url.port || 443,
rejectUnauthorized: false,
ALPNProtocols: ['h2', 'http/1.1'],
servername: url.hostname,
})).alpnProtocol;
}
const httpVersion = alpnProtocol === 'h2' ? '2' : '1';
let generatedHeaders;
if (sessionData) {
if (!sessionData.headers) {
sessionData.headers = createHeadersPair();
}
generatedHeaders = sessionData.headers[httpVersion];
}
else {
generatedHeaders = headerGenerator.getHeaders({
httpVersion,
...headerGeneratorOptions,
});
}
// TODO: Use `options.merge({headers: generatedHeaders})` instead
options.headers = mergeHeaders(generatedHeaders, options.headers);
}
exports.browserHeadersHook = browserHeadersHook;
//# sourceMappingURL=browser-headers.js.map