got-scraping
Version:
HTTP client made for scraping based on got.
61 lines • 2.27 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.gotScraping = void 0;
const tslib_1 = require("tslib");
const http_1 = tslib_1.__importDefault(require("http"));
const https_1 = tslib_1.__importDefault(require("https"));
const got_cjs_1 = require("got-cjs");
// @ts-expect-error Missing types
const header_generator_1 = tslib_1.__importDefault(require("header-generator"));
const transform_headers_agent_1 = require("./agent/transform-headers-agent");
const options_validation_1 = require("./hooks/options-validation");
const custom_options_1 = require("./hooks/custom-options");
const browser_headers_1 = require("./hooks/browser-headers");
const proxy_1 = require("./hooks/proxy");
const http2_1 = require("./hooks/http2");
const insecure_parser_1 = require("./hooks/insecure-parser");
const tls_1 = require("./hooks/tls");
const storage_1 = require("./hooks/storage");
const gotScraping = got_cjs_1.got.extend({
mutableDefaults: true,
// Most of the new browsers use HTTP/2
http2: true,
https: {
// In contrast to browsers, we don't usually do login operations.
// We want the content.
rejectUnauthorized: false,
},
// Don't fail on 404
throwHttpErrors: false,
timeout: { request: 60000 },
retry: { limit: 0 },
headers: {
'user-agent': undefined,
},
context: {
headerGenerator: new header_generator_1.default(),
useHeaderGenerator: true,
insecureHTTPParser: true,
},
agent: {
http: new transform_headers_agent_1.TransformHeadersAgent(http_1.default.globalAgent),
https: new transform_headers_agent_1.TransformHeadersAgent(https_1.default.globalAgent),
},
hooks: {
init: [
options_validation_1.optionsValidationHandler,
custom_options_1.customOptionsHook,
],
beforeRequest: [
insecure_parser_1.insecureParserHook,
storage_1.sessionDataHook,
http2_1.http2Hook,
proxy_1.proxyHook,
browser_headers_1.browserHeadersHook,
tls_1.tlsHook,
],
},
});
exports.gotScraping = gotScraping;
tslib_1.__exportStar(require("got-cjs"), exports);
//# sourceMappingURL=index.js.map