@wabarc/cairn/dist/utils/http.js

Node package and CLI tool for saving web page as single HTML file

"use strict";
/*
 * Copyright 2023 Wayback Archiver. All rights reserved.
 * Use of this source code is governed by the MIT
 * license that can be found in the LICENSE file.
 */
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
    return new (P || (P = Promise))(function (resolve, reject) {
        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
        step((generator = generator.apply(thisArg, _arguments || [])).next());
    });
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.http = void 0;
const axios_1 = __importDefault(require("axios"));
const socks_proxy_agent_1 = require("socks-proxy-agent");
const http_proxy_agent_1 = require("http-proxy-agent");
const https_proxy_agent_1 = require("https-proxy-agent");
const _1 = require(".");
const chardet_1 = __importDefault(require("chardet"));
const iconv_lite_1 = __importDefault(require("iconv-lite"));
class HTTP {
    constructor() {
        this.timeout = 60;
        this.responseType = 'arraybuffer';
        // Proxy environment not working for axios, reset it to avoid getting caught.
        process.env.https_proxy = '';
        process.env.http_proxy = '';
        process.env.all_proxy = '';
        process.env.HTTPS_PROXY = '';
        process.env.HTTP_PROXY = '';
        process.env.ALL_PROXY = '';
        const ua = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36';
        if (!global.axios) {
            global.axios = axios_1.default.create({
                timeout: 1000 * this.timeout,
                headers: { 'User-Agent': ua },
                responseType: this.responseType,
            });
            global.axios.interceptors.response.use(function (response) {
                if (response.data === undefined) {
                    return response;
                }
                let charset = response.headers['content-type'] || '';
                if (!charset || typeof charset !== 'string' || charset.includes('charset') === false) {
                    charset = chardet_1.default.detect(Buffer.from(response.data));
                }
                // refer: https://github.com/ashtuchkin/iconv-lite/wiki/Supported-Encodings
                const types = { big5: 'big5', gb2312: 'gb2312', gbk: 'gbk', gb18030: 'gb18030' };
                for (const [type, encoding] of Object.entries(types)) {
                    response.data = charset.toLowerCase().includes(type) ? iconv_lite_1.default.decode(response.data, encoding) : response.data;
                }
                return response;
            });
        }
    }
    setHeader(key, val) {
        if (key.length < 1) {
            return this;
        }
        global.axios.defaults.headers[key] = val;
        return this;
    }
    setResponseType(responseType) {
        this.responseType = responseType;
        return this;
    }
    setOptions(args) {
        var _a, _b, _c;
        if (args.timeout)
            global.axios.defaults.timeout = 1000 * args.timeout;
        if (args.proxy) {
            const proxy = new URL(args.proxy);
            let agent;
            if (((_a = proxy.protocol) === null || _a === void 0 ? void 0 : _a.indexOf(`socks`)) == 0) {
                agent = new socks_proxy_agent_1.SocksProxyAgent(proxy.toString());
            }
            else if (((_b = proxy.protocol) === null || _b === void 0 ? void 0 : _b.indexOf(`https`)) == 0) {
                agent = new https_proxy_agent_1.HttpsProxyAgent(proxy.toString());
            }
            else if (((_c = proxy.protocol) === null || _c === void 0 ? void 0 : _c.indexOf(`http`)) == 0) {
                agent = new http_proxy_agent_1.HttpProxyAgent(proxy.toString());
            }
            global.axios.defaults.httpsAgent = agent;
            global.axios.defaults.httpAgent = agent;
        }
        return this;
    }
    fetch(url) {
        return __awaiter(this, void 0, void 0, function* () {
            if (url.startsWith('data:') || url.startsWith('about:') || !(0, _1.isValidURL)(url)) {
                return;
            }
            // response keys: status, statusText, headers, config, request, data
            return yield global.axios.get(url).catch((err) => {
                if (err.response) {
                    console.warn(`Cairn: fetch resource failed, [status: ${err.status || 0}, message: ${err.message}, url: ${url}]`);
                }
                else if (err.request) {
                    console.warn(`Cairn: fetch resource failed, [url: ${url}, message: error request.]`);
                }
                else {
                    console.warn(`Cairn: fetch resource failed, [message: ${err.message}, url: ${url}]`);
                }
                return err;
            });
        });
    }
}
const http = new HTTP();
exports.http = http;
//# sourceMappingURL=http.js.map