contract-scraper/dist/src/fetcher/request.js

A customisable data scraper for the web based on JSON contracts

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const axios_1 = __importDefault(require("axios"));
const random_useragent_1 = __importDefault(require("random-useragent"));
const encoding_1 = require("../tools/encoding");
class RequestFetcher {
    constructor(url) {
        this.url = url;
    }
    getPage() {
        return this.getPageResponse().then((response) => {
            const { headers, data: body } = response;
            const encoding = (0, encoding_1.guessEncoding)(headers['Content-Type'], body);
            const page = {
                encoding,
                contents: (0, encoding_1.encodePageContents)(encoding, body),
                url: this.url,
            };
            return page;
        });
    }
    getRequestLibrary() {
        return axios_1.default;
    }
    getPageResponse() {
        const userAgent = random_useragent_1.default.getRandom();
        return this.getRequestLibrary().get(this.url, {
            headers: {
                'Content-Type': 'text/html; charset=utf-8',
                Accept: 'text/html',
                'Accept-Language': 'fr-FR',
                'User-Agent': userAgent,
            },
        });
    }
}
exports.default = RequestFetcher;