contract-scraper/dist/src/fetcher/puppeteer.js

A customisable data scraper for the web based on JSON contracts

"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
    return new (P || (P = Promise))(function (resolve, reject) {
        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
        step((generator = generator.apply(thisArg, _arguments || [])).next());
    });
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const random_useragent_1 = __importDefault(require("random-useragent"));
const puppeteer_1 = __importDefault(require("puppeteer"));
const encoding_1 = require("../tools/encoding");
class PuppeteerFetcher {
    constructor(url, waitForPageLoadSelector, options) {
        this.url = url;
        this.waitForPageLoadSelector = waitForPageLoadSelector;
        this.options = options || { headless: true };
    }
    getBrowserType() {
        return puppeteer_1.default;
    }
    setupBrowser() {
        return __awaiter(this, void 0, void 0, function* () {
            const browserType = this.getBrowserType();
            const userAgent = random_useragent_1.default.getRandom();
            const browser = yield browserType.launch({
                headless: this.options.headless,
                args: [
                    '--no-sandbox',
                    '--disable-setuid-sandbox',
                    '--allow-insecure',
                    '--disable-web-security',
                ],
                ignoreHTTPSErrors: true,
                slowMo: 10,
            });
            const page = yield browser.newPage();
            try {
                yield page.setUserAgent(userAgent);
                yield page.setExtraHTTPHeaders({
                    'Content-Type': 'text/html; charset=utf-8',
                    Accept: 'text/html',
                    'Accept-Language': 'fr-FR',
                });
                yield page.setViewport({ width: 1280, height: 1000 });
                const response = yield page.goto(this.url);
                if (this.waitForPageLoadSelector) {
                    yield page.waitForSelector(this.waitForPageLoadSelector);
                }
                const contents = yield page.content();
                yield page.close();
                yield browser.close();
                return { response, contents };
            }
            catch (e) {
                yield page.close();
                yield browser.close();
                console.log(`Error fetching ${this.url} with puppeteer`);
                throw e;
            }
        });
    }
    getPage() {
        return __awaiter(this, void 0, void 0, function* () {
            const { response, contents } = yield this.setupBrowser();
            const contentType = (0, encoding_1.getContentTypeHeaders)(yield response.headers());
            return {
                contents,
                encoding: (0, encoding_1.guessEncoding)(contentType, contents),
                url: this.url,
            };
        });
    }
}
exports.default = PuppeteerFetcher;