@rr0/cms/dist/time/datasource/HttpSource.js

RR0 Content Management System (CMS)

import { Browser, Builder, By } from "selenium-webdriver";
import { JSDOM } from "jsdom";
import { FetchHttpFetcher } from "./FetchHttpFetcher.js";
import { ArchiveHttpFetcher } from "./ArchiveHttpFetcher.js";
import assert from "assert";
export class MimeType {
}
MimeType.csv = "text/csv";
MimeType.json = "application/json";
MimeType.txt = "text/plain";
MimeType.xls = "application/vnd.ms-excel";
MimeType.pdf = "application/pdf";
export class HttpSourceError extends Error {
    constructor(url, status, statusText) {
        super(`Could not fetch ${url}: ${`${statusText}(${status})` || `HTTP error ${status}`}`);
        this.status = status;
        this.statusText = statusText;
    }
}
/**
 * A source for cases that can be fetched online.
 */
export class HttpSource {
    constructor(options = { fetchers: [new FetchHttpFetcher(), new ArchiveHttpFetcher()] }) {
        this.options = options;
    }
    // TODO: Move selenium web driver in SeleniumHttpFetcher
    async getDriver() {
        if (!this.driver) {
            const seleniumOptions = this.options.selenium;
            assert.ok(seleniumOptions, "selenium options expected in " + JSON.stringify(this.options));
            this.driver = await new Builder().forBrowser(Browser.CHROME).build();
            await this.driver.manage().setTimeouts(seleniumOptions.timeout);
        }
        return this.driver;
    }
    async get(queryUrl, init = {}, resOut = {}) {
        let pageSource;
        const seleniumOptions = this.options.selenium;
        // TODO: Move selenium web driver in SeleniumHttpFetcher
        if (seleniumOptions) {
            const driver = await this.getDriver();
            await driver.get(queryUrl.href);
            const resultSelector = seleniumOptions.selector || "html";
            const selector = By.css(resultSelector);
            await driver.findElements(selector);
            pageSource = await driver.getPageSource();
        }
        else {
            pageSource = await this.fetch(queryUrl, init, resOut);
        }
        return new JSDOM(pageSource).window.document.documentElement;
    }
    async fetch(url, init = {}, resOut = {}) {
        let previousError;
        for (const fetcher of this.options.fetchers) {
            try {
                return await fetcher.fetch(url, init, resOut, previousError);
            }
            catch (e) {
                console.warn(e.toString());
                previousError = e;
            }
        }
        throw previousError;
    }
    async submitForm(url, obj, headers = {}) {
        const formData = new FormData();
        Object.entries(obj).forEach(entry => formData.append(entry[0], encodeURIComponent(entry[1])));
        const init = { method: "POST", headers, body: formData };
        return await this.fetch(url, init);
    }
    async close() {
        if (this.driver) {
            await this.driver.quit();
            this.driver = undefined;
        }
    }
}