@aliceplex/core/lib/scrap/utils.js

[![License][license_badge]][license] [![Pipelines][pipelines_badge]][pipelines] [![Coverage][coverage_badge]][pipelines] [![NPM][npm_badge]][npm] [![semantic-release][semantic_release_badge]][semantic_release]

"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
    return new (P || (P = Promise))(function (resolve, reject) {
        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
        step((generator = generator.apply(thisArg, _arguments || [])).next());
    });
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const cross_fetch_1 = __importDefault(require("cross-fetch"));
const cheerio_1 = __importDefault(require("cheerio"));
const lodash_1 = __importDefault(require("lodash"));
const addUserAgent = (request) => lodash_1.default.merge({
    headers: {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36"
    }
}, request);
const baseFetch = (url, request) => __awaiter(void 0, void 0, void 0, function* () {
    const res = yield cross_fetch_1.default(url, addUserAgent(request));
    if (!res.ok) {
        throw new Error(`Error response code ${res.status}`);
    }
    return res;
});
exports.fetchText = (url, request = {}) => __awaiter(void 0, void 0, void 0, function* () {
    const response = yield baseFetch(url, request);
    return response.text();
});
exports.fetchTextMemorized = lodash_1.default.memoize(exports.fetchText);
exports.fetchDom = (url, request = {}, cheerio) => __awaiter(void 0, void 0, void 0, function* () {
    const html = yield exports.fetchText(url, request);
    return cheerio_1.default.load(html, cheerio);
});
exports.fetchDomMemorized = lodash_1.default.memoize(exports.fetchDom);
exports.fetchJson = (url, request = {}) => __awaiter(void 0, void 0, void 0, function* () {
    const response = yield baseFetch(url, request);
    return response.json();
});
exports.fetchJsonMemorized = lodash_1.default.memoize(exports.fetchJson);
//# sourceMappingURL=utils.js.map