@xcrap/core/dist/base-client.js

Xcrap Core is the core package of the Xcrap framework for web scraping, offering tools such as HttpClient, BaseClient, Randomizer, Rotator, and support for proxies and pagination.

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.BaseClient = void 0;
const constants_1 = require("./constants");
const delay_1 = require("./utils/delay");
class BaseClient {
    constructor({ proxy, userAgent, proxyUrl }) {
        this.proxy = proxy;
        this.userAgent = userAgent !== null && userAgent !== void 0 ? userAgent : constants_1.defaultUserAgent;
        this.proxyUrl = proxyUrl;
    }
    get currentProxyUrl() {
        const currentProxyUrl = typeof this.proxyUrl === "function" ?
            this.proxyUrl() :
            this.proxyUrl;
        return currentProxyUrl;
    }
    get currentUserAgent() {
        const currentUserAgent = typeof this.userAgent === "function" ?
            this.userAgent() :
            this.userAgent;
        return currentUserAgent;
    }
    get currentProxy() {
        const currentProxy = typeof this.proxy === "function" ?
            this.proxy() :
            this.proxy;
        return currentProxy;
    }
    shouldThrottle(executing, concurrency) {
        return concurrency !== undefined && executing.length >= concurrency;
    }
    cleanCompletedPromises(executing) {
        for (let i = executing.length - 1; i >= 0; i--) {
            executing[i].then(() => executing.splice(i, 1)).catch(() => executing.splice(i, 1));
        }
    }
    async handleConcurrency(executing) {
        await Promise.race(executing);
        this.cleanCompletedPromises(executing);
    }
    async executeRequest({ index, request, results, requestDelay }) {
        if (requestDelay !== undefined && requestDelay > 0 && index > 0) {
            await (0, delay_1.delay)(requestDelay);
        }
        if ("fetch" in this && typeof this.fetch === "function") {
            results[index] = await this.fetch(request);
        }
    }
    isSuccess(statusCode) {
        return statusCode >= 200 && statusCode < 300;
    }
}
exports.BaseClient = BaseClient;