@xcrap/core/dist/base-client.d.ts

Xcrap Core is the core package of the Xcrap framework for web scraping, offering tools such as HttpClient, BaseClient, Randomizer, Rotator, and support for proxies and pagination.

import { HttpResponse } from "./http-response";
export type ProxyUrlFuction = () => string;
export type ProxyFunction<ProxyReturn = any> = () => ProxyReturn;
export type UserAgentFunction = () => string;
export type BaseClientOptions<Proxy> = {
    proxyUrl?: string | ProxyUrlFuction;
    proxy?: Proxy | ProxyFunction<Proxy>;
    userAgent?: string | UserAgentFunction;
};
export type ExecuteRequestOptions = {
    request: any;
    index: number;
    requestDelay?: number;
    results: HttpResponse[];
};
export declare class BaseClient<Proxy> {
    readonly proxy?: Proxy | ProxyFunction<Proxy>;
    readonly userAgent?: string | UserAgentFunction;
    readonly proxyUrl?: string | ProxyUrlFuction;
    constructor({ proxy, userAgent, proxyUrl }: BaseClientOptions<Proxy>);
    protected get currentProxyUrl(): string | undefined;
    protected get currentUserAgent(): string | undefined;
    protected get currentProxy(): string | ReturnType<ProxyFunction<Proxy>> | undefined;
    protected shouldThrottle(executing: Promise<void>[], concurrency?: number): boolean;
    protected cleanCompletedPromises(executing: Promise<void>[]): void;
    protected handleConcurrency(executing: Promise<void>[]): Promise<void>;
    protected executeRequest({ index, request, results, requestDelay }: ExecuteRequestOptions): Promise<void>;
    protected isSuccess(statusCode: number): boolean;
}