@crawlee/core/storages/request_manager_tandem.d.ts

The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.

import type { Dictionary } from '@crawlee/types';
import type { Request, Source } from '../request';
import type { IRequestList } from './request_list';
import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, IRequestManager, RequestQueueOperationInfo, RequestQueueOperationOptions, RequestsLike } from './request_provider';
/**
 * A request manager that combines a RequestList and a RequestQueue.
 * It first reads requests from the RequestList and then, when needed,
 * transfers them in batches to the RequestQueue.
 */
export declare class RequestManagerTandem implements IRequestManager {
    private log;
    private requestList;
    private requestQueue;
    constructor(requestList: IRequestList, requestQueue: IRequestManager);
    /**
     * Transfers a batch of requests from the RequestList to the RequestQueue.
     * Handles both successful transfers and failures appropriately.
     * @private
     */
    private transferNextBatchToQueue;
    /**
     * Fetches the next request from the RequestQueue. If the queue is empty and the RequestList
     * is not finished, it will transfer a batch of requests from the list to the queue first.
     * @inheritdoc
     */
    fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
    /**
     * @inheritdoc
     */
    isFinished(): Promise<boolean>;
    /**
     * @inheritdoc
     */
    isEmpty(): Promise<boolean>;
    /**
     * @inheritdoc
     */
    handledCount(): Promise<number>;
    /**
     * @inheritdoc
     */
    getTotalCount(): number;
    /**
     * @inheritdoc
     */
    getPendingCount(): number;
    /**
     * @inheritdoc
     */
    [Symbol.asyncIterator](): AsyncGenerator<Request<Dictionary>, void, unknown>;
    /**
     * @inheritdoc
     */
    markRequestHandled(request: Request): Promise<RequestQueueOperationInfo | void | null>;
    /**
     * @inheritdoc
     */
    reclaimRequest(request: Request, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
    /**
     * @inheritdoc
     */
    addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
    /**
     * @inheritdoc
     */
    addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
}
//# sourceMappingURL=request_manager_tandem.d.ts.map