website-scrap-engine/lib/downloader/types.d.ts

Configurable website scraper in typescript

import type { MessagePort } from 'node:worker_threads';
import type { DownloadOptions } from '../options.js';
import type { RawResource } from '../resource.js';
export interface DownloaderStats {
    firstPeriodCount: number;
    lastPeriodTotalCount: number;
    currentPeriodCount: number;
    lastPeriodCount: number;
}
export interface DownloaderWithMeta {
    readonly meta: DownloaderStats;
    readonly options: DownloadOptions;
    /**
     * Concurrency of the queue.
     */
    concurrency: number;
    /**
     * Size of the queue.
     */
    readonly queueSize: number;
    /**
     * Number of pending promises.
     */
    readonly queuePending: number;
    /**
     * Number of downloaded resource.
     */
    readonly downloadedCount: number;
}
export interface PendingPromise<T = unknown, E = unknown> {
    resolve: ((value?: T | PromiseLike<T>) => void) | ((value: T | PromiseLike<T>) => void);
    reject: (reason?: E) => void;
}
export interface PendingPromiseWithBody<R = unknown, E = unknown, B = unknown> extends PendingPromise<R, E> {
    taskId: number;
    body: B;
    transferList?: Array<ArrayBuffer | MessagePort>;
}
export declare enum WorkerMessageType {
    Log = 0,
    Complete = 1
}
export interface WorkerMessage<T = unknown> {
    taskId: number;
    type: WorkerMessageType;
    body: T;
    error?: Error | unknown | void;
}
export interface DownloadWorkerMessage extends WorkerMessage<RawResource[]> {
    /**
     * Available if processed redirect url differs from url
     */
    redirectedUrl?: string;
}
//# sourceMappingURL=types.d.ts.map