website-scrap-engine/lib/downloader/main.d.ts

Configurable website scraper in typescript

import PQueue from 'p-queue';
import type { DownloadOptions, StaticDownloadOptions } from '../options.js';
import type { RawResource } from '../resource.js';
import type { DownloaderStats, DownloaderWithMeta } from './types.js';
import { PipelineExecutorImpl } from './pipeline-executor-impl.js';
export declare abstract class AbstractDownloader implements DownloaderWithMeta {
    pathToOptions: string;
    readonly queue: PQueue;
    readonly _asyncOptions: Promise<DownloadOptions>;
    readonly _overrideOptions?: Partial<StaticDownloadOptions> & {
        pathToWorker?: string;
    };
    _options?: DownloadOptions;
    _isInit: boolean;
    _pipeline?: PipelineExecutorImpl;
    _initOptions: Promise<void>;
    readonly downloadedUrl: Set<string>;
    readonly queuedUrl: Set<string>;
    readonly meta: DownloaderStats;
    adjustTimer: ReturnType<typeof setInterval> | void;
    protected constructor(pathToOptions: string, overrideOptions?: Partial<StaticDownloadOptions> & {
        pathToWorker?: string;
    });
    protected _internalInit(options: DownloadOptions): Promise<void>;
    get options(): DownloadOptions;
    get pipeline(): PipelineExecutorImpl;
    get concurrency(): number;
    set concurrency(newConcurrency: number);
    get queueSize(): number;
    get queuePending(): number;
    addInitialResource(urlArr: string[]): Promise<void>;
    protected _addProcessedResource(res: RawResource): boolean | void;
    abstract downloadAndProcessResource(res: RawResource): Promise<boolean | void>;
    addProcessedResource(res: RawResource): boolean | void;
    handleError(err: Error | unknown | null, cause: string, resource: RawResource): void;
    get downloadedCount(): number;
    start(): void;
    stop(): void;
    onIdle(): Promise<void>;
    dispose(): Promise<void>;
}
//# sourceMappingURL=main.d.ts.map