website-scrap-engine/lib/downloader/multi.d.ts

Configurable website scraper in typescript

import type { WorkerFactory } from './worker-pool.js';
import { WorkerPool } from './worker-pool.js';
import type { RawResource, Resource } from '../resource.js';
import type { DownloadWorkerMessage } from './types.js';
import type { DownloadOptions, StaticDownloadOptions } from '../options.js';
import { AbstractDownloader } from './main.js';
export interface MultiThreadDownloaderOptions extends StaticDownloadOptions {
    pathToWorker?: string;
    maxLoad: number;
}
export declare class MultiThreadDownloader extends AbstractDownloader {
    pathToOptions: string;
    private _workerFactory?;
    private _pool;
    readonly init: Promise<void>;
    workerDispose: Promise<void>[];
    constructor(pathToOptions: string, overrideOptions?: Partial<MultiThreadDownloaderOptions>, _workerFactory?: WorkerFactory | undefined);
    protected _internalInit(options: DownloadOptions): Promise<void>;
    get pool(): WorkerPool<RawResource, DownloadWorkerMessage>;
    downloadAndProcessResource(res: Resource): Promise<boolean | void>;
    dispose(): Promise<void>;
}
//# sourceMappingURL=multi.d.ts.map