website-scrap-engine/lib/downloader/pipeline-executor-impl.d.ts

Configurable website scraper in typescript

import type { StaticDownloadOptions } from '../options.js';
import type { Resource, ResourceEncoding, ResourceType } from '../resource.js';
import type { DownloadResource, ProcessingLifeCycle, RequestOptions, SubmitResourceFunc } from '../life-cycle/types.js';
import type { PipelineExecutor } from '../life-cycle/pipeline-executor.js';
import type { Cheerio } from '../types.js';
import type { DownloaderWithMeta } from './types.js';
import type { WorkerInfo } from './worker-pool.js';
/**
 * Pipeline executor
 */
export declare class PipelineExecutorImpl implements PipelineExecutor {
    lifeCycle: ProcessingLifeCycle;
    requestOptions: RequestOptions;
    options: StaticDownloadOptions;
    constructor(lifeCycle: ProcessingLifeCycle, requestOptions: RequestOptions, options: StaticDownloadOptions);
    init(pipeline: PipelineExecutor, downloader?: DownloaderWithMeta): Promise<void>;
    createAndProcessResource(rawUrl: string, defaultType: ResourceType, depth: number | void | null, element: Cheerio | null, parent: Resource): Promise<Resource | void>;
    linkRedirect(url: string, element: Cheerio | null, parent: Resource | null): Promise<string | void>;
    detectResourceType(url: string, type: ResourceType, element: Cheerio | null, parent: Resource | null): Promise<ResourceType | void>;
    createResource(type: ResourceType, depth: number, url: string, refUrl: string, localRoot?: string, encoding?: ResourceEncoding, refSavePath?: string, refType?: ResourceType): Resource;
    processBeforeDownload(res: Resource, element: Cheerio | null, parent: Resource | null, options?: StaticDownloadOptions): Promise<Resource | void>;
    download(res: Resource, requestOptions?: RequestOptions, options?: StaticDownloadOptions): Promise<DownloadResource | void>;
    /**
     * Process resource after download, in worker thread
     * @param res resource received from main thread
     * @param submit function to submit resource to pipeline
     * @param options
     */
    processAfterDownload(res: DownloadResource, submit: SubmitResourceFunc, options?: StaticDownloadOptions): Promise<DownloadResource | void>;
    saveToDisk(res: DownloadResource, options?: StaticDownloadOptions): Promise<DownloadResource | void>;
    dispose(pipeline: PipelineExecutor, downloader: DownloaderWithMeta, workerInfo?: WorkerInfo, workerExitCode?: number): Promise<void>;
}
//# sourceMappingURL=pipeline-executor-impl.d.ts.map