website-scrap-engine/lib/life-cycle/adapters.d.ts

Configurable website scraper in typescript

import type { Resource } from '../resource.js';
import { ResourceType } from '../resource.js';
import type { AsyncResult, DownloadResource, LinkRedirectFunc, ProcessResourceAfterDownloadFunc, ProcessResourceBeforeDownloadFunc, SubmitResourceFunc } from './types.js';
import type { StaticDownloadOptions } from '../options.js';
import type { PipelineExecutor } from './pipeline-executor.js';
import type { Cheerio, CheerioStatic } from '../types.js';
export interface SkipProcessFunc {
    (url: string, element: Cheerio | null, parent: Resource | null): boolean;
}
export declare const skipProcess: (fn: SkipProcessFunc) => LinkRedirectFunc;
export interface DropResourceFunc {
    (res: Resource): boolean;
}
export declare const dropResource: (fn: DropResourceFunc) => ProcessResourceBeforeDownloadFunc;
export interface PreProcessResourceFunc {
    (url: string, element: Cheerio | null, res: Resource, parent: Resource | null): void;
}
export declare const preProcess: (fn: PreProcessResourceFunc) => ProcessResourceBeforeDownloadFunc;
export interface RequestRedirectFunc {
    (url: string, res: Resource): string | void;
}
export declare const requestRedirect: (fn: RequestRedirectFunc) => ProcessResourceBeforeDownloadFunc;
export declare const redirectFilter: (fn: RequestRedirectFunc) => ProcessResourceAfterDownloadFunc;
export declare function processRedirectedUrl(res: DownloadResource, submit: SubmitResourceFunc, options: StaticDownloadOptions, pipeline: PipelineExecutor): Promise<DownloadResource | void>;
export interface HtmlProcessFunc {
    ($: CheerioStatic, res: Resource & {
        type: ResourceType.Html;
    }): CheerioStatic;
}
export declare const parseHtml: (res: DownloadResource & {
    type: ResourceType.Html;
}, options: StaticDownloadOptions) => CheerioStatic;
export declare const processHtml: (fn: HtmlProcessFunc) => ProcessResourceAfterDownloadFunc;
export interface AsyncHtmlProcessFunc {
    ($: CheerioStatic, res: Resource & {
        type: ResourceType.Html;
    }): AsyncResult<CheerioStatic>;
}
export declare const processHtmlAsync: (fn: AsyncHtmlProcessFunc) => ProcessResourceAfterDownloadFunc;
//# sourceMappingURL=adapters.d.ts.map