crawler-ts-htmlparser2/dist/esm/crawl.d.ts

Lightweight crawler written in TypeScript using ES6 generators.

import { Document } from 'domhandler';
import { Options as OptionsBase } from 'crawler-ts';
import { Response } from './filter';
export declare type Parsed = Document;
export declare function createRequester(delayMilliseconds?: number): (url: URL) => Promise<Response>;
export declare function parser<R extends Response>({ response }: {
    response: R;
}): Promise<Parsed>;
export declare function follower({ location, parsed }: {
    location: URL;
    parsed: Parsed;
}): AsyncGenerator<URL>;
export declare type Options = Omit<OptionsBase<URL, Response, Parsed>, 'follower' | 'parser' | 'requester'>;
export declare function createCrawler(options: Options): (start: URL) => AsyncGenerator<import("crawler-ts").PostParseProps<URL, Response, Document>, any, unknown>;