crawler-ts-htmlparser2/dist/esm/filter.d.ts

Lightweight crawler written in TypeScript using ES6 generators.

import { Logger, Filter } from 'crawler-ts';
export interface Response {
    status: number;
    headers: Record<string, string>;
    body: string;
}
export declare type UrlFilter = Filter<URL>;
export declare const urlToString: (url: URL) => string;
export declare const allowUrlRegex: (allowUrls: RegExp[], logger?: Logger | undefined) => Filter<URL>;
export declare const allowUrlExtensions: (allowedExtensions: string[], logger?: Logger | undefined) => Filter<URL>;
export declare const ignoreUrlRegex: (ignoredUrls: RegExp[], logger?: Logger | undefined) => Filter<URL>;
export declare const ignoreUrlDoubles: (logger?: Logger | undefined) => Filter<URL>;
export declare function allowHttpOk(logger?: Logger): <Res extends Response>({ response }: {
    response: Res;
}) => boolean;
export declare function allowHtml(logger?: Logger): <Res extends Response>({ location, response }: {
    location: URL;
    response: Res;
}) => boolean;
export declare function allowHosts(allowedHosts: string[], logger?: Logger): UrlFilter;
export declare function allowProtocols(allowedProtocols: string[], logger?: Logger): UrlFilter;