@siren-js/client/dist/crawl/politeness-policy.d.ts

Siren API client library

import { Target } from '../utils/types';
/** [Politeness policy](https://en.wikipedia.org/wiki/Web_crawler#Politeness_policy) for a Crawler */
export interface PolitenessPolicy {
    /** Number of milliseconds to wait between requests. Use `0` to avoid waiting. */
    delay: number;
    /** @returns whether `target` can be crawled */
    canCrawl(target: Target): boolean;
}
/**
 * Default {@linkcode PolitenessPolicy} that returns `true` for all {@linkcode Target Targets}
 * and has a delay of 1,000 milliseconds.
 */
export declare class DefaultPolitenessPolicy implements PolitenessPolicy {
    readonly delay: number;
    static readonly DEFAULT_DELAY = 1000;
    constructor(delay?: number);
    canCrawl(_: Target): boolean;
}