@avenga/linkinator/build/src/crawler.d.ts

Find broken links, missing images, etc in your HTML. Scurry around your site and find all those broken links.

import { EventEmitter } from 'node:events';
import { type CheckOptions, type InternalCheckOptions } from './options.ts';
import { Queue } from './queue.ts';
import { type CrawlResult, type ElementMetadata, type LinkResult, type RetryInfo } from './types.ts';
export type CrawlOptions = {
    url: URL;
    parent?: string;
    crawl: boolean;
    results: LinkResult[];
    cache: Set<string>;
    delayCache: Map<string, number>;
    retryErrorsCache: Map<string, number>;
    retryNoHeaderCache: Map<string, number>;
    checkOptions: InternalCheckOptions;
    queue: Queue;
    rootPath: string;
    retry: boolean;
    retryNoHeader: boolean;
    retryNoHeaderCount: number;
    retryNoHeaderDelay: number;
    retryErrors: boolean;
    retryErrorsCount: number;
    retryErrorsJitter: number;
    extraHeaders: {
        [key: string]: string;
    };
    elementMetadata?: ElementMetadata;
};
/**
 * Instance class used to perform a crawl job.
 */
export declare class LinkChecker extends EventEmitter {
    on(event: 'link', listener: (result: LinkResult) => void): this;
    on(event: 'pagestart', listener: (link: string) => void): this;
    on(event: 'retry', listener: (details: RetryInfo) => void): this;
    /**
     * Crawl a given url or path, and return a list of visited links along with
     * status codes.
     * @param options Options to use while checking for 404s
     */
    check(options_: CheckOptions): Promise<CrawlResult>;
    private setupLocalServer;
    /**
     * Crawl a given url with the provided options.
     * @param opts List of options used to do the crawl
     */
    crawl(opts: CrawlOptions): Promise<void>;
    private requestWithRetry;
    private emitResult;
    /**
     * Checks for HTTP 429 status in the incoming response and for the existence
     * of a `retry-after` header. Adds the request to the queue again after
     * calculating the date when it should be retried, depending on a
     * `retry-after` header existing and configuration.
     * @param response Response returned from the request
     * @param opts CrawlOptions used during this request
     */
    shouldRetryAfter(response: Response, options: CrawlOptions): boolean;
    /**
     * If the response is a 5xx or synthetic 0 response retry N times.
     * @param status Status returned by request or 0 if request threw.
     * @param opts CrawlOptions used during this request
     */
    shouldRetryOnError(status: number, options: CrawlOptions): boolean;
    /**
     * If `crawl` is enabled and the response is HTML, recursively check its links
     */
    private maybeRecurse;
    private shouldSkip;
    /**
     * Adds the link to the results as skipped and returns `true` when
     * the protocol is not http or https.
     */
    private skipProtocol;
    /**
     * Adds the link to the results as skipped and returns `true` when a function
     * is defined as condition to skip links and calling the function with the
     * URL returns true.
     */
    private skipLinksFunction;
    /**
     * Adds the link to the results as skipped and returns `true` when an
     * array of link patterns to skip has been configured and at least
     * one of them matches.
     */
    private skipLinksArray;
    /**
     * Adds the link to the queue again and returns `true` when an existing
     * active delay for this URL exists.
     */
    private handleExistingDelay;
}