spido/dist/lib/core/crawler.d.ts

Web crawler/spider for node.js & nest.js server.

import { Queue } from "./queue";
import { CrawlerOptions, HttpResponse, Metadata } from "./interfaces";
export declare class Spido {
    private url;
    private options;
    private cache;
    private utils;
    visited: Set<string>;
    queue: Queue;
    websiteSeoData: Metadata[];
    constructor(url: string, options?: CrawlerOptions);
    crawl(): Promise<Metadata[]>;
    private isDepthExceeded;
    handleResponse(url: string, response: HttpResponse): Promise<void>;
    private enqueueURLs;
    fetch(url: string): Promise<number>;
    private internalLinksEnabled;
}