microfrontier/dist/types/URLFrontier.d.ts

A web crawler frontier implementation in TypeScript backed by Redis. MicroFrontier is a scalable and distributed frontier implemented through Redis Queues.

import IORedis from "ioredis";
import { URLFrontierConfig } from "./URLFrontierConfig";
export declare class URLFrontier {
    redisClient: IORedis.Redis;
    config: URLFrontierConfig;
    defaultConfig: URLFrontierConfig;
    /**
     * @param  {Redis} redisClient - Your ioredis client
     * @param  {UrlFrontierConfig=defaultConfig} config - UrlFrontier configuration
     */
    constructor(configuration: {
        config?: URLFrontierConfig;
        redisClient?: IORedis.Redis;
        redisOptions?: IORedis.RedisOptions;
    });
    /**
     * Add an URL to the frontier Front-end queue
     * @param  {string} url - URL to be added
     * @param  {string} priority - Priority queue name
     */
    add(url: string, priority: string, meta?: any): Promise<void>;
    /**
     * Set a hostname crawl delay
     * @param  {string} hostname - eg. www.google.com
     * @param  {number} delay - In milliseconds
     */
    setHostnameCrawlDelay(hostname: string, delay: number): Promise<number>;
    /**
     * Get redis key name for queue+priority
     * @param  {string} priority
     */
    static getFrontendQueueName(config: URLFrontierConfig, priority: string): string;
    static getBackendQueueName(config: URLFrontierConfig, hostname: string): string;
    static getHeapName(config: URLFrontierConfig): string;
    static getHostnameUrlCountName(config: URLFrontierConfig): string;
    static getHostnameCrawlDelayName(config: URLFrontierConfig): string;
    get(): Promise<any>;
    getHeap(cursor?: number): Promise<[string, string[]]>;
    getHostnameUrlsCount(hostname: string): Promise<number>;
    getBackend(hostname: string, cursor?: number, start?: number, stop?: number): Promise<string[]>;
}