chen-crawler/dist/queue.d.ts

Web Crawler Provider for Chen Framework

import { Storage } from './storage';
import { Url } from 'url';
/**
 * QueueFilter interface
 */
export interface QueueFilter extends Function {
    (url: Url): boolean;
}
/**
 * Queue class
 */
export declare class Queue {
    private storage;
    private id;
    /**
     * Suffix
     * @type {String}
     */
    private static QUEUE_SUFFIX;
    /**
     * Queue constructor
     * @param {Storage} private storage
     * @param {string} private id
     */
    constructor(storage: Storage, id: string);
    /**
     * Add to queue
     * @param {string} url
     */
    push(url: string): Promise<void>;
    /**
     * Pop from queue
     */
    shift(): Promise<string>;
}
/**
 * ProcessingList class
 */
export declare class ProcessingList {
    private storage;
    private id;
    /**
     * Suffix
     * @type {string}
     */
    private static LIST_SUFFIX;
    /**
     * ProcessingList constructor
     * @param {Storage} private storage
     * @param {string}  private id
     */
    constructor(storage: Storage, id: string);
    /**
     * Push url to processing list
     * @param  {string} url
     * @return {Promise<void>}
     */
    add(url: string): Promise<void>;
    /**
     * Check if url exists in processing list
     * @param  {string} url
     * @return {Promise<boolean>}
     */
    has(url: string): Promise<boolean>;
    /**
     * Remove url in processing list
     * @param  {string} url
     * @return {Promise<number>}
     */
    remove(url: string): Promise<number>;
}