chen-crawler/dist/crawler/page.d.ts

Web Crawler Provider for Chen Framework

import { Crawler } from './base';
import { HttpClientOptions } from 'chen/web';
import { Storage } from '../storage';
/**
 * PageCrawler class
 */
export declare class PageCrawler extends Crawler {
    /**
     * Worker Process ID
     * @type {string}
     */
    protected worker: string;
    /**
     * PageCrawler constructor
     * @param {Storage} storage
     * @param {string} name
     * @param {string} startingUrl
     * @param {HttpClientOptions} config
     */
    constructor(storage: Storage, name: string, startingUrl: string, config: HttpClientOptions);
    /**
    * Crawl url data
    */
    private crawlUrlData();
    /**
     * Start crawling
     */
    protected crawl(): Promise<void>;
}