chen-crawler/dist/manager.d.ts

Web Crawler Provider for Chen Framework

import { Service } from 'chen/core';
import { HttpClientOptions } from 'chen/web';
import { PageCrawler, SitemapCrawler } from './crawler';
import { StorageConfig } from './storage';
/**
 * WebCrawlerConfig interface
 */
export interface WebCrawlerConfig extends HttpClientOptions {
    httpClient: HttpClientOptions;
    storage: StorageConfig;
}
/**
 * WebCrawlerManager class
 */
export declare class WebCrawlerManager extends Service {
    /**
     * Web crawler config
     * @type {WebCrawlerConfig}
     */
    private defaultConfig;
    /**
     * Storage
     * @type {Storage}
     */
    private storage;
    /**
     * Init hook
     */
    init(): void;
    /**
     * Create page crawler
     * @param  {string} name
     * @param  {string} startingUrl
     * @param  {WebCrawlerConfig} customConfig
     * @return {PageCrawler}
     */
    createPageCrawler(name: string, startingUrl: string, customConfig?: HttpClientOptions): PageCrawler;
    /**
     * Create sitemap crawler
     * @param  {string} name
     * @param  {string} startingUrl
     * @param  {HttpClientOptions} customConfig
     * @return {SitemapCrawler}
     */
    createSitemapCrawler(name: string, startingUrl: string, customConfig?: HttpClientOptions): SitemapCrawler;
}