gpt-research/dist/scrapers/BaseScraper.d.ts

Autonomous AI research agent that conducts comprehensive research on any topic and generates detailed reports with citations

import { EventEmitter } from 'events';
import { ScrapedContent } from '../types';
export interface ScraperConfig {
    timeout?: number;
    headers?: Record<string, string>;
    userAgent?: string;
    maxRetries?: number;
    followRedirects?: boolean;
}
export declare abstract class BaseScraper extends EventEmitter {
    protected config: ScraperConfig;
    protected name: string;
    constructor(name: string, config?: ScraperConfig);
    /**
     * Main scraping method that must be implemented by each scraper
     */
    abstract scrape(url: string): Promise<ScrapedContent>;
    /**
     * Batch scraping for multiple URLs
     */
    scrapeMultiple(urls: string[], concurrency?: number): Promise<ScrapedContent[]>;
    /**
     * Get the name of the scraper
     */
    getName(): string;
    /**
     * Validate URL before scraping
     */
    protected validateUrl(url: string): boolean;
    /**
     * Clean and normalize text content
     */
    protected cleanText(text: string): string;
    /**
     * Extract metadata from HTML
     */
    protected extractMetadata(html: string): Record<string, any>;
    /**
     * Convert HTML to Markdown (basic implementation)
     */
    protected htmlToMarkdown(html: string): string;
    /**
     * Handle errors consistently
     */
    protected handleError(error: any, url: string): ScrapedContent;
    /**
     * Retry logic for scraping
     */
    protected retry<T>(fn: () => Promise<T>, retries?: number, delay?: number): Promise<T>;
}
export declare class ScraperFactory {
    private static scrapers;
    static register(name: string, scraper: typeof BaseScraper): void;
    static create(name: string, config?: ScraperConfig): BaseScraper;
    static getAvailableScrapers(): string[];
}
//# sourceMappingURL=BaseScraper.d.ts.map