crewai-ts/knowledge/source/URLKnowledgeSource.d.ts

TypeScript port of crewAI for agent-based workflows

/**
 * URL Knowledge Source
 * Process web content as knowledge sources with optimized fetching and streaming
 */
import { BaseKnowledgeSource, KnowledgeSourceOptions } from './BaseKnowledgeSource.js';
import { KnowledgeChunk, Metadata } from '../types.js';
/**
 * Options specific to URL knowledge sources
 */
export interface URLKnowledgeSourceOptions extends KnowledgeSourceOptions {
    /**
     * URL or array of URLs to fetch
     */
    url: string | string[];
    /**
     * Additional request headers
     */
    headers?: Record<string, string>;
    /**
     * Enable request caching with ETag and Last-Modified
     * @default true
     */
    enableCaching?: boolean;
    /**
     * Cache TTL in milliseconds
     * @default 3600000 (1 hour)
     */
    cacheTTL?: number;
    /**
     * Additional metadata to attach to all chunks
     */
    metadata?: Metadata;
    /**
     * Follow URLs found in content (depth of crawling)
     * @default 0 (no following)
     */
    followDepth?: number;
    /**
     * Request timeout in milliseconds
     * @default 30000 (30 seconds)
     */
    timeout?: number;
    /**
     * Maximum retries for failed requests
     * @default 3
     */
    maxRetries?: number;
    /**
     * Retry delay in milliseconds (base value for exponential backoff)
     * @default 1000
     */
    retryDelay?: number;
    /**
     * Rate limit in milliseconds (minimum time between requests)
     * @default 100
     */
    rateLimit?: number;
    /**
     * Extract text from HTML
     * @default true
     */
    extractText?: boolean;
    /**
     * Include CSS selectors to prioritize in HTML extraction (e.g., 'article', '.content')
     */
    includeSelectors?: string[];
    /**
     * Exclude CSS selectors from HTML extraction (e.g., 'nav', 'footer', '.ads')
     */
    excludeSelectors?: string[];
    /**
     * Restrict URL following to specific domains
     */
    restrictToDomains?: string[];
}
/**
 * Knowledge source for processing web content
 * Optimized for efficient fetching, caching, and content extraction
 */
export declare class URLKnowledgeSource extends BaseKnowledgeSource {
    /**
     * URLs to process
     * @private
     */
    private urls;
    /**
     * Configured URL options with defaults
     * @private
     */
    private urlOptions;
    /**
     * Request cache for ETag and Last-Modified based caching
     * @private
     */
    private requestCache;
    /**
     * Timestamp of last request (for rate limiting)
     * @private
     */
    private lastRequestTime;
    /**
     * Constructor for URLKnowledgeSource
     * @param options - Configuration options
     */
    constructor(options: URLKnowledgeSourceOptions);
    /**
     * Process and add all URLs to storage
     * Implements efficient fetching with caching and rate limiting
     * @override
     */
    add(): Promise<void>;
    /**
     * Create a knowledge chunk from URL content with URL-specific metadata
     * @param content - Content for the chunk
     * @param additionalMetadata - Additional metadata or source URL
     * @returns Knowledge chunk object
     * @private
     */
    protected createChunk(content: string, additionalMetadata?: Metadata | string): KnowledgeChunk;
    /**
     * Process a single URL
     * Implements caching, rate limiting, and retry logic
     * @param url - URL to process
     * @param depth - Current crawl depth
     * @private
     */
    private processURL;
    /**
     * Fetch URL content with retry logic and caching
     * @param url - URL to fetch
     * @returns Content and found URLs
     * @private
     */
    private fetchWithRetry;
    /**
     * Fetch URL with caching support
     * @param url - URL to fetch
     * @returns Content and found URLs
     * @private
     */
    private fetchWithCaching;
    /**
     * Apply rate limiting
     * Ensures minimum time between requests
     * @private
     */
    private applyRateLimit;
    /**
     * Check if content is HTML
     * @param content - Content to check
     * @returns True if content appears to be HTML
     * @private
     */
    private isHtml;
    /**
     * Extract text content from HTML
     * Implements efficient HTML parsing with selector support
     * @param htmlContent - HTML content
     * @returns Extracted text
     * @private
     */
    private extractTextFromHtml;
    /**
     * Extract URLs from HTML content
     * @param htmlContent - HTML content
     * @param baseUrl - Base URL for resolving relative URLs
     * @returns Array of absolute URLs
     * @private
     */
    private extractUrls;
}
//# sourceMappingURL=URLKnowledgeSource.d.ts.map