crewai-ts/knowledge/source/FileKnowledgeSource.d.ts

TypeScript port of crewAI for agent-based workflows

/**
 * File Knowledge Source
 * Process local files as knowledge sources with optimized streaming
 */
import { BaseKnowledgeSource, KnowledgeSourceOptions } from './BaseKnowledgeSource.js';
import { KnowledgeChunk, Metadata } from '../types.js';
export interface FileKnowledgeSourceOptions extends KnowledgeSourceOptions {
    /**
     * Path to the file(s) to load
     * Can be a single file path or array of file paths
     */
    filePath: string | string[];
    /**
     * File encoding to use
     * @default 'utf-8'
     */
    encoding?: BufferEncoding;
    /**
     * Automatically detect file type based on extension
     * and apply appropriate pre-processing
     * @default true
     */
    autoDetectType?: boolean;
    /**
     * Additional metadata to attach to all chunks
     */
    metadata?: Metadata;
    /**
     * Process recursively if directory is provided
     * @default false
     */
    recursive?: boolean;
    /**
     * File extensions to include (if directory is provided)
     * @default ['.txt', '.md', '.html', '.pdf', '.csv', '.json']
     */
    includedExtensions?: string[];
    /**
     * Maximum file size in bytes to process
     * @default 10485760 (10MB)
     */
    maxFileSize?: number;
}
/**
 * Knowledge source for processing file content
 * Optimized for memory efficiency with streaming and incremental processing
 */
export declare class FileKnowledgeSource extends BaseKnowledgeSource {
    /**
     * File paths to process
     * @private
     */
    private filePaths;
    /**
     * Configured file options with defaults
     * @private
     */
    private fileOptions;
    /**
     * Constructor for FileKnowledgeSource
     * @param options - Configuration options
     */
    constructor(options: FileKnowledgeSourceOptions);
    /**
     * Process and add all file content to storage
     * Implements streaming and chunking for memory efficiency
     * @override
     */
    add(): Promise<void>;
    /**
     * Create a knowledge chunk from file content with file-specific metadata
     * @param content - File content for the chunk
     * @param additionalMetadata - Additional metadata or file path
     * @returns Knowledge chunk object
     * @private
     */
    protected createChunk(content: string, additionalMetadata?: Metadata | string): KnowledgeChunk;
    /**
     * Process a single file
     * Uses streaming for memory efficiency with large files
     * @param filePath - Path to the file
     * @private
     */
    private processFile;
    /**
     * Expands file paths to include files from directories if recursive option is enabled
     * @returns Expanded list of file paths
     * @private
     */
    private expandFilePaths;
    /**
     * Gets all files from a directory recursively
     * @param dirPath - Directory path
     * @returns List of file paths
     * @private
     */
    private getFilesFromDirectory;
    /**
     * Process a plain text file with streaming
     * @param filePath - Path to the text file
     * @private
     */
    private processTextFile;
    /**
     * Process a PDF file
     * @param filePath - Path to the PDF file
     * @private
     */
    private processPdfFile;
    /**
     * Process an HTML file
     * @param filePath - Path to the HTML file
     * @private
     */
    private processHtmlFile;
    /**
     * Process a CSV file
     * @param filePath - Path to the CSV file
     * @private
     */
    private processCsvFile;
    /**
     * Process a JSON file
     * @param filePath - Path to the JSON file
     * @private
     */
    private processJsonFile;
    /**
     * Process JSON data recursively
     * @param data - JSON data
     * @param filePath - Source file path
     * @param path - Current JSON path
     * @returns Array of knowledge chunks
     * @private
     */
    private processJsonData;
    /**
     * Extract text content from HTML
     * Simplified version for demonstration
     * @param htmlContent - HTML content
     * @returns Extracted text
     * @private
     */
    private extractTextFromHtml;
}
//# sourceMappingURL=FileKnowledgeSource.d.ts.map