@neureus/rag

import { z } from 'zod'; import { ChatMessage, AIProvider as AIProvider$1 } from '@neureus/ai-gateway'; import { VectorEntry, VectorSearchResult, VectorId } from '@neureus/vector-db'; type KVNamespace = any; type R2Bucket = any; type D1Database = any; type AnalyticsEngineDataset = any; type AIProvider = 'openai' | 'anthropic' | 'google' | 'cloudflare' | 'cohere' | 'mistral'; type DocumentFormat = 'markdown' | 'html' | 'pdf' | 'txt' | 'docx' | 'csv' | 'json' | 'xml' | 'image' | 'audio' | 'video'; type DocumentSource = 'file' | 'url' | 'text' | 's3' | 'r2' | 'github' | 'webhook' | 'email' | 'auto'; interface DocumentMetadata { id: string; title?: string; author?: string; createdAt?: number; updatedAt?: number; url?: string; filePath?: string; fileSize?: number; format: DocumentFormat; source: DocumentSource; tags?: string[]; language?: string; custom?: Record<string, any>; } declare const DocumentSchema: z.ZodObject<{ id: z.ZodString; content: z.ZodString; metadata: z.ZodObject<{ id: z.ZodString; title: z.ZodOptional<z.ZodString>; author: z.ZodOptional<z.ZodString>; createdAt: z.ZodOptional<z.ZodNumber>; updatedAt: z.ZodOptional<z.ZodNumber>; url: z.ZodOptional<z.ZodString>; filePath: z.ZodOptional<z.ZodString>; fileSize: z.ZodOptional<z.ZodNumber>; format: z.ZodEnum<{ markdown: "markdown"; html: "html"; pdf: "pdf"; txt: "txt"; docx: "docx"; csv: "csv"; json: "json"; xml: "xml"; image: "image"; audio: "audio"; video: "video"; }>; source: z.ZodEnum<{ file: "file"; url: "url"; text: "text"; s3: "s3"; r2: "r2"; github: "github"; auto: "auto"; webhook: "webhook"; email: "email"; }>; tags: z.ZodOptional<z.ZodArray<z.ZodString>>; language: z.ZodOptional<z.ZodString>; custom: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>; }, z.core.$strip>; }, z.core.$strip>; type Document = z.infer<typeof DocumentSchema>; type ChunkingStrategy = 'fixed_size' | 'semantic' | 'recursive' | 'sentence' | 'paragraph' | 'custom'; interface ChunkingConfig { strategy: ChunkingStrategy; size: number; overlap: number; minChunkSize?: number; maxChunkSize?: number; separators?: string[]; preserveStructure?: boolean; customSplitter?: (text: string, config: ChunkingConfig) => Chunk[]; } interface ChunkMetadata { chunkId: string; documentId: string; index: number; startIndex: number; endIndex: number; title?: string; section?: string; headers?: string[]; pageNumber?: number; custom?: Record<string, any>; } declare const ChunkSchema: z.ZodObject<{ id: z.ZodString; content: z.ZodString; tokens: z.ZodNumber; embedding: z.ZodOptional<z.ZodArray<z.ZodNumber>>; metadata: z.ZodObject<{ chunkId: z.ZodString; documentId: z.ZodString; index: z.ZodNumber; startIndex: z.ZodNumber; endIndex: z.ZodNumber; title: z.ZodOptional<z.ZodString>; section: z.ZodOptional<z.ZodString>; headers: z.ZodOptional<z.ZodArray<z.ZodString>>; pageNumber: z.ZodOptional<z.ZodNumber>; custom: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>; }, z.core.$strip>; }, z.core.$strip>; type Chunk = z.infer<typeof ChunkSchema>; interface EmbeddingConfig { model: string; provider: AIProvider; dimensions: number; batchSize: number; maxRetries: number; timeout: number; } declare const RAGConfigSchema: z.ZodObject<{ name: z.ZodString; description: z.ZodOptional<z.ZodString>; embedding: z.ZodObject<{ model: z.ZodString; provider: z.ZodEnum<{ openai: "openai"; anthropic: "anthropic"; google: "google"; cloudflare: "cloudflare"; cohere: "cohere"; mistral: "mistral"; }>; dimensions: z.ZodDefault<z.ZodNumber>; batchSize: z.ZodDefault<z.ZodNumber>; maxRetries: z.ZodDefault<z.ZodNumber>; timeout: z.ZodDefault<z.ZodNumber>; }, z.core.$strip>; chunking: z.ZodObject<{ strategy: z.ZodDefault<z.ZodEnum<{ fixed_size: "fixed_size"; semantic: "semantic"; recursive: "recursive"; sentence: "sentence"; paragraph: "paragraph"; custom: "custom"; }>>; size: z.ZodDefault<z.ZodNumber>; overlap: z.ZodDefault<z.ZodNumber>; minChunkSize: z.ZodDefault<z.ZodNumber>; maxChunkSize: z.ZodDefault<z.ZodNumber>; separators: z.ZodOptional<z.ZodArray<z.ZodString>>; preserveStructure: z.ZodDefault<z.ZodBoolean>; }, z.core.$strip>; retrieval: z.ZodObject<{ topK: z.ZodDefault<z.ZodNumber>; minSimilarity: z.ZodDefault<z.ZodNumber>; hybridWeight: z.ZodDefault<z.ZodNumber>; rerankModel: z.ZodOptional<z.ZodString>; maxContextTokens: z.ZodDefault<z.ZodNumber>; }, z.core.$strip>; generation: z.ZodObject<{ model: z.ZodString; provider: z.ZodEnum<{ openai: "openai"; anthropic: "anthropic"; google: "google"; cloudflare: "cloudflare"; cohere: "cohere"; mistral: "mistral"; }>; temperature: z.ZodDefault<z.ZodNumber>; maxTokens: z.ZodDefault<z.ZodNumber>; systemPrompt: z.ZodOptional<z.ZodString>; includeSource: z.ZodDefault<z.ZodBoolean>; streaming: z.ZodDefault<z.ZodBoolean>; }, z.core.$strip>; analytics: z.ZodObject<{ enabled: z.ZodDefault<z.ZodBoolean>; trackQueries: z.ZodDefault<z.ZodBoolean>; trackPerformance: z.ZodDefault<z.ZodBoolean>; sampleRate: z.ZodDefault<z.ZodNumber>; }, z.core.$strip>; }, z.core.$strip>; type RAGConfig = z.infer<typeof RAGConfigSchema>; declare const IngestionRequestSchema: z.ZodObject<{ source: z.ZodString; type: z.ZodDefault<z.ZodEnum<{ file: "file"; url: "url"; text: "text"; s3: "s3"; r2: "r2"; github: "github"; auto: "auto"; webhook: "webhook"; email: "email"; }>>; format: z.ZodOptional<z.ZodEnum<{ markdown: "markdown"; html: "html"; pdf: "pdf"; txt: "txt"; docx: "docx"; csv: "csv"; json: "json"; xml: "xml"; image: "image"; audio: "audio"; video: "video"; }>>; metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>; recursive: z.ZodDefault<z.ZodBoolean>; filters: z.ZodOptional<z.ZodArray<z.ZodString>>; excludes: z.ZodOptional<z.ZodArray<z.ZodString>>; }, z.core.$strip>; type IngestionRequest = z.infer<typeof IngestionRequestSchema>; declare const QueryRequestSchema: z.ZodObject<{ query: z.ZodString; topK: z.ZodDefault<z.ZodNumber>; minSimilarity: z.ZodDefault<z.ZodNumber>; filter: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>; namespace: z.ZodOptional<z.ZodString>; includeSource: z.ZodDefault<z.ZodBoolean>; streaming: z.ZodDefault<z.ZodBoolean>; conversationHistory: z.ZodOptional<z.ZodArray<z.ZodObject<{ role: z.ZodEnum<{ system: "system"; user: "user"; assistant: "assistant"; }>; content: z.ZodString; }, z.core.$strip>>>; customPrompt: z.ZodOptional<z.ZodString>; rerankResults: z.ZodDefault<z.ZodBoolean>; }, z.core.$strip>; type QueryRequest = z.infer<typeof QueryRequestSchema>; interface ContextResult { chunks: Chunk[]; sources: DocumentMetadata[]; totalTokens: number; retrievalTime: number; relevanceScores: number[]; } interface QueryResponse { answer: string; sources: Array<{ documentId: string; chunkId: string; title?: string; url?: string; relevanceScore: number; content: string; }>; context: ContextResult; usage: { promptTokens: number; completionTokens: number; totalTokens: number; }; performance: { retrievalTime: number; generationTime: number; totalTime: number; }; conversationId?: string; requestId: string; } interface QueryStreamChunk { type: 'context' | 'answer' | 'source' | 'complete' | 'error'; data: any; requestId: string; } interface RAGAnalyticsEvent { type: 'ingestion' | 'query' | 'error' | 'performance'; timestamp: number; pipelineName: string; requestId: string; userId?: string; metadata: { documentsCount?: number; chunksCount?: number; processingTime?: number; query?: string; responseTime?: number; retrievalTime?: number; generationTime?: number; sourcesCount?: number; tokensUsed?: number; error?: string; errorCode?: string; latency?: number; throughput?: number; cost?: number; }; } type RAGPipelineStatus = 'initializing' | 'ready' | 'processing' | 'error' | 'disabled'; interface RAGPipelineInfo { name: string; status: RAGPipelineStatus; config: RAGConfig; stats: { documentsCount: number; chunksCount: number; totalQueries: number; avgResponseTime: number; lastUsed?: number; createdAt: number; updatedAt: number; }; error?: string; } interface ProcessingResult { document: Document; chunks: Chunk[]; embeddings: number[][]; processingTime: number; tokenCount: number; errors?: string[]; } interface BatchProcessingResult { results: ProcessingResult[]; totalDocuments: number; totalChunks: number; totalTokens: number; processingTime: number; errors: Array<{ documentId: string; error: string; }>; } interface AutoIndexingConfig { enabled: boolean; sources: DocumentSource[]; patterns: string[]; excludePatterns: string[]; maxFileSize: number; supportedFormats: DocumentFormat[]; scheduleCron?: string; webhookSecret?: string; } declare class RAGError extends Error { code: string; statusCode: number; pipelineName?: string; originalError?: Error; constructor(message: string, code: string, statusCode?: number, pipelineName?: string, originalError?: Error); } declare class DocumentProcessingError extends RAGError { constructor(documentId: string, message: string, originalError?: Error); } declare class EmbeddingError extends RAGError { constructor(message: string, originalError?: Error); } declare class RetrievalError extends RAGError { constructor(message: string, originalError?: Error); } declare class GenerationError extends RAGError { constructor(message: string, originalError?: Error); } interface RAGEnv { VECTOR_KV: KVNamespace; VECTOR_BUCKET: R2Bucket; VECTOR_DB: D1Database; CACHE: KVNamespace; ANALYTICS: AnalyticsEngineDataset; RAG_BUCKET: R2Bucket; RAG_KV: KVNamespace; RAG_QUEUE?: any; AI?: any; VECTORIZE?: any; OPENAI_API_KEY: string; ANTHROPIC_API_KEY: string; GOOGLE_API_KEY: string; COHERE_API_KEY: string; MISTRAL_API_KEY: string; RAG_DEFAULT_EMBEDDING_MODEL?: string; RAG_DEFAULT_GENERATION_MODEL?: string; RAG_MAX_CONTEXT_TOKENS?: string; RAG_BATCH_SIZE?: string; RAG_AUTO_INDEXING?: string; RAG_SECURITY_MODE?: string; ENVIRONMENT: string; DEBUG?: string; } /** * Main RAG pipeline that orchestrates all components */ declare class RAGPipeline { private env; private config; private status; private error?; private documentLoader; private chunker; private embeddingService; private retriever; private answerGenerator; private stats; constructor(env: RAGEnv, config: RAGConfig); /** * Initialize the RAG pipeline */ initialize(): Promise<void>; /** * Ingest documents into the RAG pipeline */ ingest(request: IngestionRequest | IngestionRequest[]): Promise<BatchProcessingResult>; /** * Query the RAG pipeline */ query(request: QueryRequest): Promise<QueryResponse>; /** * Query with streaming response */ queryStream(request: QueryRequest, onChunk?: (chunk: QueryStreamChunk) => void): Promise<ReadableStream<QueryStreamChunk>>; /** * Process a batch of documents */ private processBatch; /** * Get pipeline information and statistics */ getInfo(): Promise<RAGPipelineInfo>; /** * Update pipeline configuration */ updateConfig(newConfig: Partial<RAGConfig>): Promise<void>; /** * Delete the pipeline and all its data */ delete(): Promise<void>; /** * Create necessary database tables */ private createTables; /** * Store pipeline configuration in KV */ private storePipelineConfig; /** * Check for duplicate content */ private checkDuplicate; /** * Store content hash for deduplication */ private storeContentHash; /** * Record analytics event */ private recordAnalytics; /** * Get pipeline status */ getStatus(): RAGPipelineStatus; /** * Check if pipeline is ready */ isReady(): boolean; /** * Get pipeline configuration */ getConfig(): RAGConfig; } /** * RAG Manager that handles multiple RAG pipelines */ declare class RAGManager { private env; private pipelines; private initialized; constructor(env: RAGEnv); /** * Initialize the RAG manager */ initialize(): Promise<void>; /** * Create a new RAG pipeline */ createPipeline(config: RAGConfig): Promise<RAGPipeline>; /** * Get a pipeline by name */ getPipeline(name: string): RAGPipeline | null; /** * Get pipeline or throw error if not found */ private requirePipeline; /** * List all pipelines */ listPipelines(): Promise<RAGPipelineInfo[]>; /** * Delete a pipeline */ deletePipeline(name: string): Promise<void>; /** * Update a pipeline configuration */ updatePipeline(name: string, config: Partial<RAGConfig>): Promise<void>; /** * Ingest documents into a pipeline */ ingest(pipelineName: string, request: IngestionRequest | IngestionRequest[]): Promise<BatchProcessingResult>; /** * Query a pipeline */ query(pipelineName: string, request: QueryRequest): Promise<QueryResponse>; /** * Query a pipeline with streaming response */ queryStream(pipelineName: string, request: QueryRequest, onChunk?: (chunk: QueryStreamChunk) => void): Promise<ReadableStream<QueryStreamChunk>>; /** * Get pipeline information */ getPipelineInfo(name: string): Promise<RAGPipelineInfo>; /** * Check pipeline health */ checkHealth(): Promise<{ status: 'healthy' | 'degraded' | 'unhealthy'; pipelines: Array<{ name: string; status: RAGPipelineStatus; error?: string; }>; totalPipelines: number; healthyPipelines: number; }>; /** * Get manager statistics */ getStats(): Promise<{ totalPipelines: number; pipelinesByStatus: Record<RAGPipelineStatus, number>; totalDocuments: number; totalChunks: number; totalQueries: number; avgResponseTime: number; }>; /** * Create a default RAG pipeline with optimized settings */ createDefaultPipeline(name: string, description?: string): Promise<RAGPipeline>; /** * Bulk operations for multiple pipelines */ bulkQuery(queries: Array<{ pipelineName: string; request: QueryRequest; }>): Promise<Array<{ pipelineName: string; response?: QueryResponse; error?: string; }>>; /** * Load existing pipelines from storage */ private loadExistingPipelines; /** * Store the list of pipeline names */ private storePipelineList; /** * Clean up resources */ cleanup(): Promise<void>; } /** * Document loader that supports multiple formats and sources */ declare class DocumentLoader { private turndownService; private markdownIt; private env; constructor(env: RAGEnv); /** * Load a document from various sources */ loadDocument(request: IngestionRequest): Promise<Document>; /** * Load multiple documents in batch */ loadDocuments(requests: IngestionRequest[]): Promise<Document[]>; /** * Load document from file path */ private loadFromFile; /** * Load document from URL */ private loadFromUrl; /** * Load document from R2/S3 storage */ private loadFromStorage; /** * Load document from GitHub */ private loadFromGitHub; /** * Process content based on format */ private processContent; /** * Process HTML content */ private processHTML; /** * Process PDF content (already extracted) */ private processPDF; /** * Extract text from PDF buffer */ private extractFromPDF; /** * Process DOCX content */ private processDocx; /** * Process Markdown content */ private processMarkdown; /** * Process CSV content */ private processCSV; /** * Process JSON content */ private processJSON; /** * Process XML content */ private processXML; /** * Convert JSON to readable text */ private jsonToText; /** * Create document metadata */ private createMetadata; /** * Detect format from content type */ private detectFormatFromContentType; /** * Detect format from filename */ private detectFormatFromFilename; /** * Extract title from URL */ private extractTitleFromUrl; /** * Extract title from filename */ private extractTitleFromFilename; } /** * Document chunking service that supports multiple strategies */ declare class DocumentChunker { /** * Chunk a document using the specified strategy */ chunkDocument(document: Document, config: ChunkingConfig): Promise<Chunk[]>; /** * Chunk multiple documents */ chunkDocuments(documents: Document[], config: ChunkingConfig): Promise<Chunk[]>; /** * Fixed-size chunking with overlap */ private fixedSizeChunking; /** * Semantic chunking based on content structure */ private semanticChunking; /** * Recursive chunking with hierarchical splitting */ private recursiveChunking; /** * Recursively split text using hierarchical separators */ private recursiveSplit; /** * Sentence-based chunking */ private sentenceChunking; /** * Paragraph-based chunking */ private paragraphChunking; /** * Split chunks that are too large */ private splitLargeChunks; /** * Create a chunk object */ private createChunk; /** * Validate that a chunk meets the configuration requirements */ private isValidChunk; /** * Get default chunking configuration */ static getDefaultConfig(): ChunkingConfig; /** * Get optimized configuration for specific document types */ static getOptimizedConfig(documentFormat: string): ChunkingConfig; } /** * Embedding service that uses the AI Gateway for generating embeddings */ declare class EmbeddingService { private gateway; private env; private config; private requestIdCounter; constructor(env: RAGEnv, config: EmbeddingConfig); /** * Generate embeddings for a single chunk */ generateEmbedding(chunk: Chunk): Promise<number[]>; /** * Generate embeddings for multiple chunks in batch */ generateEmbeddings(chunks: Chunk[]): Promise<number[][]>; /** * Generate embeddings for text chunks with enhanced content */ generateEnhancedEmbeddings(chunks: Chunk[]): Promise<number[][]>; /** * Generate embedding for a query */ generateQueryEmbedding(query: string): Promise<number[]>; /** * Process a batch of chunks */ private processBatch; /** * Call the embedding model via Cloudflare Workers AI or fallback providers */ private callEmbeddingModel; /** * Call Cloudflare Workers AI for embeddings */ private callCloudflareEmbedding; /** * Call provider-specific embedding APIs through AI Gateway */ private callProviderEmbedding; /** * Call OpenAI embedding API */ private callOpenAIEmbedding; /** * Call Cohere embedding API */ private callCohereEmbedding; /** * Call Google embedding API */ private callGoogleEmbedding; /** * Simulate embedding generation (fallback for unsupported providers) */ private simulateEmbedding; /** * Prepare text for embedding by cleaning and truncating */ private prepareTextForEmbedding; /** * Enhance chunk content with metadata for better embeddings */ private enhanceChunkContent; /** * Get fallback providers for embedding generation */ private getFallbackProviders; /** * Validate embedding dimensions */ validateEmbedding(embedding: number[]): boolean; /** * Calculate embedding similarity */ calculateSimilarity(embedding1: number[], embedding2: number[]): number; /** * Get embedding statistics for monitoring */ getEmbeddingStats(): { provider: string; model: string; dimensions: number; batchSize: number; }; /** * Update embedding configuration */ updateConfig(newConfig: Partial<EmbeddingConfig>): void; /** * Create embedding service with default configuration */ static createDefault(env: RAGEnv): EmbeddingService; /** * Create embedding service for specific model */ static createForModel(env: RAGEnv, model: string, provider: string): EmbeddingService; } /** * Advanced retrieval system with hybrid search capabilities */ declare class HybridRetriever { private vectorDB; private embeddingService; private env; constructor(env: RAGEnv, embeddingService: EmbeddingService); /** * Initialize the retriever */ initialize(): Promise<void>; /** * Retrieve relevant context for a query using hybrid search */ retrieveContext(query: string, options?: { topK?: number; minSimilarity?: number; hybridWeight?: number; rerankResults?: boolean; maxContextTokens?: number; filter?: Record<string, any>; namespace?: string; includeMetadata?: boolean; }): Promise<ContextResult>; /** * Perform vector similarity search */ private vectorSearch; /** * Perform keyword-based search */ private keywordSearch; /** * Get all chunks from a namespace (placeholder implementation) */ private getAllChunks; /** * Fuse vector and keyword search results using hybrid scoring */ private fuseResults; /** * Rerank results using advanced scoring */ private rerankResults; /** * Build context from selected chunks */ private buildContext; /** * Get chunk by ID from storage */ private getChunkById; /** * Get document metadata by ID */ private getDocumentMetadata; /** * Store chunks in the vector database */ storeChunks(chunks: Chunk[], namespace?: string): Promise<void>; /** * Store chunks in D1 database for metadata queries */ private storeChunksInD1; /** * Store document metadata */ storeDocumentMetadata(metadata: DocumentMetadata): Promise<void>; /** * Search for similar chunks (for deduplication) */ findSimilarChunks(chunk: Chunk, threshold?: number, namespace?: string): Promise<Chunk[]>; /** * Delete chunks by document ID */ deleteChunksByDocument(documentId: string, namespace?: string): Promise<number>; /** * Get retrieval statistics */ getStats(): Promise<{ totalChunks: number; totalDocuments: number; namespaces: string[]; indexStats: any; }>; } /** * Answer generation service with context injection and streaming support */ declare class AnswerGenerator { private gateway; private env; private config; constructor(env: RAGEnv, config: RAGConfig); /** * Generate answer from query and context */ generateAnswer(query: string, context: ContextResult, options?: { conversationHistory?: ChatMessage[]; customPrompt?: string; includeSource?: boolean; maxTokens?: number; temperature?: number; }): Promise<QueryResponse>; /** * Generate streaming answer */ generateStreamingAnswer(query: string, context: ContextResult, options?: { conversationHistory?: ChatMessage[]; customPrompt?: string; includeSource?: boolean; maxTokens?: number; temperature?: number; onChunk?: (chunk: QueryStreamChunk) => void; }): Promise<ReadableStream<QueryStreamChunk>>; /** * Generate answer with multi-step reasoning */ generateReasonedAnswer(query: string, context: ContextResult, options?: { reasoningSteps?: number; includeSource?: boolean; }): Promise<QueryResponse & { reasoningSteps: string[]; }>; /** * Validate answer quality and detect hallucinations */ validateAnswer(answer: string, query: string, context: ContextResult): Promise<{ isValid: boolean; confidence: number; issues: string[]; suggestions: string[]; }>; /** * Build the main prompt with context injection */ private buildPrompt; /** * Build analysis prompt for multi-step reasoning */ private buildAnalysisPrompt; /** * Format context for inclusion in prompts */ private formatContext; /** * Build messages array for chat completion */ private buildMessages; /** * Build sources array for response */ private buildSources; /** * Create RAG-specific streaming response */ private createRAGStream; /** * Get fallback providers for answer generation */ private getFallbackProviders; /** * Update generation configuration */ updateConfig(newConfig: Partial<RAGConfig['generation']>): void; /** * Get generation statistics */ getStats(): { provider: AIProvider$1; model: string; temperature: number; maxTokens: number; }; } /** * Enhanced document processor with automatic format detection and processing * Supports zero-setup document ingestion from multiple sources */ declare class AutoDocumentProcessor { private env; private loader; private autoIndexingConfig; constructor(env: RAGEnv, autoIndexingConfig: AutoIndexingConfig); /** * Automatically detect and process documents from various sources */ processAutoSources(): Promise<Document[]>; /** * Process documents from a specific source */ private processSource; /** * Process documents from R2 bucket */ private processR2Bucket; /** * Process a single R2 object */ private processR2Object; /** * Process GitHub repositories */ private processGitHubRepository; /** * Process webhook queue */ private processWebhookQueue; /** * Process email queue */ private processEmailQueue; /** * Process auto-detected sources */ private processAutoDetectedSources; /** * Check if file should be processed based on patterns */ private shouldProcessFile; /** * Match filename against glob-like pattern */ private matchPattern; /** * Detect document format from filename */ private detectFormat; /** * Process PDF files using Cloudflare Workers AI */ private processPDF; /** * Process DOCX files */ private processDocx; /** * Process image files using Cloudflare Workers AI OCR */ private processImage; /** * Process audio files using Cloudflare Workers AI */ private processAudio; /** * Process video files using Cloudflare Workers AI */ private processVideo; /** * Process text-based files */ private processText; /** * Extract title from filename and content */ private extractTitle; /** * Set up continuous monitoring for auto-indexing */ setupContinuousMonitoring(): Promise<void>; /** * Process webhook payload for document updates */ processWebhookPayload(payload: any, signature?: string): Promise<Document[]>; /** * Verify webhook signature */ private verifyWebhookSignature; /** * Process file from webhook */ private processWebhookFile; /** * Get processing statistics */ getProcessingStats(): { supportedFormats: DocumentFormat[]; enabledSources: DocumentSource[]; maxFileSize: number; patternsCount: number; }; } /** * Create auto document processor with default configuration */ declare function createAutoDocumentProcessor(env: RAGEnv, config?: Partial<AutoIndexingConfig>): AutoDocumentProcessor; /** * Continuous indexing system with background monitoring * Automatically detects, processes, and indexes new documents */ declare class ContinuousIndexer { private env; private processor; private pipelines; private isRunning; private processedFiles; private lastIndexTime; constructor(env: RAGEnv, config: AutoIndexingConfig); /** * Start continuous monitoring and indexing */ start(): Promise<void>; /** * Stop continuous monitoring */ stop(): Promise<void>; /** * Register a RAG pipeline for continuous indexing */ registerPipeline(pipeline: RAGPipeline, pipelineName: string): Promise<void>; /** * Perform initial indexing of existing documents */ private performInitialIndexing; /** * Set up monitoring systems */ private setupMonitoring; /** * Set up R2 bucket monitoring */ private setupR2Monitoring; /** * Set up webhook monitoring for external document sources */ private setupWebhookMonitoring; /** * Set up file system monitoring (for local development) */ private setupFileSystemMonitoring; /** * Schedule periodic indexing checks */ private schedulePeriodicIndexing; /** * Perform incremental indexing of new/changed documents */ private performIncrementalIndexing; /** * Process a batch of documents across all registered pipelines */ private processBatch; /** * Process documents for a specific pipeline */ private processBatchForPipeline; /** * Handle webhook events for real-time indexing */ handleWebhookEvent(event: any): Promise<void>; /** * Force reindexing of all documents */ forceReindexing(): Promise<void>; /** * Get indexing status and statistics */ getStatus(): { isRunning: boolean; lastIndexTime: number; processedFilesCount: number; registeredPipelines: number; }; /** * Load previous state from storage */ private loadState; /** * Save current state to storage */ private saveState; /** * Record analytics event */ private recordAnalyticsEvent; /** * Create batches from array */ private createBatches; /** * Utility delay function */ private delay; /** * Clean up resources */ cleanup(): Promise<void>; } /** * Factory function to create continuous indexer */ declare function createContinuousIndexer(env: RAGEnv, config: AutoIndexingConfig): ContinuousIndexer; /** * Enterprise security features for RAG systems * Ensures data stays in customer's Cloudflare account with strict security controls */ declare class EnterpriseSecurityManager { private env; private securityMode; private auditLogger; private accessController; constructor(env: RAGEnv, securityMode?: 'strict' | 'relaxed'); /** * Validate document before ingestion */ validateDocumentIngestion(document: Document, userId?: string): Promise<{ allowed: boolean; reason?: string; sanitizedDocument?: Document; }>; /** * Validate query request */ validateQueryRequest(query: QueryRequest, userId?: string): Promise<{ allowed: boolean; reason?: string; sanitizedQuery?: QueryRequest; }>; /** * Filter query response based on user permissions */ filterQueryResponse(response: QueryResponse, userId?: string): Promise<QueryResponse>; /** * Encrypt document content at rest */ encryptDocument(document: Document): Promise<{ encryptedContent: string; encryptionKey: string; metadata: any; }>; /** * Decrypt document content */ decryptDocument(encryptedContent: string, encryptionKey: string, userId?: string): Promise<string>; /** * Generate secure encryption key */ private generateEncryptionKey; /** * Encrypt content using AES-256-GCM */ private encrypt; /** * Decrypt content using AES-256-GCM */ private decrypt; /** * Sanitize document content */ private sanitizeDocument; /** * Sanitize query */ private sanitizeQuery; /** * Detect sensitive data in content */ private detectSensitiveData; /** * Detect malicious query patterns */ private detectMaliciousQuery; /** * Redact sensitive information from response */ private redactSensitiveInformation; } /** * Create enterprise security manager */ declare function createEnterpriseSecurityManager(env: RAGEnv, securityMode?: 'strict' | 'relaxed'): EnterpriseSecurityManager; interface VectorSearchOptions { topK?: number; minSimilarity?: number; includeValues?: boolean; filter?: Record<string, any>; hybridWeight?: number; } /** * Cloudflare Vectorize storage implementation * Provides native vector storage with high performance and global distribution */ declare class VectorizeStorage { private env; private vectorize; private namespace; constructor(env: RAGEnv, namespace?: string); /** * Initialize vectorize index */ initialize(): Promise<void>; /** * Insert vectors into Vectorize */ insertVectors(entries: VectorEntry[]): Promise<void>; /** * Upsert vectors (insert or update) */ upsertVectors(entries: VectorEntry[]): Promise<void>; /** * Search for similar vectors */ searchSimilar(queryVector: number[], options?: VectorSearchOptions): Promise<VectorSearchResult[]>; /** * Get vector by ID */ getVector(id: VectorId): Promise<VectorEntry | null>; /** * Delete vectors by IDs */ deleteVectors(ids: VectorId[]): Promise<boolean>; /** * Get index information and statistics */ getIndexInfo(): Promise<{ name: string; dimensions: number; vectorCount: number; metric: string; }>; /** * Batch operations for better performance */ batchOperations(operations: { insert?: VectorEntry[]; update?: VectorEntry[]; delete?: VectorId[]; }): Promise<void>; /** * Hybrid search combining vector similarity and metadata filtering */ hybridSearch(queryVector: number[], textQuery?: string, options?: VectorSearchOptions): Promise<VectorSearchResult[]>; /** * Create namespaced ID to avoid conflicts */ private createNamespacedId; /** * Build filter for Vectorize query */ private buildFilter; /** * Clean metadata by removing internal fields */ private cleanMetadata; /** * Calculate simple text similarity (placeholder) */ private calculateTextSimilarity; /** * Combine vector and text similarity scores */ private combineScores; /** * Create multiple namespaces for different use cases */ createNamespace(namespace: string): Promise<VectorizeStorage>; /** * Clear all vectors in the current namespace */ clearNamespace(): Promise<void>; /** * List all vectors in namespace (for management purposes) */ private listAllVectors; /** * Performance monitoring */ getPerformanceMetrics(): Promise<{ avgQueryLatency: number; indexUtilization: number; errorRate: number; }>; } /** * Factory function to create VectorizeStorage with error handling */ declare function createVectorizeStorage(env: RAGEnv, namespace?: string): VectorizeStorage | null; /** * Generate a unique ID for documents, chunks, etc. */ declare function generateId(prefix?: string): string; /** * Sanitize and normalize text content */ declare function sanitizeText(text: string): string; /** * Extract headers from markdown or HTML content */ declare function extractHeaders(content: string): string[]; /** * Count tokens in text (rough estimate) */ declare function estimateTokens(text: string): number; /** * Create overlapping chunks from text */ declare function createOverlappingChunks(text: string, chunkSize: number, overlapSize: number, splitOnWords?: boolean): Array<{ content: string; startIndex: number; endIndex: number; }>; /** * Find sentence boundaries in text */ declare function findSentenceBoundaries(text: string): number[]; /** * Find paragraph boundaries in text */ declare function findParagraphBoundaries(text: string): number[]; /** * Calculate cosine similarity between two vectors */ declare function cosineSimilarity(a: number[], b: number[]): number; /** * Calculate text similarity using simple metrics */ declare function textSimilarity(text1: string, text2: string): number; /** * Clean and normalize query text */ declare function normalizeQuery(query: string): string; /** * Extract keywords from text */ declare function extractKeywords(text: string, minLength?: number, maxCount?: number): string[]; /** * Format time duration in human-readable format */ declare function formatDuration(milliseconds: number): string; /** * Format file size in human-readable format */ declare function formatFileSize(bytes: number): string; /** * Create a hash of content for deduplication */ declare function hashContent(content: string): string; declare function createRAGManager(env: RAGEnv): RAGManager; declare function createRAGPipeline(env: RAGEnv, config: RAGConfig): RAGPipeline; declare const DEFAULT_RAG_CONFIG: Partial<RAGConfig>; declare function quickRAG(env: RAGEnv, options: { name: string; documents: IngestionRequest[]; query: string; }): Promise<QueryResponse>; declare const VERSION = "0.1.0"; export { AnswerGenerator, type BatchProcessingResult, type Chunk, type ChunkMetadata, ChunkSchema, type ChunkingConfig, type ChunkingStrategy, type ContextResult, DEFAULT_RAG_CONFIG, type Document, DocumentChunker, type DocumentFormat, DocumentLoader, type DocumentMetadata, DocumentProcessingError, DocumentSchema, type DocumentSource, type EmbeddingConfig, EmbeddingError, EmbeddingService, GenerationError, HybridRetriever, type IngestionRequest, IngestionRequestSchema, type ProcessingResult, type QueryRequest, QueryRequestSchema, type QueryResponse, type QueryStreamChunk, type RAGAnalyticsEvent, type RAGConfig, RAGConfigSchema, type RAGEnv, RAGError, RAGManager, RAGPipeline, type RAGPipelineInfo, type RAGPipelineStatus, RetrievalError, VERSION, cosineSimilarity, createAutoDocumentProcessor, createContinuousIndexer, createEnterpriseSecurityManager, createOverlappingChunks, createRAGManager, createRAGPipeline, createVectorizeStorage, estimateTokens, extractHeaders, extractKeywords, findParagraphBoundaries, findSentenceBoundaries, formatDuration, formatFileSize, generateId, hashContent, normalizeQuery, quickRAG, sanitizeText, textSimilarity };