@escher-dbai/rag-module/src/db/VectorStore.js

Enterprise RAG module with chat context storage, vector search, and session management. Complete chat history retrieval and streaming content extraction for Electron apps.

const { QdrantClient } = require('@qdrant/js-client-rest');
const LocalFileVectorStore = require('./LocalFileVectorStore');
const EmbeddedQdrantVectorStore = require('./EmbeddedQdrantVectorStore');
const fs = require('fs-extra');
const path = require('path');
const { EventEmitter } = require('events');
const { v4: uuidv4 } = require('uuid');

/**
 * Vector Store Factory - Supports multiple backends
 * - Qdrant Server (current)
 * - Embedded Qdrant (local with HNSW performance)
 * - Local Files (pure JavaScript, no dependencies)
 */
class VectorStore extends EventEmitter {
  constructor(basePath, configManager, encryptionService) {
    super();
    
    this.basePath = basePath;
    this.configManager = configManager;
    this.encryptionService = encryptionService;
    this.dataPath = path.join(basePath, 'data');
    
    // Backend selection
    this.backend = null;
    this.backendType = null;
    
    // Qdrant server client (for server mode)
    this.client = null;
    this.dimensions = null; // Will be set from embedding service
    
    // Collection Manager for dual-collection architecture
    this.collectionManager = null;
    this.collections = {
      chat: 'chat_history',
      estate: 'aws_estate'
    };
    
    // Legacy support
    this.collectionName = 'rag-desktop'; // Deprecated, use collections instead
    
    // Configuration
    this.hnswConfig = {
      m: 16,
      efConstruction: 200,
      efSearch: 50,
      maxConnections: 16
    };
    
    this.initialized = false;
  }

  /**
   * Set vector dimensions from embedding service
   */
  setDimensions(dimensions) {
    this.dimensions = dimensions;
  }

  /**
   * Set collection manager for business architecture
   */
  setCollectionManager(collectionManager) {
    this.collectionManager = collectionManager;
  }

  /**
   * Get the underlying client for business architecture services
   */
  getClient() {
    if (this.backend && this.backend.client) {
      return this.backend.client;
    }
    if (this.client) {
      return this.client;
    }
    return this.backend; // For local files backend
  }

  /**
   * Initialize vector store with selected backend
   */
  async initialize() {
    try {
      await fs.ensureDir(this.dataPath);
      
      // Determine backend type from configuration
      const config = this.configManager.getConfig();
      this.backendType = config.vectorStore || 'qdrant';
      
      this.emit('initializing', `Initializing vector store with backend: ${this.backendType}`);
      
      switch (this.backendType) {
        case 'local-files':
          await this._initializeLocalFiles();
          break;
          
        case 'qdrant-embedded':
          await this._initializeEmbeddedQdrant();
          break;
          
        case 'qdrant':
        default:
          await this._initializeQdrantServer();
          break;
      }
      
      // Initialize dual collections for business architecture
      await this._initializeDualCollections();
      
      this.initialized = true;
      this.emit('initialized', `Vector store initialized successfully with ${this.backendType}`);
      
      return true;
    } catch (error) {
      this.emit('error', `Vector store initialization failed: ${error.message}`);
      throw error;
    }
  }

  async _initializeLocalFiles() {
    this.backend = new LocalFileVectorStore(
      this.basePath,
      this.configManager,
      this.encryptionService
    );
    
    this.backend.setDimensions(this.dimensions);
    await this.backend.initialize();
    
    // Forward events
    this.backend.on('initialized', (msg) => this.emit('connection', `Local files: ${msg}`));
    this.backend.on('error', (msg) => this.emit('error', `Local files: ${msg}`));
  }

  async _initializeEmbeddedQdrant() {
    this.backend = new EmbeddedQdrantVectorStore(
      this.basePath,
      this.configManager,
      this.encryptionService
    );
    
    this.backend.setDimensions(this.dimensions);
    await this.backend.initialize();
    
    // Forward events
    this.backend.on('initialized', (msg) => this.emit('connection', `Embedded Qdrant: ${msg}`));
    this.backend.on('error', (msg) => this.emit('error', `Embedded Qdrant: ${msg}`));
  }

  async _initializeQdrantServer() {
    // Initialize local Qdrant client (existing implementation)
    this.client = new QdrantClient({
      host: 'localhost',
      port: 6333
    });
    
    // Collection Manager is initialized by RagModule after VectorStore initialization
    
    // Legacy support: Create default collection if needed
    await this._createLegacyCollection();
  }

  /**
   * Create legacy collection for backward compatibility
   */
  async _createLegacyCollection() {
    const config = this.configManager.getConfig();
    // Only use config dimensions if not already set by embedding service
    if (this.dimensions === null) {
      this.dimensions = config.embeddingDimensions || 1024;
    }
    
    try {
      // Check if collection exists
      const collections = await this.client.getCollections();
      const exists = collections.collections?.some(c => c.name === this.collectionName);
      
      if (!exists) {
        await this.client.createCollection(this.collectionName, {
          vectors: {
            size: this.dimensions,
            distance: 'Cosine'
          },
          hnsw_config: this.hnswConfig,
          optimizers_config: {
            default_segment_number: 2,
            deleted_threshold: 0.2,
            vacuum_min_vector_number: 1000,
            flush_interval_sec: 5
          }
        });
        
        this.emit('collection-created', `Collection '${this.collectionName}' created`);
      }
    } catch (error) {
      // For in-memory implementation, create simple structure
      this.client.collections = this.client.collections || {};
      this.client.collections[this.collectionName] = {
        vectors: {},
        metadata: {},
        dimensions: this.dimensions
      };
    }
  }

  /**
   * Initialize dual collections for business architecture
   * Works with embedded backend - collections are handled internally
   */
  async _initializeDualCollections() {
    console.log('🏗️  Initializing dual collection architecture...');
    
    try {
      // For embedded backend, delegate to the backend to handle collection initialization
      if (this.backend && this.backend.initializeDualCollections) {
        await this.backend.initializeDualCollections();
      } else {
        // Fallback: Collections will be created on first use
        console.log('📁 Collections will be created automatically on first use');
      }
      
      console.log('✅ Both collections initialized successfully');
    } catch (error) {
      console.error('❌ Failed to initialize dual collections:', error.message);
      throw error;
    }
  }

  // ============ DUAL COLLECTION METHODS (Business Architecture) ============

  /**
   * Insert document to specific collection (chat or estate)
   * @param {string} collectionType - 'chat' or 'estate'
   * @param {Object} document - Document with id, content, embedding, metadata
   * @returns {Promise<string>} Document ID
   */
  async insertToCollection(collectionType, document) {
    this._ensureInitialized();
    
    const collectionName = this.collections[collectionType];
    if (!collectionName) {
      throw new Error(`Invalid collection type: ${collectionType}. Use 'chat' or 'estate'.`);
    }
    
    const { id, content, vector, embedding, payload } = document;
    
    // For chat collection, always use 1D dummy vector regardless of input
    // For estate collection, use provided embedding vector
    let documentVector;
    if (collectionType === 'chat') {
      documentVector = [0.0]; // Always use dummy vector for chat
    } else if (collectionType === 'estate') {
      documentVector = vector || embedding;
      if (!documentVector || documentVector.length !== 1024) {
        throw new Error('Estate collection requires 1024D BGE-M3 vector');
      }
    } else {
      documentVector = vector || embedding || [0.0];
    }
    
    // Insert using appropriate backend (direct client or abstraction layer)
    if (this.client && typeof this.client.upsert === 'function') {
      // Use direct Qdrant client
      await this.client.upsert(collectionName, {
        wait: true,
        points: [{
          id: id,
          vector: documentVector,
          payload: {
            content: await this.encryptionService.encrypt(content),
            ...(payload || {})
          }
        }]
      });
    } else if (this.backend && typeof this.backend.insertDocument === 'function') {
      // Use backend abstraction layer (EmbeddedQdrantVectorStore)
      await this.backend.insertDocument({
        id: id,
        content: content, // Backend will handle encryption
        embedding: documentVector,
        metadata: {
          ...payload,
          collectionType: collectionType, // Pass collection type to backend
          collection: collectionName // Pass collection name to backend
        }
      });
    } else {
      throw new Error('Neither Qdrant client nor backend available for document insertion');
    }
    
    this.emit('document-inserted', { collection: collectionName, id });
    return id;
  }

  /**
   * Search in specific collection
   * @param {string} collectionType - 'chat' or 'estate'
   * @param {number[]|null} queryVector - Query vector (null for filter-only)
   * @param {Object} options - Search options
   * @returns {Promise<Array>} Search results
   */
  async searchCollection(collectionType, queryVector, options = {}) {
    this._ensureInitialized();
    
    const collectionName = this.collections[collectionType];
    if (!collectionName) {
      throw new Error(`Invalid collection type: ${collectionType}. Use 'chat' or 'estate'.`);
    }
    
    let results;
    
    if (collectionType === 'chat' || !queryVector) {
      // Chat collection: Use scroll (filter-only, no vector search)
      results = await this.client.scroll(collectionName, {
        filter: options.filter || {},
        limit: options.limit || 100,
        with_payload: true
      });
      results = results.points || [];
    } else {
      // Estate collection: Use vector search
      const searchResult = await this.client.search(collectionName, {
        vector: queryVector,
        limit: options.limit || 10,
        filter: options.filter || {},
        with_payload: true
      });
      results = searchResult || [];
    }
    
    // Decrypt content for results
    for (const result of results) {
      if (result.payload && result.payload.content) {
        try {
          result.payload.content = await this.encryptionService.decrypt(result.payload.content);
        } catch (error) {
          console.warn(`Failed to decrypt content for document ${result.id}:`, error.message);
        }
      }
    }
    
    return results;
  }

  /**
   * Scroll through collection using filters (business architecture method)
   * @param {string} collectionType - 'chat' or 'estate' 
   * @param {Object} options - Scroll options
   * @returns {Promise<Array>} Filtered results
   */
  async scrollCollection(collectionType, options = {}) {
    this._ensureInitialized();

    const collectionName = collectionType === 'chat' ? this.collections.chat : this.collections.estate;

    // Handle embedded backend vs. server backend
    if (this.backend) {
      // For embedded backend, delegate to backend search method
      try {
        if (collectionType === 'chat') {
          // For chat collection - use backend search with dummy vector and filter options
          const dummyVector = [1.0]; // 1D dummy vector for chat
          const results = await this.backend.search(dummyVector, {
            collectionName: collectionName,
            limit: options.limit || 100,
            scoreThreshold: 0.0, // Accept all scores since we're using dummy vector
            filter: options.filter || {}
          });

          // Convert backend results to expected format
          // For embedded backend, we need to get the original document to access plain metadata fields
          return results.map(result => {
            // Get the stored document to access both encrypted and plain metadata
            const collectionDocs = this.backend.documents.get(collectionName);
            const originalDoc = collectionDocs ? collectionDocs.get(result.id) : null;

            return {
              id: result.id,
              payload: {
                content: result.content,
                ...result.metadata, // Decrypted metadata
                // Also include any plain metadata fields from the original document
                ...(originalDoc?.metadata || {})
              },
              score: result.score
            };
          });
        } else {
          // For estate collection - use search with actual vectors (this shouldn't be called for chat history)
          throw new Error('Estate collection scroll not supported - use vector search instead');
        }
      } catch (error) {
        console.error(`❌ Embedded backend scroll collection '${collectionName}' failed:`, error.message);
        throw error;
      }
    } else if (this.client) {
      // For server backend, use client directly
      try {
        let results = [];

        if (collectionType === 'chat') {
          // For chat collection - use search with dummy vector since it uses 1D dummy vectors
          const dummyVector = [1.0]; // 1D dummy vector
          const searchResult = await this.client.search(collectionName, {
            vector: dummyVector,
            limit: options.limit || 100,
            filter: options.filter || {},
            with_payload: options.withPayload !== false
          });
          results = searchResult || [];
        } else {
          // For estate collection - use search with actual vectors (this shouldn't be called for chat history)
          throw new Error('Estate collection scroll not supported - use vector search instead');
        }

        // Decrypt content for results if needed
        if (options.withPayload !== false) {
          for (const result of results) {
            if (result.payload && result.payload.content) {
              try {
                result.payload.content = await this.encryptionService.decrypt(result.payload.content);
              } catch (error) {
                console.warn(`Failed to decrypt content for document ${result.id}:`, error.message);
              }
            }
          }
        }

        return results;
      } catch (error) {
        console.error(`❌ Server backend scroll collection '${collectionName}' failed:`, error.message);
        throw error;
      }
    } else {
      throw new Error('Neither Qdrant client nor backend properly initialized');
    }
  }

  /**
   * Get collection health information
   * @returns {Promise<Object>} Collection health data
   */
  async getCollectionsHealth() {
    this._ensureInitialized();
    
    if (this.collectionManager) {
      return await this.collectionManager.getCollectionsHealth();
    }
    
    // Fallback for non-collection-manager setups
    return {
      chat_history: { status: 'unknown', points_count: 0 },
      aws_estate: { status: 'unknown', points_count: 0 }
    };
  }

  /**
   * Get collection names for easy access
   * @returns {Object} Collection name mappings
   */
  getCollectionNames() {
    return this.collections;
  }

  // ============ LEGACY METHODS (Backward Compatibility) ============

  /**
   * Insert encrypted document
   * @param {Object} document - Document to insert
   * @param {string} document.id - Document ID
   * @param {string} document.content - Document content
   * @param {number[]} document.embedding - Embedding vector
   * @param {Object} document.metadata - Document metadata
   * @returns {Promise<string>} - Inserted document ID
   */
  async insertDocument(document) {
    this._ensureInitialized();
    
    // Delegate to selected backend
    if (this.backend) {
      return await this.backend.insertDocument(document);
    }
    
    // Qdrant server implementation continues below
    const { id, content, embedding, metadata } = document;
    
    if (!id || !content || !embedding) {
      throw new Error('Document must have id, content, and embedding');
    }
    
    if (!Array.isArray(embedding)) {
      throw new Error('Embedding must be an array');
    }
    
    if (embedding.length !== this.dimensions) {
      throw new Error(`Embedding dimensions mismatch: expected ${this.dimensions}, got ${embedding.length}`);
    }
    
    try {
      // Generate unique vector ID
      const vectorId = uuidv4();
      
      // Encrypt content and metadata
      const encryptedContent = await this.encryptionService.encryptContent(content);
      const encryptedMetadata = await this.encryptionService.encryptContent(JSON.stringify(metadata || {}));
      
      // Encrypt embeddings (store original format for encryption)
      const encryptedEmbedding = await this.encryptionService.encryptEmbedding(
        Array.isArray(embedding) ? embedding : denseVector
      );
      
      // Prepare point for insertion with hybrid vectors
      const point = {
        id: vectorId,
        payload: {
          document_id: id,
          content_encrypted: encryptedContent,
          metadata_encrypted: encryptedMetadata,
          embedding_encrypted: encryptedEmbedding,
          original_id: id,
          created_at: new Date().toISOString(),
          encryption_version: '1.0'
        }
      };
      
      // Set vector directly (original format)
      point.vector = embedding;
      
      // Insert into collection with correct API format
      try {
        await this.client.upsert(this.collectionName, {
          wait: true,
          points: [point]
        });
        console.log(`✅ Successfully inserted document`);
      } catch (qdrantError) {
        console.log(`❌ Qdrant upsert error:`, qdrantError.message);
        throw new Error(`Failed to insert document: ${qdrantError.message}`);
      }
      
      this.emit('document-inserted', { documentId: id, vectorId });
      return vectorId;
    } catch (error) {
      this.emit('error', `Failed to insert document: ${error.message}`);
      throw error;
    }
  }

  /**
   * Search for similar vectors
   * @param {number[]} queryEmbedding - Query embedding vector
   * @param {Object} options - Search options
   * @param {number} [options.limit=10] - Maximum results
   * @param {number} [options.scoreThreshold=0.5] - Minimum similarity score
   * @param {Object} [options.filter] - Metadata filters
   * @returns {Promise<Array>} - Search results
   */
  async search(queryEmbedding, options = {}) {
    this._ensureInitialized();
    
    // Delegate to selected backend if it has search method
    if (this.backend && this.backend.search) {
      return await this.backend.search(queryEmbedding, options);
    }
    
    // For embedded backend or fallback, return empty results for now
    // Chat messages don't need vector search anyway
    if (this.backend && !this.backend.search) {
      console.log('📝 Backend search not available, returning empty results for legacy search');
      return [];
    }
    
    // Qdrant server implementation continues below
    const { limit = 10, scoreThreshold = 0.5, filter = {} } = options;
    
    if (!Array.isArray(queryEmbedding)) {
      throw new Error('Query embedding must be an array');
    }
    
    if (queryEmbedding.length !== this.dimensions) {
      throw new Error(`Query embedding dimensions mismatch: expected ${this.dimensions}, got ${queryEmbedding.length}`);
    }
    
    try {
      let results = [];
      
      if (this.client.search) {
        const searchResult = await this.client.search(this.collectionName, {
          vector: queryEmbedding,
          limit: limit * 2, // Get more to filter
          score_threshold: scoreThreshold,
          with_payload: true,
          with_vector: false
        });
        
        results = searchResult.map(point => ({
          id: point.id,
          score: point.score,
          payload: point.payload
        }));
      } else {
        // Fallback: manual cosine similarity search (use dense vector only for backward compatibility)
        results = await this._manualSearch(denseQuery || queryEmbedding, { limit, scoreThreshold, filter });
      }
      
      // Decrypt and process results
      const decryptedResults = [];
      for (const result of results.slice(0, limit)) {
        try {
          const decryptedContent = await this.encryptionService.decryptContent(result.payload.content_encrypted);
          const decryptedMetadata = JSON.parse(await this.encryptionService.decryptContent(result.payload.metadata_encrypted));
          
          // Apply metadata filters
          if (this._matchesFilter(decryptedMetadata, filter)) {
            decryptedResults.push({
              id: result.payload.document_id,
              content: decryptedContent,
              metadata: decryptedMetadata,
              score: result.score,
              vectorId: result.id
            });
          }
        } catch (decryptError) {
          this.emit('warning', `Failed to decrypt result: ${decryptError.message}`);
        }
      }
      
      this.emit('search-completed', { query: 'vector', results: decryptedResults.length });
      return decryptedResults;
    } catch (error) {
      this.emit('error', `Search failed: ${error.message}`);
      throw error;
    }
  }

  /**
   * Get document by ID
   * @param {string} documentId - Document ID
   * @returns {Promise<Object|null>} - Document or null if not found
   */
  async getById(documentId) {
    this._ensureInitialized();
    
    // Delegate to selected backend
    if (this.backend) {
      return await this.backend.getById(documentId);
    }
    
    // Qdrant server implementation continues below
    try {
      // Search by document_id in payload
      const filter = {
        must: [{
          key: 'document_id',
          match: { value: documentId }
        }]
      };
      
      let results = [];
      
      if (this.client.scroll) {
        const scrollResult = await this.client.scroll(this.collectionName, {
          filter,
          limit: 1,
          with_payload: true,
          with_vector: false
        });
        
        results = scrollResult.points || [];
      } else {
        // Manual search in collections (fallback only)
        if (this.client.collections && this.client.collections[this.collectionName]) {
          const collection = this.client.collections[this.collectionName];
          if (collection.vectors) {
            for (const [vectorId, point] of Object.entries(collection.vectors)) {
              if (point.payload.document_id === documentId) {
                results.push({ id: vectorId, payload: point.payload });
                break;
              }
            }
          }
        }
        // If no fallback collections, return empty (document not found)
      }
      
      if (results.length === 0) {
        return null;
      }
      
      const point = results[0];
      const decryptedContent = await this.encryptionService.decryptContent(point.payload.content_encrypted);
      const decryptedMetadata = JSON.parse(await this.encryptionService.decryptContent(point.payload.metadata_encrypted));
      
      return {
        id: documentId,
        content: decryptedContent,
        metadata: decryptedMetadata,
        createdAt: new Date(point.payload.created_at),
        vectorId: point.id
      };
    } catch (error) {
      this.emit('error', `Failed to get document ${documentId}: ${error.message}`);
      return null;
    }
  }

  /**
   * Update document content and/or metadata
   * @param {string} documentId - Document ID
   * @param {Object} updates - Updates to apply
   * @param {string} [updates.content] - New content
   * @param {Object} [updates.metadata] - New metadata
   * @param {number[]} [updates.embedding] - New embedding
   * @returns {Promise<boolean>}
   */
  async updateDocument(documentId, updates) {
    this._ensureInitialized();
    
    // Delegate to selected backend
    if (this.backend) {
      return await this.backend.updateDocument(documentId, updates);
    }
    
    // Qdrant server implementation continues below
    try {
      // First, get the existing document
      const existing = await this.getById(documentId);
      if (!existing) {
        return false;
      }
      
      // Prepare updated data
      const updatedContent = updates.content !== undefined ? updates.content : existing.content;
      const updatedMetadata = updates.metadata !== undefined 
        ? { ...existing.metadata, ...updates.metadata }
        : existing.metadata;
      
      // Encrypt updated data
      const encryptedContent = await this.encryptionService.encryptContent(updatedContent);
      const encryptedMetadata = await this.encryptionService.encryptContent(JSON.stringify(updatedMetadata));
      
      // Update point
      const updatePayload = {
        content_encrypted: encryptedContent,
        metadata_encrypted: encryptedMetadata,
        updated_at: new Date().toISOString()
      };
      
      // Handle embedding update
      if (updates.embedding) {
        const encryptedEmbedding = await this.encryptionService.encryptEmbedding(updates.embedding);
        updatePayload.embedding_encrypted = encryptedEmbedding;
      }
      
      // If we need to update embedding, use upsert
      if (updates.embedding) {
        const point = {
          id: existing.vectorId,
          vector: updates.embedding,
          payload: {
            document_id: documentId,
            ...existing.payload,
            ...updatePayload
          }
        };
        
        await this.client.upsert(this.collectionName, {
          wait: true,
          points: [point]
        });
      } else {
        // Just update payload using setPayload (more efficient for metadata-only updates)
        await this.client.setPayload(this.collectionName, {
          payload: updatePayload,
          points: [existing.vectorId],
          wait: true
        });
      }
      
      this.emit('document-updated', { documentId });
      return true;
    } catch (error) {
      this.emit('error', `Failed to update document ${documentId}: ${error.message}`);
      return false;
    }
  }

  /**
   * Delete document by ID
   * @param {string} documentId - Document ID
   * @returns {Promise<boolean>}
   */
  async deleteDocument(documentId) {
    this._ensureInitialized();
    
    // Delegate to selected backend
    if (this.backend) {
      return await this.backend.deleteDocument(documentId);
    }
    
    // Qdrant server implementation continues below
    try {
      // Find ALL instances of this document ID (in case of duplicates)
      const allResults = await this.listDocuments({ limit: 1000 });
      const matchingDocs = allResults.filter(doc => doc.id === documentId);
      
      if (matchingDocs.length === 0) {
        return false;
      }
      
      // Delete all matching documents
      const vectorIds = matchingDocs.map(doc => doc.vectorId).filter(Boolean);
      if (vectorIds.length > 0) {
        await this.client.delete(this.collectionName, {
          wait: true,
          points: vectorIds
        });
      }
      
      this.emit('document-deleted', { documentId, count: vectorIds.length });
      return true;
    } catch (error) {
      console.error(`❌ Delete error for ${documentId}:`, error);
      this.emit('error', `Failed to delete document ${documentId}: ${error.message}`);
      return false;
    }
  }

  /**
   * List documents using scroll API
   * @param {Object} options - List options
   * @returns {Promise<Array>} - List of documents
   */
  async listDocuments(options = {}) {
    this._ensureInitialized();
    
    // Delegate to selected backend
    if (this.backend) {
      return await this.backend.listDocuments(options);
    }
    
    // Qdrant server implementation continues below
    const { limit = 50, filter = {}, offset = 0 } = options;
    
    try {
      let results = [];
      
      if (this.client.scroll) {
        const scrollResult = await this.client.scroll(this.collectionName, {
          limit: Math.max(limit + offset, 100), // Use at least 100 to get reasonable batch size
          with_payload: true,
          with_vector: false,
          filter: this._buildQdrantFilter(filter)
        });
        
        results = scrollResult.points || [];
      } else {
        // Manual list for in-memory (fallback only)
        if (this.client.collections && this.client.collections[this.collectionName]) {
          const collection = this.client.collections[this.collectionName];
          if (collection.vectors) {
            results = Object.entries(collection.vectors).map(([id, point]) => ({
              id,
              payload: point.payload
            }));
          }
        }
      }
      
      // Decrypt and process results
      const decryptedResults = [];
      for (const result of results.slice(offset, offset + limit)) {
        try {
          const decryptedContent = await this.encryptionService.decryptContent(result.payload.content_encrypted);
          const decryptedMetadata = JSON.parse(await this.encryptionService.decryptContent(result.payload.metadata_encrypted));
          
          decryptedResults.push({
            id: result.payload.document_id,
            content: decryptedContent,
            metadata: decryptedMetadata,
            vectorId: result.id
          });
        } catch (decryptError) {
          this.emit('warning', `Failed to decrypt result: ${decryptError.message}`);
        }
      }
      
      this.emit('documents-listed', { count: decryptedResults.length });
      return decryptedResults;
    } catch (error) {
      this.emit('error', `List documents failed: ${error.message}`);
      throw error;
    }
  }

  /**
   * Build Qdrant filter from metadata filter
   */
  _buildQdrantFilter(filter) {
    if (!filter || Object.keys(filter).length === 0) {
      return undefined;
    }
    
    const must = [];
    
    for (const [key, value] of Object.entries(filter)) {
      if (value !== undefined && value !== null) {
        must.push({
          key,
          match: { value }
        });
      }
    }
    
    return must.length > 0 ? { must } : undefined;
  }

  /**
   * Get collection statistics
   * @returns {Promise<Object>}
   */
  async getStats() {
    this._ensureInitialized();
    
    // Delegate to selected backend
    if (this.backend) {
      return await this.backend.getStats();
    }
    
    // Qdrant server implementation continues below
    try {
      if (this.client.getCollection) {
        const info = await this.client.getCollection(this.collectionName);
        return {
          totalPoints: info.points_count || 0,
          dimensions: this.dimensions,
          collectionName: this.collectionName
        };
      } else {
        // Manual count for in-memory (fallback only)
        if (this.client.collections && this.client.collections[this.collectionName]) {
          const collection = this.client.collections[this.collectionName];
          return {
            totalPoints: collection.vectors ? Object.keys(collection.vectors).length : 0,
            dimensions: this.dimensions,
            collectionName: this.collectionName
          };
        } else {
          return {
            totalPoints: 0,
            dimensions: this.dimensions,
            collectionName: this.collectionName
          };
        }
      }
    } catch (error) {
      return {
        totalPoints: 0,
        dimensions: this.dimensions,
        collectionName: this.collectionName,
        error: error.message
      };
    }
  }

  /**
   * Close vector store connection
   */
  async close() {
    if (this.client && this.client.close) {
      await this.client.close();
    }
    
    this.emit('closed', 'Vector store closed');
  }

  // ============ PRIVATE METHODS ============

  /**
   * Manual cosine similarity search for fallback
   */
  async _manualSearch(queryEmbedding, options) {
    const { limit, scoreThreshold } = options;
    
    if (!this.client.collections || !this.client.collections[this.collectionName]) {
      return [];
    }
    
    const collection = this.client.collections[this.collectionName];
    if (!collection.vectors) {
      return [];
    }
    
    const results = [];
    
    for (const [vectorId, point] of Object.entries(collection.vectors)) {
      const similarity = this._cosineSimilarity(queryEmbedding, point.vector);
      
      if (similarity >= scoreThreshold) {
        results.push({
          id: vectorId,
          score: similarity,
          payload: point.payload
        });
      }
    }
    
    // Sort by score descending and limit
    results.sort((a, b) => b.score - a.score);
    return results.slice(0, limit);
  }

  /**
   * Calculate cosine similarity
   */
  _cosineSimilarity(vec1, vec2) {
    let dotProduct = 0;
    let norm1 = 0;
    let norm2 = 0;
    
    for (let i = 0; i < vec1.length; i++) {
      dotProduct += vec1[i] * vec2[i];
      norm1 += vec1[i] * vec1[i];
      norm2 += vec2[i] * vec2[i];
    }
    
    return dotProduct / (Math.sqrt(norm1) * Math.sqrt(norm2));
  }

  /**
   * Check if metadata matches filter
   */
  _matchesFilter(metadata, filter) {
    for (const [key, value] of Object.entries(filter)) {
      if (Array.isArray(value)) {
        if (!value.includes(metadata[key])) {
          return false;
        }
      } else if (metadata[key] !== value) {
        return false;
      }
    }
    return true;
  }

  _ensureInitialized() {
    if (!this.initialized) {
      throw new Error('Vector store must be initialized before use');
    }
  }
}

module.exports = VectorStore;