@krunal_tarale-5/ultimate-streaming-package/lib/advancedMongoConnector.js

🚀 Ultimate Real-Time Streaming Package v2.1.9 - Multi-Platform, Multi-Collection Architecture with Native MongoDB & MySQL Support, 99.96% Performance Improvement. Enterprise-grade real-time data streaming with Socket.IO integration, dynamic schema evolut

const { MongoClient } = require('mongodb');
const EventEmitter = require('events');

class AdvancedMongoConnector extends EventEmitter {
  constructor() {
    super();
    this.client = null;
    this.db = null;
    this.connected = false;
    this.config = null;
    this.activeWatchers = new Map(); // collection -> { callbacks: Set, options }
    this.changeStreams = new Map(); // collection -> changeStream
    this.lastKnownState = new Map(); // collection -> lastData
    this.metrics = {
      connections: 0,
      writesHandled: 0,
      readsHandled: 0,
      changesProcessed: 0,
      errorsHandled: 0,
      activeWatchers: 0,
      activeStreams: 0
    };
    this.healthCheckInterval = null;
    this.reconnectAttempts = 0;
    this.maxReconnectAttempts = 10;
    this.collectionsWithIndexes = new Set(); // Track collections with indexes
  }

  async connect(config) {
    if (this.connected) {
      console.log('MongoDB already connected');
      return;
    }

    this.config = config;
    const connectionString = this.buildConnectionString(config);

    try {
      console.log('🔄 Connecting to MongoDB...');
      
      this.client = new MongoClient(connectionString, {
        maxPoolSize: config.maxConnections || 50,
        serverSelectionTimeoutMS: 5000,
        socketTimeoutMS: 45000,
        useUnifiedTopology: true
      });

      await this.client.connect();
      this.db = this.client.db(config.database);
      this.connected = true;
      this.reconnectAttempts = 0;

      console.log('✅ MongoDB connected successfully');
      
      // Set up connection monitoring
      this.setupConnectionMonitoring();
      
      // Start health checks
      this.startHealthChecks();
      
      this.emit('connected');
      
    } catch (error) {
      console.error('❌ MongoDB connection failed:', error.message);
      this.emit('error', error);
      throw error;
    }
  }

  buildConnectionString(config) {
    const { host, port, user, password, database } = config;
    
    if (user && password) {
      return `mongodb://${user}:${password}@${host}:${port}/${database}?authSource=admin`;
    } else {
      return `mongodb://${host}:${port}/${database}`;
    }
  }

  setupConnectionMonitoring() {
    this.client.on('serverHeartbeatSucceeded', () => {
      this.metrics.connections++;
    });

    this.client.on('serverHeartbeatFailed', async (error) => {
      console.error('MongoDB heartbeat failed:', error);
      await this.handleConnectionError(error);
    });

    this.client.on('close', async () => {
      console.log('MongoDB connection closed');
      this.connected = false;
      await this.handleDisconnection();
    });
  }

  async handleConnectionError(error) {
    this.metrics.errorsHandled++;
    this.emit('error', error);
    
    if (this.reconnectAttempts < this.maxReconnectAttempts) {
      this.reconnectAttempts++;
      console.log(`🔄 Attempting to reconnect (${this.reconnectAttempts}/${this.maxReconnectAttempts})...`);
      
      setTimeout(async () => {
        try {
          await this.connect(this.config);
          await this.restoreWatchers();
        } catch (reconnectError) {
          console.error('Reconnection failed:', reconnectError.message);
        }
      }, 1000 * this.reconnectAttempts);
    } else {
      console.error('Max reconnection attempts reached');
      this.emit('maxReconnectAttemptsReached');
    }
  }

  async handleDisconnection() {
    this.connected = false;
    this.emit('disconnected');
    
    // Clear change streams
    for (const [collection, changeStream] of this.changeStreams) {
      try {
        await changeStream.close();
      } catch (error) {
        console.error(`Error closing change stream for ${collection}:`, error);
      }
    }
    this.changeStreams.clear();
  }

  async restoreWatchers() {
    console.log('🔄 Restoring active watchers...');
    
    for (const [collection, watcherInfo] of this.activeWatchers) {
      try {
        await this.startRealTimeWatch(collection, watcherInfo.callback, watcherInfo.options);
        console.log(`✅ Restored watcher for collection: ${collection}`);
      } catch (error) {
        console.error(`❌ Failed to restore watcher for collection ${collection}:`, error);
      }
    }
  }

  startHealthChecks() {
    if (this.healthCheckInterval) {
      clearInterval(this.healthCheckInterval);
    }

    this.healthCheckInterval = setInterval(async () => {
      try {
        await this.db.admin().ping();
        this.emit('healthCheck', { status: 'healthy', metrics: this.getMetrics() });
      } catch (error) {
        console.error('Health check failed:', error);
        this.emit('healthCheck', { status: 'unhealthy', error: error.message });
      }
    }, 30000); // Every 30 seconds
  }

  // 🆕 NEW: Write data to specific collection
  async writeData(collection, data, options = {}) {
    if (!this.connected) {
      throw new Error('MongoDB not connected');
    }

    try {
      const coll = this.db.collection(collection);
      const document = {
        ...data,
        _createdAt: new Date(),
        _updatedAt: new Date(),
        _ttl: options.ttl ? new Date(Date.now() + options.ttl * 1000) : null,
        _tags: options.tags || [],
        _metadata: options.metadata || {}
      };

      // Create indexes automatically for new collections
      if (!this.collectionsWithIndexes.has(collection)) {
        try {
          await this.setupOptimizedIndexes(collection);
          this.collectionsWithIndexes.add(collection);
        } catch (error) {
          console.log(`Index creation for ${collection} will be retried later`);
        }
      }

      // Use transactions for consistency (only if supported)
      const useTransactions = this.config.useChangeStreams !== false;
      let result;
      
      if (useTransactions) {
        try {
          const session = this.client.startSession();
          try {
            await session.withTransaction(async () => {
              result = await coll.insertOne(document, { session });
            });
          } finally {
            await session.endSession();
          }
        } catch (error) {
          // Fallback to non-transactional operation if transactions not supported
          if (error.code === 20) {
            console.log('Transactions not supported, using non-transactional operation');
            result = await coll.insertOne(document);
          } else {
            throw error;
          }
        }
      } else {
        // Non-transactional operation for local MongoDB
        result = await coll.insertOne(document);
      }

      this.metrics.writesHandled++;
      
      return {
        success: true,
        collection: collection,
        insertedId: result.insertedId,
        timestamp: document._createdAt
      };
    } catch (error) {
      console.error('MongoDB write error:', error.message);
      this.metrics.errorsHandled++;
      throw error;
    }
  }

  // 🆕 NEW: Update data in specific collection with query
  async updateData(collection, query, updateData, options = {}) {
    if (!this.connected) {
      throw new Error('MongoDB not connected');
    }

    try {
      const coll = this.db.collection(collection);
      
      // Prepare update document
      const updateDocument = {
        $set: {
          ...updateData,
          _updatedAt: new Date(),
          _ttl: options.ttl ? new Date(Date.now() + options.ttl * 1000) : null,
          _tags: options.tags || [],
          _metadata: options.metadata || {}
        }
      };

      // Use transactions for consistency (only if supported)
      const useTransactions = this.config.useChangeStreams !== false;
      let result;
      
      if (useTransactions) {
        try {
          const session = this.client.startSession();
          try {
            await session.withTransaction(async () => {
              result = await coll.updateOne(
                query,
                updateDocument,
                { session }
              );
            });
          } finally {
            await session.endSession();
          }
        } catch (error) {
          // Fallback to non-transactional update
          console.log('Transaction failed, falling back to regular update');
          result = await coll.updateOne(
            query,
            updateDocument
          );
        }
      } else {
        result = await coll.updateOne(
          query,
          updateDocument
        );
      }

      this.metrics.writesHandled++;
      
      return {
        success: true,
        collection: collection,
        found: result.matchedCount > 0,
        updated: result.modifiedCount > 0,
        timestamp: new Date(),
        query: query
      };
    } catch (error) {
      console.error('MongoDB update error:', error.message);
      this.metrics.errorsHandled++;
      throw error;
    }
  }

  // 🆕 NEW: Read data from specific collection
  async readData(collection, query = {}, options = {}) {
    if (!this.connected) {
      throw new Error('MongoDB not connected');
    }

    try {
      const coll = this.db.collection(collection);
      const result = await coll.findOne(
        query,
        { 
          readPreference: 'primaryPreferred',
          ...options
        }
      );
      
      if (!result) return null;

      // Check TTL
      if (result._ttl && new Date() > result._ttl) {
        await this.deleteData(collection, query);
        return null;
      }

      return {
        collection: collection,
        data: result,
        timestamp: result._createdAt,
        lastModified: result._updatedAt,
        tags: result._tags || [],
        metadata: result._metadata || {}
      };
    } catch (error) {
      console.error('MongoDB read error:', error.message);
      throw error;
    }
  }

  // 🆕 NEW: Start real-time watch for specific collection
  async startRealTimeWatch(collection, callback, options = {}) {
    if (!this.connected) {
      throw new Error('MongoDB not connected');
    }

    try {
      // Store watcher info for reconnection
      this.activeWatchers.set(collection, { callback, options });

      // Check if change streams are enabled in config
      const useChangeStreams = this.config.useChangeStreams !== false;
      
      if (useChangeStreams) {
        return await this.startChangeStreamWatch(collection, callback, options);
      } else {
        return await this.startPollingWatch(collection, callback, options);
      }

    } catch (error) {
      console.error(`Failed to start real-time watch for collection ${collection}:`, error);
      this.activeWatchers.delete(collection);
      
      // If change streams fail, fallback to polling
      if (this.config.useChangeStreams !== false) {
        console.log(`Falling back to polling for collection: ${collection}`);
        return await this.startPollingWatch(collection, callback, options);
      }
      
      throw error;
    }
  }

  async startChangeStreamWatch(collection, callback, options = {}) {
    const coll = this.db.collection(collection);
    
    // Create change stream with advanced options
    const pipeline = [
      {
        $match: {
          operationType: { $in: ['insert', 'update', 'replace', 'delete'] }
        }
      }
    ];

    const changeStreamOptions = {
      fullDocument: 'updateLookup',
      resumeAfter: options.resumeToken,
      startAtOperationTime: options.startAtOperationTime,
      maxAwaitTimeMS: 1000,
      batchSize: 1
    };

    const changeStream = coll.watch(pipeline, changeStreamOptions);
    this.changeStreams.set(collection, changeStream);
    this.metrics.activeStreams++;

    // Handle change events
    changeStream.on('change', (change) => {
      try {
        this.metrics.changesProcessed++;
        const changeType = this.mapOperationType(change.operationType);
        const data = change.fullDocument || null;
        
        // Create metadata object
        const meta = {
          collection: collection,
          changeType: changeType,
          timestamp: new Date(),
          operationType: change.operationType,
          resumeToken: change._id,
          clusterTime: change.clusterTime,
          txnNumber: change.txnNumber,
          lsid: change.lsid,
          source: 'changeStream'
        };

        callback(data, meta);
        this.emit('change', { collection, data, meta });
      } catch (error) {
        console.error(`Error processing change for collection ${collection}:`, error);
        this.emit('error', error);
      }
    });

    // Handle change stream errors
    changeStream.on('error', (error) => {
      console.error(`Change stream error for collection ${collection}:`, error);
      this.metrics.errorsHandled++;
      
      // If change streams aren't supported, fallback to polling
      if (error.code === 40573) {
        console.log(`Change streams not supported, falling back to polling for collection: ${collection}`);
        this.changeStreams.delete(collection);
        this.startPollingWatch(collection, callback, options);
        return;
      }
        
      // Attempt to restart the change stream
      setTimeout(() => {
        this.restartChangeStream(collection);
      }, 1000);
    });

    changeStream.on('close', () => {
      console.log(`Change stream closed for collection: ${collection}`);
      this.changeStreams.delete(collection);
      this.metrics.activeStreams--;
    });

    // Get initial data
    const initialData = await this.readData(collection);
    if (initialData) {
      const meta = {
        collection: collection,
        changeType: 'initial',
        timestamp: new Date(),
        source: 'changeStream'
      };
      callback(initialData.data, meta);
    }

    console.log(`Real-time change stream started for collection: ${collection}`);
    return changeStream;
  }

  async startPollingWatch(collection, callback, options = {}) {
    const pollingInterval = this.config.pollingInterval || 2000;
    let lastModified = null;
    
    // Store polling timer
    const pollTimer = setInterval(async () => {
      try {
        const currentData = await this.readData(collection);
        
        if (!currentData) {
          // Data was deleted
          if (lastModified !== null) {
            const meta = {
              collection: collection,
              changeType: 'deleted',
              timestamp: new Date(),
              source: 'polling'
            };
            callback(null, meta);
            this.emit('change', { collection, data: null, meta });
            lastModified = null;
          }
          return;
        }
        
        const currentModified = currentData.lastModified?.getTime();
        
        if (lastModified === null) {
          // Initial data
          const meta = {
            collection: collection,
            changeType: 'initial',
            timestamp: new Date(),
            source: 'polling'
          };
          callback(currentData.data, meta);
          this.emit('change', { collection, data: currentData.data, meta });
          lastModified = currentModified;
        } else if (currentModified > lastModified) {
          // Data changed
          const meta = {
            collection: collection,
            changeType: 'updated',
            timestamp: new Date(),
            source: 'polling'
          };
          callback(currentData.data, meta);
          this.emit('change', { collection, data: currentData.data, meta });
          lastModified = currentModified;
        }
      } catch (error) {
        console.error(`Error in polling for collection ${collection}:`, error);
      }
    }, pollingInterval);

    // Store timer for cleanup
    this.activeWatchers.set(collection, { 
      callback, 
      options, 
      timer: pollTimer 
    });

    console.log(`Polling started for collection: ${collection} (${pollingInterval}ms)`);
    return { collection, timer: pollTimer };
  }

  async restartChangeStream(collection) {
    const watcherInfo = this.activeWatchers.get(collection);
    if (watcherInfo) {
      try {
        await this.startChangeStreamWatch(collection, watcherInfo.callback, watcherInfo.options);
        console.log(`✅ Restarted change stream for collection: ${collection}`);
      } catch (error) {
        console.error(`❌ Failed to restart change stream for collection ${collection}:`, error);
      }
    }
  }

  mapOperationType(operationType) {
    const mapping = {
      'insert': 'created',
      'update': 'updated',
      'replace': 'updated',
      'delete': 'deleted',
      'drop': 'deleted',
      'rename': 'updated',
      'dropDatabase': 'deleted'
    };
    return mapping[operationType] || 'updated';
  }

  async stopWatch(collection) {
    // Stop change stream
    const changeStream = this.changeStreams.get(collection);
    if (changeStream) {
      try {
        await changeStream.close();
        this.changeStreams.delete(collection);
        this.metrics.activeStreams--;
      } catch (error) {
        console.error(`Error closing change stream for collection ${collection}:`, error);
      }
    }

    // Stop polling timer
    const watcherInfo = this.activeWatchers.get(collection);
    if (watcherInfo && watcherInfo.timer) {
      clearInterval(watcherInfo.timer);
    }

    this.activeWatchers.delete(collection);
    this.lastKnownState.delete(collection);
    this.metrics.activeWatchers--;

    console.log(`Stopped watching collection: ${collection}`);
  }

  // 🆕 NEW: Query data from specific collection
  async queryData(collection, query = {}, options = {}) {
    if (!this.connected) {
      throw new Error('MongoDB not connected');
    }

    try {
      const coll = this.db.collection(collection);
      
      // Build MongoDB query
      const mongoQuery = this.buildMongoQuery(query);
      const projection = options.fields ? this.buildProjection(options.fields) : {};
      
      const cursor = coll.find(mongoQuery, {
        projection,
        sort: options.sort || { _createdAt: -1 },
        limit: options.limit || 100,
        skip: options.skip || 0
      });

      const results = await cursor.toArray();
      
      return results.map(doc => ({
        collection: collection,
        data: doc,
        timestamp: doc._createdAt,
        lastModified: doc._updatedAt,
        tags: doc._tags || [],
        metadata: doc._metadata || {}
      }));
    } catch (error) {
      console.error('MongoDB query error:', error.message);
      throw error;
    }
  }

  buildMongoQuery(query) {
    const mongoQuery = {};
    
    // Handle different query types
    if (query.where) {
      Object.assign(mongoQuery, query.where);
    }
    
    if (query.text) {
      mongoQuery.$text = { $search: query.text };
    }
    
    if (query.geoNear) {
      // Handle geospatial queries
      return {
        $geoNear: {
          near: query.geoNear.near,
          distanceField: query.geoNear.distanceField,
          maxDistance: query.geoNear.maxDistance,
          spherical: true
        }
      };
    }
    
    return mongoQuery;
  }

  buildProjection(fields) {
    const projection = {};
    
    if (Array.isArray(fields)) {
      fields.forEach(field => {
        projection[field] = 1;
      });
    } else if (typeof fields === 'object') {
      Object.assign(projection, fields);
    }
    
    return projection;
  }

  // 🆕 NEW: Get all collections
  async getAllCollections() {
    if (!this.connected) {
      throw new Error('MongoDB not connected');
    }

    try {
      const collections = await this.db.listCollections().toArray();
      return collections.map(col => col.name);
    } catch (error) {
      console.error('MongoDB get collections error:', error.message);
      throw error;
    }
  }

  // 🆕 NEW: Delete data from specific collection
  async deleteData(collection, query = {}) {
    if (!this.connected) {
      throw new Error('MongoDB not connected');
    }

    try {
      const coll = this.db.collection(collection);
      const result = await coll.deleteOne(query);
      
      return {
        success: true,
        collection: collection,
        deleted: result.deletedCount > 0,
        timestamp: new Date()
      };
    } catch (error) {
      console.error('MongoDB delete error:', error.message);
      this.metrics.errorsHandled++;
      throw error;
    }
  }

  // 🆕 NEW: Create indexes for specific collection
  async createIndex(collection, indexSpec, options = {}) {
    if (!this.connected) {
      throw new Error('MongoDB not connected');
    }

    try {
      const coll = this.db.collection(collection);
      const result = await coll.createIndex(indexSpec, options);
      
      console.log(`✅ Created index for collection ${collection}:`, result);
      return result;
    } catch (error) {
      console.error(`❌ Failed to create index for collection ${collection}:`, error.message);
      throw error;
    }
  }

  // 🆕 NEW: Setup optimized indexes for specific collection
  async setupOptimizedIndexes(collection) {
    try {
      // Create common indexes
      await this.createIndex(collection, { _createdAt: -1 });
      await this.createIndex(collection, { _updatedAt: -1 });
      await this.createIndex(collection, { _ttl: 1 }, { expireAfterSeconds: 0 });
      
      console.log(`✅ Optimized indexes created for collection: ${collection}`);
    } catch (error) {
      console.error(`❌ Failed to setup indexes for collection ${collection}:`, error.message);
    }
  }

  getMetrics() {
    return {
      ...this.metrics,
      connected: this.connected,
      activeWatchers: this.activeWatchers.size,
      activeStreams: this.changeStreams.size,
      reconnectAttempts: this.reconnectAttempts
    };
  }

  async disconnect() {
    if (this.healthCheckInterval) {
      clearInterval(this.healthCheckInterval);
    }

    // Stop all watchers
    for (const [collection] of this.activeWatchers) {
      await this.stopWatch(collection);
    }

    // Close all change streams
    for (const [collection, changeStream] of this.changeStreams) {
      try {
        await changeStream.close();
      } catch (error) {
        console.error(`Error closing change stream for ${collection}:`, error);
      }
    }

    if (this.client) {
      await this.client.close();
    }

    this.connected = false;
    this.activeWatchers.clear();
    this.changeStreams.clear();
    this.lastKnownState.clear();

    console.log('MongoDB disconnected');
  }

  isConnected() {
    return this.connected;
  }
}

module.exports = AdvancedMongoConnector;