@krunal_tarale-5/ultimate-streaming-package/lib/advancedMysqlConnector.js

🚀 Ultimate Real-Time Streaming Package v2.1.9 - Multi-Platform, Multi-Collection Architecture with Native MongoDB & MySQL Support, 99.96% Performance Improvement. Enterprise-grade real-time data streaming with Socket.IO integration, dynamic schema evolut

const mysql = require('mysql2/promise');
const EventEmitter = require('events');

class AdvancedMysqlConnector extends EventEmitter {
  constructor() {
    super();
    this.pool = null;
    this.connected = false;
    this.config = null;
    this.activeWatchers = new Map(); // table -> { callbacks: Set, options }
    this.lastKnownState = new Map(); // table -> lastData
    this.zongJi = null; // Binlog reader
    this.metrics = {
      connections: 0,
      writesHandled: 0,
      readsHandled: 0,
      changesProcessed: 0,
      errorsHandled: 0,
      activeWatchers: 0,
      activeStreams: 0
    };
    this.healthCheckInterval = null;
    this.reconnectAttempts = 0;
    this.maxReconnectAttempts = 10;
  }

  async connect(config) {
    if (this.connected) {
      console.log('MySQL already connected');
      return;
    }

    this.config = config;
    
    try {
      console.log('🔄 Connecting to MySQL...');
      
      // Create connection pool
      this.pool = mysql.createPool({
        host: config.host,
        port: config.port || 3306,
        user: config.user,
        password: config.password,
        database: config.database,
        connectionLimit: config.maxConnections || 50,
        charset: 'utf8mb4',
        supportBigNumbers: true,
        bigNumberStrings: true,
        dateStrings: true
      });

      // Test connection
      const connection = await this.pool.getConnection();
      await connection.ping();
      connection.release();
      
      this.connected = true;
      this.reconnectAttempts = 0;

      console.log('✅ MySQL connected successfully');
      
      // Set up connection monitoring
      this.setupConnectionMonitoring();
      
      // Start health checks
      this.startHealthChecks();
      
      // Setup binlog monitoring if enabled
      if (config.useBinlog !== false) {
        await this.setupBinlogMonitoring();
      }
      
      this.emit('connected');
      
    } catch (error) {
      console.error('❌ MySQL connection failed:', error.message);
      this.emit('error', error);
      throw error;
    }
  }

  setupConnectionMonitoring() {
    this.pool.on('connection', (connection) => {
      this.metrics.connections++;
    });

    this.pool.on('error', async (error) => {
      console.error('MySQL pool error:', error);
      await this.handleConnectionError(error);
    });
  }

  async handleConnectionError(error) {
    this.metrics.errorsHandled++;
    this.emit('error', error);
    
    if (this.reconnectAttempts < this.maxReconnectAttempts) {
      this.reconnectAttempts++;
      console.log(`🔄 Attempting to reconnect (${this.reconnectAttempts}/${this.maxReconnectAttempts})...`);
      
      setTimeout(async () => {
        try {
          await this.connect(this.config);
          await this.restoreWatchers();
        } catch (reconnectError) {
          console.error('Reconnection failed:', reconnectError.message);
        }
      }, 1000 * this.reconnectAttempts);
    } else {
      console.error('Max reconnection attempts reached');
      this.emit('maxReconnectAttemptsReached');
    }
  }

  async handleDisconnection() {
    this.connected = false;
    this.emit('disconnected');
    
    // Close binlog reader
    if (this.zongJi) {
      try {
        this.zongJi.stop();
        this.zongJi = null;
      } catch (error) {
        console.error('Error closing binlog reader:', error);
      }
    }
  }

  async restoreWatchers() {
    console.log('🔄 Restoring active watchers...');
    
    for (const [table, watcherInfo] of this.activeWatchers) {
      try {
        await this.startRealTimeWatch(table, watcherInfo.callback, watcherInfo.options);
        console.log(`✅ Restored watcher for table: ${table}`);
      } catch (error) {
        console.error(`❌ Failed to restore watcher for table ${table}:`, error);
      }
    }
  }

  startHealthChecks() {
    if (this.healthCheckInterval) {
      clearInterval(this.healthCheckInterval);
    }

    this.healthCheckInterval = setInterval(async () => {
      try {
        const connection = await this.pool.getConnection();
        await connection.ping();
        connection.release();
        this.emit('healthCheck', { 
          status: 'healthy', 
          metrics: this.getMetrics(),
          binlogActive: !!this.zongJi
        });
      } catch (error) {
        console.error('Health check failed:', error);
        this.emit('healthCheck', { 
          status: 'unhealthy', 
          error: error.message,
          binlogActive: !!this.zongJi
        });
      }
    }, 30000);
  }

  // 🆕 NEW: Write data to specific table
  async writeData(table, data, options = {}) {
    if (!this.connected) {
      throw new Error('MySQL not connected');
    }

    try {
      const connection = await this.pool.getConnection();
      
      try {
        await connection.beginTransaction();
        
        // Build INSERT query dynamically based on data fields
        const fields = Object.keys(data);
        const placeholders = fields.map(() => '?').join(', ');
        const values = fields.map(field => data[field]);
        
        const query = `
          INSERT INTO \`${table}\` (${fields.map(f => `\`${f}\``).join(', ')})
          VALUES (${placeholders})
        `;
        
        let result;
        try {
          result = await connection.execute(query, values);
        } catch (error) {
          // If table doesn't exist, create it and retry
          if (error.message.includes("doesn't exist")) {
            console.log(`🔄 Table ${table} doesn't exist, creating it...`);
            await this.ensureTableExists(connection, table, data);
            result = await connection.execute(query, values);
          } else {
            throw error;
          }
        }
        
        await connection.commit();
        
        return {
          success: true,
          table: table,
          insertedId: result.insertId,
          timestamp: new Date()
        };
      } catch (error) {
        await connection.rollback();
        throw error;
      } finally {
        connection.release();
      }
    } catch (error) {
      console.error('MySQL write error:', error.message);
      this.metrics.errorsHandled++;
      throw error;
    }
  }

  // Helper method to create table if it doesn't exist
  async ensureTableExists(connection, table, sampleData) {
    try {
      console.log(`🔄 Ensuring table exists: ${table}`);
      
      // Build the CREATE TABLE query
      const columnDefinitions = [
        '`id` INT AUTO_INCREMENT PRIMARY KEY',
        '`createdAt` DATETIME DEFAULT CURRENT_TIMESTAMP',
        '`updatedAt` DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP'
      ];
      
      // Add data-specific columns
      Object.keys(sampleData).forEach(field => {
        // Skip if it's a standard column
        if (['id', 'createdAt', 'updatedAt'].includes(field)) {
          return;
        }
        
        const value = sampleData[field];
        let columnDef = `\`${field}\``;
        
        // Handle nested objects as JSON
        if (typeof value === 'object' && value !== null && !(value instanceof Date) && !Array.isArray(value)) {
          columnDef += ' JSON';
        } else if (typeof value === 'string') {
          if (field.includes('Id') || field.includes('ID')) {
            columnDef += ' VARCHAR(255)';
          } else if (field.includes('email')) {
            columnDef += ' VARCHAR(255)';
          } else if (field.includes('name')) {
            columnDef += ' VARCHAR(255)';
          } else if (field.includes('status')) {
            columnDef += ' VARCHAR(50)';
          } else if (field.includes('preferences') || field.includes('items')) {
            columnDef += ' JSON';
          } else {
            columnDef += ' TEXT';
          }
        } else if (typeof value === 'number') {
          if (Number.isInteger(value)) {
            columnDef += ' INT';
          } else {
            columnDef += ' DECIMAL(10,2)';
          }
        } else if (value instanceof Date) {
          columnDef += ' DATETIME';
        } else if (typeof value === 'boolean') {
          columnDef += ' BOOLEAN';
        } else if (Array.isArray(value)) {
          columnDef += ' JSON';
        } else {
          columnDef += ' JSON';
        }
        
        columnDefinitions.push(columnDef);
      });
      
      // Add indexes for common fields
      const indexDefinitions = [];
      Object.keys(sampleData).forEach(field => {
        if (field.includes('Id') || field.includes('ID') || field.includes('email')) {
          indexDefinitions.push(`INDEX idx_${field} (\`${field}\`)`);
        }
      });
      
      // Build the complete CREATE TABLE query
      const allDefinitions = [...columnDefinitions, ...indexDefinitions];
      console.log(`🔄 Column definitions:`, columnDefinitions);
      console.log(`🔄 Index definitions:`, indexDefinitions);
      console.log(`🔄 All definitions:`, allDefinitions);
      
      const createTableQuery = `
        CREATE TABLE IF NOT EXISTS \`${table}\` (
          ${allDefinitions.join(',\n          ')}
        ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
      `;
      
      console.log(`🔄 Creating table with query:`, createTableQuery);
      await connection.execute(createTableQuery);
      console.log(`✅ Table created: ${table}`);
    } catch (error) {
      console.error(`❌ Failed to create table ${table}:`, error.message);
      console.error('Error details:', error);
      throw error;
    }
  }

  // 🆕 NEW: Update data in specific table with query
  async updateData(table, query, updateData, options = {}) {
    if (!this.connected) {
      throw new Error('MySQL not connected');
    }

    try {
      const connection = await this.pool.getConnection();
      
      try {
        await connection.beginTransaction();

        // Ensure all update fields exist as columns
        await this.ensureColumnsExist(connection, table, updateData);
        
        // Build WHERE clause
        const whereConditions = [];
        const whereValues = [];
        
        Object.keys(query).forEach(field => {
          whereConditions.push(`\`${field}\` = ?`);
          whereValues.push(query[field]);
        });
        
        const whereClause = whereConditions.length > 0 ? `WHERE ${whereConditions.join(' AND ')}` : '';
        
        // Build SET clause
        const setFields = [];
        const setValues = [];
        
        Object.keys(updateData).forEach(field => {
          setFields.push(`\`${field}\` = ?`);
          setValues.push(updateData[field]);
        });
        
        const setClause = setFields.join(', ');
        
        const updateQuery = `
          UPDATE \`${table}\` 
          SET ${setClause}
          ${whereClause}
        `;
        
        const [result] = await connection.execute(updateQuery, [...setValues, ...whereValues]);
        
        await connection.commit();
        
        return {
          success: true,
          table: table,
          found: result.affectedRows > 0,
          updated: result.affectedRows > 0,
          timestamp: new Date(),
          query: query
        };
      } catch (error) {
        await connection.rollback();
        throw error;
      } finally {
        connection.release();
      }
    } catch (error) {
      console.error('MySQL update error:', error.message);
      this.metrics.errorsHandled++;
      throw error;
    }
  }

  // Helper to ensure columns exist for updateData
  async ensureColumnsExist(connection, table, updateData) {
    try {
      // Get current columns
      const [columns] = await connection.execute(
        'SELECT COLUMN_NAME, DATA_TYPE FROM information_schema.columns WHERE table_schema = ? AND table_name = ?',
        [this.config.database, table]
      );
      const existing = new Set(columns.map(col => col.COLUMN_NAME));
      const alters = [];
      for (const field of Object.keys(updateData)) {
        if (!existing.has(field)) {
          // Infer type
          const value = updateData[field];
          let type = 'TEXT';
          if (typeof value === 'string') {
            if (field.includes('Id') || field.includes('ID')) type = 'VARCHAR(255)';
            else if (field.includes('email')) type = 'VARCHAR(255)';
            else if (field.includes('name')) type = 'VARCHAR(255)';
            else if (field.includes('status')) type = 'VARCHAR(50)';
            else if (field.includes('preferences') || field.includes('items')) type = 'JSON';
            else type = 'TEXT';
          } else if (typeof value === 'number') {
            if (Number.isInteger(value)) type = 'INT';
            else type = 'DECIMAL(10,2)';
          } else if (value instanceof Date) {
            type = 'DATETIME';
          } else if (typeof value === 'boolean') {
            type = 'BOOLEAN';
          } else {
            type = 'JSON';
          }
          alters.push(`ADD COLUMN \`${field}\` ${type}`);
        }
      }
      if (alters.length > 0) {
        const alterQuery = `ALTER TABLE \`${table}\` ${alters.join(', ')}`;
        console.log(`🔄 Altering table ${table} to add columns:`, alters);
        await connection.execute(alterQuery);
        console.log(`✅ Table ${table} altered successfully.`);
      }
    } catch (error) {
      console.error(`❌ Failed to alter table ${table}:`, error.message);
      throw error;
    }
  }

  // 🆕 NEW: Read data from specific table
  async readData(table, query = {}, options = {}) {
    if (!this.connected) {
      throw new Error('MySQL not connected');
    }

    try {
      const connection = await this.pool.getConnection();
      
      try {
        // Build WHERE clause
        const whereConditions = [];
        const whereValues = [];
        
        Object.keys(query).forEach(field => {
          whereConditions.push(`\`${field}\` = ?`);
          whereValues.push(query[field]);
        });
        
        const whereClause = whereConditions.length > 0 ? `WHERE ${whereConditions.join(' AND ')}` : '';
        
        const selectQuery = `
          SELECT * FROM \`${table}\`
          ${whereClause}
          ${options.orderBy ? `ORDER BY ${options.orderBy}` : ''}
          ${options.limit ? `LIMIT ${options.limit}` : ''}
        `;
        
        const [rows] = await connection.execute(selectQuery, whereValues);
        
        if (rows.length === 0) {
          return null;
        }

        return {
          table: table,
          data: rows[0],
          timestamp: new Date(),
          lastModified: rows[0].updated_at || new Date()
        };
      } finally {
        connection.release();
      }
    } catch (error) {
      console.error('MySQL read error:', error.message);
      throw error;
    }
  }

  // 🆕 NEW: Start real-time watch for specific table
  async startRealTimeWatch(table, callback, options = {}) {
    if (!this.connected) {
      throw new Error('MySQL not connected');
    }

    try {
      // Store watcher info
      this.activeWatchers.set(table, { callback, options });
      this.metrics.activeWatchers++;
      
      // Get initial data and store in cache
      const initialData = await this.readData(table);
      if (initialData) {
        this.lastKnownState.set(table, initialData);
        const meta = {
          table: table,
          changeType: 'initial',
          timestamp: new Date(),
          source: 'initial'
        };
        callback(initialData.data, meta);
      }

      console.log(`Real-time MySQL watch started for table: ${table}`);
      
      // If binlog is not available, the polling fallback will handle this
      return { table, active: true };
      
    } catch (error) {
      console.error(`Failed to start MySQL watch for table ${table}:`, error);
      this.activeWatchers.delete(table);
      throw error;
    }
  }

  async stopWatch(table) {
    if (this.activeWatchers.has(table)) {
      this.activeWatchers.delete(table);
      this.lastKnownState.delete(table);
      this.metrics.activeWatchers--;
      console.log(`Stopped watching table: ${table}`);
    }
  }

  // 🆕 NEW: Query data from specific table
  async queryData(table, query = {}, options = {}) {
    if (!this.connected) {
      throw new Error('MySQL not connected');
    }

    try {
      const connection = await this.pool.getConnection();
      
      try {
        // Build WHERE clause
        const whereConditions = [];
        const whereValues = [];
        
        if (query.where) {
          Object.keys(query.where).forEach(field => {
            whereConditions.push(`\`${field}\` = ?`);
            whereValues.push(query.where[field]);
          });
        }
        
        const whereClause = whereConditions.length > 0 ? `WHERE ${whereConditions.join(' AND ')}` : '';
        
        // Build SELECT clause
        const selectFields = options.fields ? options.fields.map(f => `\`${f}\``).join(', ') : '*';
        
        const selectQuery = `
          SELECT ${selectFields} FROM \`${table}\`
          ${whereClause}
          ${options.orderBy ? `ORDER BY ${options.orderBy}` : ''}
          ${options.limit ? `LIMIT ${options.limit}` : ''}
          ${options.offset ? `OFFSET ${options.offset}` : ''}
        `;
        
        const [rows] = await connection.execute(selectQuery, whereValues);
        
        return rows.map(row => ({
          table: table,
          data: row,
          timestamp: new Date(),
          lastModified: row.updated_at || new Date()
        }));
      } finally {
        connection.release();
      }
    } catch (error) {
      console.error('MySQL query error:', error.message);
      throw error;
    }
  }

  // 🆕 NEW: Get all tables
  async getAllTables() {
    if (!this.connected) {
      throw new Error('MySQL not connected');
    }

    try {
      const connection = await this.pool.getConnection();
      
      try {
        const [rows] = await connection.execute(`
          SELECT table_name 
          FROM information_schema.tables 
          WHERE table_schema = ?
        `, [this.config.database]);
        
        return rows.map(row => row.table_name);
      } finally {
        connection.release();
      }
    } catch (error) {
      console.error('MySQL get tables error:', error.message);
      throw error;
    }
  }

  // 🆕 NEW: Delete data from specific table
  async deleteData(table, query = {}) {
    if (!this.connected) {
      throw new Error('MySQL not connected');
    }

    try {
      const connection = await this.pool.getConnection();
      
      try {
        await connection.beginTransaction();
        
        // Build WHERE clause
        const whereConditions = [];
        const whereValues = [];
        
        Object.keys(query).forEach(field => {
          whereConditions.push(`\`${field}\` = ?`);
          whereValues.push(query[field]);
        });
        
        const whereClause = whereConditions.length > 0 ? `WHERE ${whereConditions.join(' AND ')}` : '';
        
        const deleteQuery = `
          DELETE FROM \`${table}\`
          ${whereClause}
        `;
        
        const [result] = await connection.execute(deleteQuery, whereValues);
        
        await connection.commit();
        
        return {
          success: true,
          table: table,
          deleted: result.affectedRows > 0,
          timestamp: new Date()
        };
      } catch (error) {
        await connection.rollback();
        throw error;
      } finally {
        connection.release();
      }
    } catch (error) {
      console.error('MySQL delete error:', error.message);
      this.metrics.errorsHandled++;
      throw error;
    }
  }

  // 🆕 NEW: Create indexes for specific table
  async createIndex(table, indexName, fields, options = {}) {
    if (!this.connected) {
      throw new Error('MySQL not connected');
    }

    try {
      const connection = await this.pool.getConnection();
      
      try {
        const fieldList = Array.isArray(fields) ? fields.map(f => `\`${f}\``).join(', ') : `\`${fields}\``;
        const indexType = options.unique ? 'UNIQUE' : '';
        
        const createIndexQuery = `
          CREATE ${indexType} INDEX \`${indexName}\` 
          ON \`${table}\` (${fieldList})
        `;
        
        await connection.execute(createIndexQuery);
        
        console.log(`✅ Created index ${indexName} for table ${table}`);
        return { success: true, indexName, table };
      } finally {
        connection.release();
      }
    } catch (error) {
      console.error(`❌ Failed to create index for table ${table}:`, error.message);
      throw error;
    }
  }

  // 🆕 NEW: Setup optimized indexes for specific table
  async setupOptimizedIndexes(table) {
    try {
      // Create common indexes
      await this.createIndex(table, `idx_${table}_created_at`, 'created_at');
      await this.createIndex(table, `idx_${table}_updated_at`, 'updated_at');
      
      console.log(`✅ Optimized indexes created for table: ${table}`);
    } catch (error) {
      console.error(`❌ Failed to setup indexes for table ${table}:`, error.message);
    }
  }

  async setupBinlogMonitoring() {
    try {
      // This would require additional binlog library
      // For now, we'll use polling fallback
      console.log('Binlog monitoring not implemented, using polling fallback');
      this.setupPollingFallback();
    } catch (error) {
      console.error('Failed to setup binlog monitoring:', error);
      this.setupPollingFallback();
    }
  }

  setupPollingFallback() {
    // Setup polling for all active watchers
    for (const [table, watcherInfo] of this.activeWatchers) {
      this.checkTableForChanges(table, watcherInfo);
    }
  }

  async checkTableForChanges(table, watcherInfo) {
    try {
      const currentData = await this.readData(table);
      const lastKnown = this.lastKnownState.get(table);
      
      if (currentData && (!lastKnown || currentData.lastModified > lastKnown.lastModified)) {
        // Data changed
        this.lastKnownState.set(table, currentData);
        
        const meta = {
          table: table,
          changeType: 'updated',
          timestamp: new Date(),
          source: 'polling'
        };
        
        watcherInfo.callback(currentData.data, meta);
        this.emit('change', { table, data: currentData.data, meta });
      }
    } catch (error) {
      console.error(`Error checking table ${table} for changes:`, error);
    }
  }

  getMetrics() {
    return {
      ...this.metrics,
      connected: this.connected,
      activeWatchers: this.activeWatchers.size,
      reconnectAttempts: this.reconnectAttempts
    };
  }

  async disconnect() {
    if (this.healthCheckInterval) {
      clearInterval(this.healthCheckInterval);
    }

    // Stop all watchers
    for (const [table] of this.activeWatchers) {
      await this.stopWatch(table);
    }

    // Close binlog reader
    if (this.zongJi) {
      try {
      this.zongJi.stop();
      } catch (error) {
        console.error('Error closing binlog reader:', error);
      }
    }

    if (this.pool) {
      await this.pool.end();
    }

    this.connected = false;
    this.activeWatchers.clear();
    this.lastKnownState.clear();

    console.log('MySQL disconnected');
  }

  isConnected() {
    return this.connected;
  }
}

module.exports = AdvancedMysqlConnector;