bigbasealpha/src/alpha.js

Professional Grade Custom Database System - A sophisticated, dependency-free database with encryption, caching, indexing, and web dashboard

/*
 * Copyright 2025 BigBaseAlpha Team
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import { EventEmitter } from 'events';
import { fileURLToPath } from 'url';
import { dirname, join } from 'path';
import { existsSync, mkdirSync } from 'fs';
import crypto from 'crypto';

import { StorageEngine } from './storage/index.js';
import { SecurityManager } from './security/index.js';
import { IndexManager } from './indexing/index.js';
import { CacheManager } from './caching/index.js';
import { PluginManager } from './plugins/index.js';
import { AuditLogger } from './utils/audit.js';
import { ConfigManager } from './utils/config.js';
import { AuthManager } from './security/auth.js';
import { BackupManager } from './backup/index.js';
import { SearchEngine } from './search/index.js';
import { QueryProfiler } from './profiler/index.js';
import { ETLEngine } from './etl/index.js';
import { StreamingEngine } from './streaming/index.js';
import { AnalyticsEngine } from './analytics/index.js';
import { APIGateway } from './gateway/index.js';
import { MLEngine } from './ml/index.js';
import { ReplicationEngine } from './replication/index.js';
import { MonitoringEngine } from './monitoring/index.js';
import { DatabaseConnectors } from './connectors/index.js';
import { GraphQLEngine } from './graphql/index.js';
import { RedisLikeCache } from './redis/index.js';
import { EventSourcingEngine } from './eventsourcing/index.js';
import { BlockchainEngine } from './blockchain/index.js';
import { DistributedComputingEngine } from './distributed/index.js';
import { StreamProcessor } from './streaming/processor.js';
import TerminalUI from './ui/index.js';
import PerformanceAnalytics from './analytics/performance.js';
import SecurityPrivacySuite from './security/privacy.js';
import CollectionManager from './collections/index.js';
import PerformanceEngine from './performance/index.js';

const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);

/**
 * BigBaseAlpha - Professional Grade Custom Database System
 * Built entirely from scratch without external database dependencies
 */
export class BigBaseAlpha extends EventEmitter {
  constructor(options = {}) {
    super();
    // Central log control
    this.silent = options.silent || false;
    this.logger = options.logger || console;
    // Default configuration
    this.config = {
      path: options.path || './bigbase_data',
      format: options.format || 'json',
      encryption: options.encryption || false,
      compression: options.compression || false,
      maxMemory: options.maxMemory || '512MB',
      backupInterval: options.backupInterval || 3600000,
      indexing: options.indexing !== false,
      caching: options.caching !== false,
      auditLog: options.auditLog !== false,
      ttlCleanup: options.ttlCleanup !== false,
      plugins: options.plugins || [],
      silent: this.silent,
      logger: this.logger,
      ...options
    };

    // Initialize managers
    this.configManager = new ConfigManager(this.config);
    this.storage = new StorageEngine(this.config);
    this.security = new SecurityManager(this.config);
    this.indexing = new IndexManager(this.config);
    this.cache = new CacheManager(this.config);
    this.plugins = new PluginManager(this.config);
    this.audit = new AuditLogger(this.config);
    this.auth = new AuthManager(this.config);
    this.backupManager = new BackupManager(this.config);
    this.searchEngine = new SearchEngine(this.config);
    this.queryProfiler = new QueryProfiler(this.config);
    this.etlEngine = new ETLEngine(this.config);
    this.streamingEngine = new StreamingEngine(this.config);
    this.analyticsEngine = new AnalyticsEngine(this.config);
    this.apiGateway = new APIGateway(this.config);
    this.mlEngine = new MLEngine(this.config);
    this.replicationEngine = new ReplicationEngine(this.config);
    this.monitoringEngine = new MonitoringEngine(this.config);
    this.databaseConnectors = new DatabaseConnectors(this.config);
    this.graphqlEngine = new GraphQLEngine(this.config);
    this.redisCache = new RedisLikeCache(this.config);
    this.eventSourcing = new EventSourcingEngine(this.config);
    this.blockchain = new BlockchainEngine(this.config);
    // this.distributedComputing = new DistributedComputingEngine(this.config); // Temporarily disabled
    this.streamProcessor = new StreamProcessor(this.config);

    // Initialize Terminal UI and Performance Analytics
    this.ui = new TerminalUI({
      theme: this.config.ui?.theme || 'default',
      colors: this.config.ui?.colors !== false,
      animation: this.config.ui?.animation !== false
    });
    
    this.performance = new PerformanceAnalytics({
      sampleInterval: this.config.performance?.sampleInterval || 1000,
      maxSamples: this.config.performance?.maxSamples || 1000,
      enableProfiling: this.config.performance?.enableProfiling !== false
    });

    // Initialize Security & Privacy Suite
    this.securitySuite = new SecurityPrivacySuite(this, {
      encryption: this.config.security?.encryption || 'AES-256-GCM',
      wipeIterations: this.config.security?.wipeIterations || 3,
      paranoidLogging: this.config.security?.paranoidLogging !== false
    });

    // Initialize Collection Manager and Performance Engine
    this.collectionManager = new CollectionManager(this, {
      maxCollections: this.config.collections?.maxCollections || 1000,
      autoCreateCollections: this.config.collections?.autoCreateCollections !== false,
      strictMode: this.config.collections?.strictMode || false
    });

    this.performanceEngine = new PerformanceEngine(this, {
      lazyWriteDelay: this.config.performance?.lazyWriteDelay || 5000,
      batchSize: this.config.performance?.batchSize || 100,
      compressionEnabled: this.config.performance?.compressionEnabled !== false
    });

    // State management
    this.isInitialized = false;
    this.collections = new Map();
    this.schemas = new Map();
    this.operationLocks = new Map();
    this.lazyWrite = false; // Performance mode flag

    this.stats = {
      totalOperations: 0,
      totalInserts: 0,
      totalReads: 0,
      totalUpdates: 0,
      totalDeletes: 0,
      startTime: null,
      lastBackup: null
    };

    // Bind event handlers
    this._bindEvents();
  }

  /**
   * Initialize the database system
   */
  async init() {
    try {
      this.emit('onInit', this);
      
      // Create data directory if it doesn't exist
      if (!existsSync(this.config.path)) {
        mkdirSync(this.config.path, { recursive: true });
      }

      // Initialize all managers
      await this.configManager.init(this.config);
      await this.security.init();
      await this.storage.init();
      await this.indexing.init();
      await this.cache.init();
      await this.audit.init();
      await this.plugins.init();
      
      // Initialize new managers
      await this.auth.init();
      await this.backupManager.init();
      await this.searchEngine.init();
      await this.queryProfiler.init();
      await this.etlEngine.init();
      
      // Streaming Engine - only init if enabled
      if (this.config.streaming?.enabled !== false) {
        await this.streamingEngine.init();
      }
      
      await this.analyticsEngine.init();
      
      // API Gateway - only init if enabled
      if (this.config.apiGateway?.enabled !== false) {
        await this.apiGateway.init();
      }
      
      await this.mlEngine.init();
      await this.replicationEngine.init();
      await this.monitoringEngine.init();
      await this.databaseConnectors.init();
      await this.graphqlEngine.init();
      
      // Redis Cache - only init if enabled
      if (this.config.redis?.enabled !== false) {
        await this.redisCache.init();
      }
      
      // Event Sourcing - only init if enabled
      if (this.config.eventSourcing?.enabled !== false) {
        await this.eventSourcing.initialize();
      }
      
      // Blockchain - only init if enabled
      if (this.config.blockchain?.enabled !== false) {
        await this.blockchain.initialize();
      }
      
      this.backupManager.setDatabase(this);
      this.etlEngine.setDatabase(this);
      
      // Set database for optional components only if enabled
      if (this.config.streaming?.enabled !== false) {
        this.streamingEngine.setDatabase(this);
      }
      
      this.analyticsEngine.setDatabase(this);
      
      if (this.config.apiGateway?.enabled !== false) {
        this.apiGateway.setDatabase(this);
      }
      
      this.mlEngine.setDatabase(this);
      this.replicationEngine.setDatabase(this);
      this.monitoringEngine.setDatabase(this);
      this.databaseConnectors.setDatabase(this);
      this.graphqlEngine.setDatabase(this);
      this.redisCache.setDatabase(this);

      // Load existing collections
      await this._loadCollections();

      // Start background tasks
      this._startBackgroundTasks();

      this.isInitialized = true;
      this.stats.startTime = new Date();

      this.audit.log('system', 'init', { 
        config: this.config,
        timestamp: new Date()
      });

      this.emit('initialized', this);
      return this;

    } catch (error) {
      this.emit('error', error);
      throw new Error(`Failed to initialize BigBaseAlpha: ${error.message}`);
    }
  }

  /**
   * Create a new collection with optional schema
   */
  async createCollection(name, schema = null) {
    this._ensureInitialized();
    
    if (this.collections.has(name)) {
      throw new Error(`Collection '${name}' already exists`);
    }

    const collection = {
      name,
      schema,
      created: new Date(),
      documents: new Map(),
      metadata: {
        totalDocuments: 0,
        totalSize: 0,
        lastModified: new Date()
      }
    };

    this.collections.set(name, collection);
    if (schema) {
      this.schemas.set(name, schema);
    }

    // Create storage structure
    await this.storage.createCollection(name);
    
    // Create indexes
    if (this.config.indexing) {
      await this.indexing.createIndexes(name, schema);
    }

    this.audit.log('collection', 'create', { name, schema });
    this.emit('collectionCreated', { name, schema });

    return collection;
  }

  /**
   * Insert a document into a collection
   */
  async insert(collectionName, data) {
    return this._withLock(`insert:${collectionName}`, async () => {
      this._ensureInitialized();
      this._ensureCollection(collectionName);

    const queryId = this._generateId();
    const profile = this.queryProfiler.startQuery(queryId, {
      collection: collectionName,
      operation: 'insert',
      query: { data: this._sanitizeForProfiling(data) }
    });      try {
        // Generate unique ID if not provided
        if (!data._id) {
          data._id = this._generateId();
        }

        // Add metadata
        data._created = new Date();
        data._modified = new Date();

        // Validate against schema if exists
        if (this.schemas.has(collectionName)) {
          this._validateSchema(data, this.schemas.get(collectionName));
        }

        // Encrypt sensitive fields if needed
        if (this.config.encryption) {
          data = await this.security.encryptDocument(data);
        }

        // Store in collection
        const collection = this.collections.get(collectionName);
        collection.documents.set(data._id, data);
        collection.metadata.totalDocuments++;
        collection.metadata.lastModified = new Date();

        // Persist to storage
        await this.storage.insert(collectionName, data);

        // Update indexes
        if (this.config.indexing) {
          await this.indexing.addToIndex(collectionName, data);
        }

        // Index for search
        await this.searchEngine.indexDocument(collectionName, data);

        // Update cache
        if (this.config.caching) {
          this.cache.set(`${collectionName}:${data._id}`, data);
        }

        // Update statistics
        this.stats.totalOperations++;
        this.stats.totalInserts++;

        // Emit events
        this.emit('document:inserted', { collectionName, document: data });

        // Audit log
        if (this.config.auditLog) {
          await this.audit.log('INSERT', { collection: collectionName, document: data._id });
        }

        this.queryProfiler.endQuery(queryId, data);
        return data;
      } catch (error) {
        this.queryProfiler.endQuery(queryId, null, error);
        throw error;
      }
    });
  }

  /**
   * Find a document by ID
   */
  async findById(collectionName, id) {
    this._ensureInitialized();
    this._ensureCollection(collectionName);

    // Check cache first
    if (this.config.caching) {
      const cached = this.cache.get(`${collectionName}:${id}`);
      if (cached) {
        this.stats.totalReads++;
        return this._decryptDocument(cached);
      }
    }

    // Check in-memory collection
    const collection = this.collections.get(collectionName);
    if (collection.documents.has(id)) {
      const doc = collection.documents.get(id);
      
      // Update cache
      if (this.config.caching) {
        this.cache.set(`${collectionName}:${id}`, doc);
      }

      this.stats.totalReads++;
      return this._decryptDocument(doc);
    }

    // Load from storage
    const doc = await this.storage.findById(collectionName, id);
    if (doc) {
      // Add to in-memory collection
      collection.documents.set(id, doc);
      
      // Update cache
      if (this.config.caching) {
        this.cache.set(`${collectionName}:${id}`, doc);
      }

      this.stats.totalReads++;
      return this._decryptDocument(doc);
    }

    return null;
  }

  /**
   * Update a document
   */
  async update(collectionName, id, updateData) {
    this._ensureInitialized();
    this._ensureCollection(collectionName);

    const existingDoc = await this.findById(collectionName, id);
    if (!existingDoc) {
      throw new Error(`Document with id '${id}' not found in collection '${collectionName}'`);
    }

    // Merge update data
    const updatedDoc = {
      ...existingDoc,
      ...updateData,
      _id: id, // Preserve ID
      _created: existingDoc._created, // Preserve creation date
      _modified: new Date()
    };

    // Validate against schema if exists
    if (this.schemas.has(collectionName)) {
      this._validateSchema(updatedDoc, this.schemas.get(collectionName));
    }

    // Encrypt if needed
    let docToStore = updatedDoc;
    if (this.config.encryption) {
      docToStore = await this.security.encryptDocument(updatedDoc);
    }

    // Update in collection
    const collection = this.collections.get(collectionName);
    collection.documents.set(id, docToStore);
    collection.metadata.lastModified = new Date();

    // Persist to storage
    await this.storage.update(collectionName, id, docToStore);

    // Update indexes
    if (this.config.indexing) {
      await this.indexing.updateIndex(collectionName, existingDoc, updatedDoc);
    }

    // Update cache
    if (this.config.caching) {
      this.cache.set(`${collectionName}:${id}`, docToStore);
    }

    this.stats.totalOperations++;
    this.stats.totalUpdates++;

    this.audit.log('document', 'update', { 
      collection: collectionName, 
      id,
      changes: Object.keys(updateData)
    });

    this.emit('documentUpdated', { 
      collection: collectionName, 
      id, 
      document: updatedDoc,
      changes: updateData
    });

    return updatedDoc;
  }

  /**
   * Delete a document
   */
  async delete(collectionName, id) {
    this._ensureInitialized();
    this._ensureCollection(collectionName);

    const doc = await this.findById(collectionName, id);
    if (!doc) {
      return false;
    }

    // Remove from collection
    const collection = this.collections.get(collectionName);
    collection.documents.delete(id);
    collection.metadata.totalDocuments--;
    collection.metadata.lastModified = new Date();

    // Remove from storage
    await this.storage.delete(collectionName, id);

    // Remove from indexes
    if (this.config.indexing) {
      await this.indexing.removeFromIndex(collectionName, doc);
    }

    // Remove from cache
    if (this.config.caching) {
      this.cache.delete(`${collectionName}:${id}`);
    }

    this.stats.totalOperations++;
    this.stats.totalDeletes++;

    this.audit.log('document', 'delete', { 
      collection: collectionName, 
      id 
    });

    this.emit('onDelete', collectionName, id);
    this.emit('documentDeleted', { collection: collectionName, id });

    return true;
  }

  /**
   * Find one document matching the query
   */
  async findOne(collectionName, query = {}) {
    this._ensureInitialized();
    this._ensureCollection(collectionName);
    
    // Use query with limit 1
    const results = await this.query(collectionName, {
      where: query,
      limit: 1
    });

    return results.length > 0 ? results[0] : null;
  }

  /**
   * Find multiple documents
   */
  async find(collectionName, query = {}, options = {}) {
    return this.query(collectionName, {
      where: query,
      ...options
    });
  }

  /**
   * Advanced query with filtering, sorting, and pagination
   */
  async query(collectionName, options = {}) {
    this._ensureInitialized();
    this._ensureCollection(collectionName);

    const queryId = this._generateId();
    const profile = this.queryProfiler.startQuery(queryId, {
      collection: collectionName,
      operation: 'query',
      query: this._sanitizeForProfiling(options.where || {}),
      options: { ...options, where: undefined } // Don't duplicate where clause
    });

    try {
      const {
        where = {},
        sort = {},
        limit = null,
        offset = 0,
        select = null
      } = options;

      let results = [];

      // Try to use indexes if available and query has conditions
      let useIndex = false;
      if (this.config.indexing && Object.keys(where).length > 0) {
        const indexResults = await this.indexing.query(collectionName, where);
        
        if (indexResults.length > 0) {
          results = indexResults;
          useIndex = true;
        }
      }

      // If index query failed or no index available, do full scan
      if (!useIndex) {
        const collection = this.collections.get(collectionName);
        results = Array.from(collection.documents.values());
      }
      
      // Apply where clause (only if not already filtered by index)
      if (Object.keys(where).length > 0 && !useIndex) {
        results = results.filter(doc => this._matchesQuery(doc, where));
      }

      // Apply sorting
      if (Object.keys(sort).length > 0) {
        results = this._sortResults(results, sort);
      }

      // Apply pagination
      if (offset > 0) {
        results = results.slice(offset);
      }
      if (limit !== null) {
        results = results.slice(0, limit);
      }

      // Apply field selection
      if (select) {
        results = results.map(doc => this._selectFields(doc, select));
      }

      // Decrypt documents
      results = await Promise.all(
        results.map(doc => this._decryptDocument(doc))
      );

      this.stats.totalReads++;
      this.queryProfiler.endQuery(queryId, results);
      
      return results;
    } catch (error) {
      this.queryProfiler.endQuery(queryId, null, error);
      throw error;
    }
  }

  /**
   * Get database statistics
   */
  getStats() {
    const uptime = this.stats.startTime ? 
      Date.now() - this.stats.startTime.getTime() : 0;

    return {
      ...this.stats,
      uptime,
      collections: this.collections.size,
      memoryUsage: process.memoryUsage(),
      cacheStats: this.cache.getStats(),
      storageStats: this.storage.getStats()
    };
  }

  /**
   * Get all collection names
   */
  getCollections() {
    return Array.from(this.collections.keys());
  }

  /**
   * List all collection names (alias for getCollections)
   */
  listCollections() {
    return this.getCollections();
  }

  /**
   * Get a collection object for chaining operations
   */
  collection(name) {
    this._ensureCollection(name);
    return {
      find: (query = {}, options = {}) => this.find(name, query, options),
      findOne: (query = {}) => this.findOne(name, query),
      insert: (data) => this.insert(name, data),
      update: (id, data) => this.update(name, id, data),
      delete: (id) => this.delete(name, id),
      count: (query = {}) => {
        const collection = this.collections.get(name);
        if (!query || Object.keys(query).length === 0) {
          return collection.documents.size;
        }
        // Count with filter
        let count = 0;
        for (const doc of collection.documents.values()) {
          if (this._matchesQuery(doc, query)) count++;
        }
        return count;
      }
    };
  }

  /**
   * Backup database
   */
  async backup(path = null) {
    this._ensureInitialized();
    
    const backupPath = path || `./backups/backup_${Date.now()}.bba`;
    await this.storage.backup(backupPath);
    
    this.stats.lastBackup = new Date();
    this.audit.log('system', 'backup', { path: backupPath });
    this.emit('onBackup', backupPath);
    
    return backupPath;
  }

  /**
   * Full-text search across collections
   */
  async search(collectionName, query, options = {}) {
    this._ensureInitialized();
    this._ensureCollection(collectionName);

    const queryId = this._generateId();
    const profile = this.queryProfiler.startQuery(queryId, {
      collection: collectionName,
      operation: 'search',
      query: { searchQuery: query, options }
    });

    try {
      const results = await this.searchEngine.search(collectionName, query, options);
      
      this.queryProfiler.endQuery(queryId, results);
      this.stats.totalReads++;
      
      return results;
    } catch (error) {
      this.queryProfiler.endQuery(queryId, null, error);
      throw error;
    }
  }

  /**
   * Get search suggestions/autocomplete
   */
  async suggest(collectionName, partial, options = {}) {
    this._ensureInitialized();
    this._ensureCollection(collectionName);
    
    return await this.searchEngine.suggest(collectionName, partial, options);
  }

  /**
   * Index a document for full-text search
   */
  async indexForSearch(collectionName, document, searchableFields = null) {
    this._ensureInitialized();
    this._ensureCollection(collectionName);
    
    await this.searchEngine.indexDocument(collectionName, document, searchableFields);
  }

  /**
   * Get search engine statistics
   */
  getSearchStats(collectionName = null) {
    this._ensureInitialized();
    return this.searchEngine.getStats(collectionName);
  }

  /**
   * Get query profiler statistics
   */
  getProfilerStats(options = {}) {
    this._ensureInitialized();
    return this.queryProfiler.getStats(options);
  }

  /**
   * Get slow queries from profiler
   */
  getSlowQueries(options = {}) {
    this._ensureInitialized();
    return this.queryProfiler.getSlowQueries(options);
  }

  /**
   * Analyze query patterns and get recommendations
   */
  analyzeQueryPatterns(options = {}) {
    this._ensureInitialized();
    return this.queryProfiler.analyzePatterns(options);
  }

  /**
   * Get real-time query metrics
   */
  getRealTimeQueryMetrics() {
    this._ensureInitialized();
    return this.queryProfiler.getRealTimeMetrics();
  }

  /**
   * Export query profiles
   */
  async exportQueryProfiles(format = 'json', options = {}) {
    this._ensureInitialized();
    return await this.queryProfiler.exportProfiles(format, options);
  }

  // === ETL & DATA PIPELINE METHODS ===

  /**
   * Create a new ETL pipeline
   */
  async createETLPipeline(config) {
    this._ensureInitialized();
    return await this.etlEngine.createPipeline(config);
  }

  /**
   * Execute an ETL pipeline
   */
  async executeETLPipeline(pipelineId, options = {}) {
    this._ensureInitialized();
    return await this.etlEngine.executePipeline(pipelineId, options);
  }

  /**
   * Get all ETL pipelines
   */
  getETLPipelines() {
    this._ensureInitialized();
    return this.etlEngine.getPipelines();
  }

  /**
   * Get specific ETL pipeline
   */
  getETLPipeline(id) {
    this._ensureInitialized();
    return this.etlEngine.getPipeline(id);
  }

  /**
   * Get active ETL pipelines
   */
  getActiveETLPipelines() {
    this._ensureInitialized();
    return this.etlEngine.getActivePipelines();
  }

  /**
   * Get ETL job history
   */
  getETLJobHistory(limit = 50) {
    this._ensureInitialized();
    return this.etlEngine.getJobHistory(limit);
  }

  /**
   * Get ETL statistics
   */
  getETLStats() {
    this._ensureInitialized();
    return this.etlEngine.getStats();
  }

  /**
   * Import data from CSV file
   */
  async importFromCSV(collectionName, filePath, options = {}) {
    this._ensureInitialized();
    
    const pipeline = await this.createETLPipeline({
      name: `CSV Import - ${collectionName}`,
      description: `Import data from ${filePath} to ${collectionName}`,
      source: {
        type: 'csv',
        path: filePath
      },
      destination: {
        type: 'collection',
        collection: collectionName
      },
      transformations: options.transformations || [],
      validations: options.validations || []
    });

    return await this.executeETLPipeline(pipeline.id);
  }

  /**
   * Export data to CSV file
   */
  async exportToCSV(collectionName, filePath, options = {}) {
    this._ensureInitialized();
    
    const pipeline = await this.createETLPipeline({
      name: `CSV Export - ${collectionName}`,
      description: `Export data from ${collectionName} to ${filePath}`,
      source: {
        type: 'collection',
        collection: collectionName,
        query: options.query || {},
        options: options.queryOptions || {}
      },
      destination: {
        type: 'csv',
        path: filePath
      },
      transformations: options.transformations || []
    });

    return await this.executeETLPipeline(pipeline.id);
  }

  /**
   * Close database connection
   */
  async close() {
    if (!this.isInitialized) {
      return;
    }

    // Stop background tasks
    this._stopBackgroundTasks();

    // Close all managers
    await this.storage.close();
    await this.cache.close();
    await this.indexing.close();
    await this.plugins.close();
    await this.searchEngine.close();
    await this.queryProfiler.close();
    await this.etlEngine.close();

    // Stop backup manager
    if (this.backupManager) {
      this.backupManager.destroy();
    }

    this.isInitialized = false;
    this.emit('closed');
  }

  // Private methods
  _ensureInitialized() {
    if (!this.isInitialized) {
      throw new Error('Database not initialized. Call init() first.');
    }
  }

  _ensureCollection(name) {
    if (!this.collections.has(name)) {
      throw new Error(`Collection '${name}' does not exist. Create it first.`);
    }
  }

  _generateId() {
    return `${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
  }

  _sanitizeForProfiling(data) {
    // Remove sensitive data for profiling logs
    if (!data || typeof data !== 'object') return data;
    
    const sanitized = JSON.parse(JSON.stringify(data));
    const sensitiveFields = ['password', 'token', 'secret', 'key', 'auth'];
    
    function sanitizeObject(obj) {
      if (typeof obj !== 'object' || obj === null) return;
      
      for (const key of Object.keys(obj)) {
        if (sensitiveFields.some(field => key.toLowerCase().includes(field))) {
          obj[key] = '[SANITIZED]';
        } else if (typeof obj[key] === 'object') {
          sanitizeObject(obj[key]);
        }
      }
    }
    
    sanitizeObject(sanitized);
    return sanitized;
  }

  _validateSchema(data, schema) {
    // Basic schema validation
    for (const [field, rules] of Object.entries(schema)) {
      if (rules.required && !(field in data)) {
        throw new Error(`Required field '${field}' is missing`);
      }
      
      if (field in data && rules.type && typeof data[field] !== rules.type) {
        throw new Error(`Field '${field}' must be of type '${rules.type}'`);
      }
    }
  }

  async _decryptDocument(doc) {
    if (this.config.encryption && doc) {
      return await this.security.decryptDocument(doc);
    }
    return doc;
  }

  _matchesQuery(doc, query) {
    // Handle logical operators first
    if (query.$and) {
      return query.$and.every(condition => this._matchesQuery(doc, condition));
    }
    
    if (query.$or) {
      return query.$or.some(condition => this._matchesQuery(doc, condition));
    }
    
    if (query.$not) {
      return !this._matchesQuery(doc, query.$not);
    }

    // Simple query matching - can be extended
    for (const [field, condition] of Object.entries(query)) {
      // Skip logical operators (already handled above)
      if (field.startsWith('$')) continue;
      
      if (typeof condition === 'object' && condition !== null) {
        // Handle operators like $gt, $lt, $eq, etc.
        for (const [op, value] of Object.entries(condition)) {
          switch (op) {
            case '$eq':
              if (doc[field] !== value) return false;
              break;
            case '$gt':
              if (doc[field] <= value) return false;
              break;
            case '$lt':
              if (doc[field] >= value) return false;
              break;
            case '$gte':
              if (doc[field] < value) return false;
              break;
            case '$lte':
              if (doc[field] > value) return false;
              break;
            case '$in':
              if (!Array.isArray(value) || !value.includes(doc[field])) return false;
              break;
            case '$nin':
              if (Array.isArray(value) && value.includes(doc[field])) return false;
              break;
            case '$regex':
              if (typeof doc[field] !== 'string') return false;
              const regex = new RegExp(value.source || value, value.flags || 'i');
              if (!regex.test(doc[field])) return false;
              break;
            case '$exists':
              if ((doc[field] !== undefined) !== value) return false;
              break;
            case '$ne':
              if (doc[field] === value) return false;
              break;
            default:
              throw new Error(`Unknown operator: ${op}`);
          }
        }
      } else {
        // Direct equality
        if (doc[field] !== condition) return false;
      }
    }
    return true;
  }

  _sortResults(results, sort) {
    return results.sort((a, b) => {
      for (const [field, direction] of Object.entries(sort)) {
        const aVal = a[field];
        const bVal = b[field];
        
        if (aVal < bVal) return direction === 1 ? -1 : 1;
        if (aVal > bVal) return direction === 1 ? 1 : -1;
      }
      return 0;
    });
  }

  _selectFields(doc, fields) {
    if (Array.isArray(fields)) {
      const selected = {};
      fields.forEach(field => {
        if (field in doc) {
          selected[field] = doc[field];
        }
      });
      return selected;
    }
    return doc;
  }

  async _loadCollections() {
    const collections = await this.storage.listCollections();
    for (const collectionName of collections) {
      const collection = {
        name: collectionName,
        schema: null,
        created: new Date(),
        documents: new Map(),
        metadata: {
          totalDocuments: 0,
          totalSize: 0,
          lastModified: new Date()
        }
      };
      
      // Load existing documents from storage
      try {
        const documents = await this.storage.listDocuments(collectionName);
        for (const doc of documents) {
          collection.documents.set(doc._id, doc);
          collection.metadata.totalDocuments++;
        }
        console.log(`✅ Loaded ${collection.metadata.totalDocuments} documents from '${collectionName}'`);
      } catch (error) {
        console.warn(`⚠️ Could not load documents from '${collectionName}':`, error.message);
      }
      
      this.collections.set(collectionName, collection);
    }
  }

  _bindEvents() {
    // Plugin event forwarding
    this.on('onInit', (...args) => this.plugins.emit('onInit', ...args));
    this.on('onWrite', (...args) => this.plugins.emit('onWrite', ...args));
    this.on('onDelete', (...args) => this.plugins.emit('onDelete', ...args));
    this.on('onBackup', (...args) => this.plugins.emit('onBackup', ...args));
  }

  _startBackgroundTasks() {
    // Auto backup
    if (this.config.backupInterval > 0) {
      this.backupTimer = setInterval(() => {
        this.backup().catch(err => this.emit('error', err));
      }, this.config.backupInterval);
    }

    // TTL cleanup
    if (this.config.ttlCleanup) {
      this.ttlTimer = setInterval(() => {
        this._cleanupExpiredDocuments().catch(err => this.emit('error', err));
      }, 60000); // Every minute
    }

    // Cache cleanup
    if (this.config.caching) {
      this.cacheTimer = setInterval(() => {
        this.cache.cleanup();
      }, 300000); // Every 5 minutes
    }
  }

  _stopBackgroundTasks() {
    if (this.backupTimer) {
      clearInterval(this.backupTimer);
      this.backupTimer = null;
    }
    if (this.ttlTimer) {
      clearInterval(this.ttlTimer);
      this.ttlTimer = null;
    }
    if (this.cacheTimer) {
      clearInterval(this.cacheTimer);
      this.cacheTimer = null;
    }
  }

  async _cleanupExpiredDocuments() {
    // Clean up documents with TTL
    for (const [collectionName, collection] of this.collections) {
      const expired = [];
      for (const [id, doc] of collection.documents) {
        if (doc._ttl && new Date() > new Date(doc._ttl)) {
          expired.push(id);
        }
      }
      
      for (const id of expired) {
        await this.delete(collectionName, id);
      }
    }
  }

  /**
   * Concurrent operation management
   */
  async _acquireLock(resource) {
    if (!this.operationLocks.has(resource)) {
      this.operationLocks.set(resource, Promise.resolve());
    }
    
    const currentLock = this.operationLocks.get(resource);
    let resolveLock;
    
    const newLock = new Promise(resolve => {
      resolveLock = resolve;
    });
    
    this.operationLocks.set(resource, newLock);
    
    await currentLock;
    return resolveLock;
  }

  async _withLock(resource, operation) {
    const releaseLock = await this._acquireLock(resource);
    try {
      return await operation();
    } finally {
      releaseLock();
    }
  }

  // ===== REAL-TIME STREAMING METHODS =====

  /**
   * Get streaming statistics
   */
  getStreamingStats() {
    this._ensureInitialized();
    return this.streamingEngine.getStats();
  }

  /**
   * Get active streaming connections
   */
  getStreamingConnections() {
    this._ensureInitialized();
    return this.streamingEngine.getConnections();
  }

  /**
   * Create data stream
   */
  createDataStream(name, source, options = {}) {
    this._ensureInitialized();
    return this.streamingEngine.createStream(name, source, options);
  }

  /**
   * Broadcast event to all streaming connections
   */
  broadcastEvent(eventType, data) {
    this._ensureInitialized();
    this.streamingEngine._broadcastEvent(eventType, data);
  }

  // ===== ADVANCED ANALYTICS METHODS =====

  /**
   * Generate analytics report
   */
  async generateAnalyticsReport(type, options = {}) {
    this._ensureInitialized();
    return await this.analyticsEngine.generateReport(type, options);
  }

  /**
   * Detect anomalies in collection data
   */
  async detectAnomalies(collection, field, options = {}) {
    this._ensureInitialized();
    return await this.analyticsEngine.detectAnomalies(collection, field, options);
  }

  /**
   * Generate predictions
   */
  async generatePredictions(type, data, options = {}) {
    this._ensureInitialized();
    return await this.analyticsEngine.generatePredictions(type, data, options);
  }

  /**
   * Get insights for collection
   */
  async getCollectionInsights(collection, options = {}) {
    this._ensureInitialized();
    return await this.analyticsEngine.getInsights(collection, options);
  }

  /**
   * Create analytics dashboard
   */
  createAnalyticsDashboard(name, config) {
    this._ensureInitialized();
    return this.analyticsEngine.createDashboard(name, config);
  }

  /**
   * Get analytics statistics
   */
  getAnalyticsStats() {
    this._ensureInitialized();
    return this.analyticsEngine.getStats();
  }

  // ===== API GATEWAY & MICROSERVICES METHODS =====

  /**
   * Register microservice
   */
  registerMicroservice(name, config) {
    this._ensureInitialized();
    return this.apiGateway.registerService(name, config);
  }

  /**
   * Register API route
   */
  registerAPIRoute(path, config) {
    this._ensureInitialized();
    return this.apiGateway.registerRoute(path, config);
  }

  /**
   * Generate API key
   */
  generateAPIKey(name, permissions = []) {
    this._ensureInitialized();
    return this.apiGateway.generateAPIKey(name, permissions);
  }

  /**
   * Get gateway statistics
   */
  getGatewayStats() {
    this._ensureInitialized();
    return this.apiGateway.getStats();
  }

  /**
   * Get registered services
   */
  getRegisteredServices() {
    this._ensureInitialized();
    return this.apiGateway.getServices();
  }

  // ===== MACHINE LEARNING METHODS =====

  /**
   * Create and train ML model
   */
  async createMLModel(name, algorithm, dataset, options = {}) {
    this._ensureInitialized();
    return await this.mlEngine.createModel(name, algorithm, dataset, options);
  }

  /**
   * Make ML prediction
   */
  async predict(modelId, input, options = {}) {
    this._ensureInitialized();
    return await this.mlEngine.predict(modelId, input, options);
  }

  /**
   * Detect patterns in collection
   */
  async detectDataPatterns(collection, options = {}) {
    this._ensureInitialized();
    return await this.mlEngine.detectPatterns(collection, options);
  }

  /**
   * Generate ML recommendations
   */
  async generateMLRecommendations(type, context, options = {}) {
    this._ensureInitialized();
    return await this.mlEngine.generateRecommendations(type, context, options);
  }

  /**
   * Analyze sentiment of text
   */
  analyzeSentiment(text) {
    this._ensureInitialized();
    return this.mlEngine.analyzeSentiment(text);
  }

  /**
   * Get ML statistics
   */
  getMLStats() {
    this._ensureInitialized();
    return this.mlEngine.getStats();
  }

  // ===== DATA REPLICATION METHODS =====

  /**
   * Get replication status
   */
  getReplicationStatus() {
    this._ensureInitialized();
    return this.replicationEngine.getReplicationStatus();
  }

  /**
   * Get connected replication nodes
   */
  getReplicationNodes() {
    this._ensureInitialized();
    return this.replicationEngine.getConnectedNodes();
  }

  /**
   * Force replication sync
   */
  async forceReplicationSync() {
    this._ensureInitialized();
    return await this.replicationEngine.forceSync();
  }

  /**
   * Switch replication role
   */
  async switchReplicationRole(role) {
    this._ensureInitialized();
    return await this.replicationEngine.switchRole(role);
  }

  /**
   * Get replication metrics
   */
  getReplicationMetrics() {
    this._ensureInitialized();
    return this.replicationEngine.getReplicationMetrics();
  }

  // ===== ADVANCED MONITORING METHODS =====

  /**
   * Get system status and health
   */
  getSystemStatus() {
    this._ensureInitialized();
    return this.monitoringEngine.getSystemStatus();
  }

  /**
   * Get monitoring metrics
   */
  getMonitoringMetrics(category, timeRange) {
    this._ensureInitialized();
    return this.monitoringEngine.getMetrics(category, timeRange);
  }

  /**
   * Get performance metrics and trends
   */
  getPerformanceMetrics() {
    this._ensureInitialized();
    return this.monitoringEngine.getPerformanceMetrics();
  }

  /**
   * Get active alerts
   */
  getActiveAlerts() {
    this._ensureInitialized();
    return this.monitoringEngine.getAlerts(false);
  }

  /**
   * Get all alerts including resolved
   */
  getAllAlerts() {
    this._ensureInitialized();
    return this.monitoringEngine.getAlerts(true);
  }

  /**
   * Acknowledge an alert
   */
  acknowledgeAlert(alertId) {
    this._ensureInitialized();
    return this.monitoringEngine.acknowledgeAlert(alertId);
  }

  /**
   * Resolve an alert
   */
  resolveAlert(alertId) {
    this._ensureInitialized();
    return this.monitoringEngine.resolveAlert(alertId);
  }

  /**
   * Register custom health check
   */
  registerHealthCheck(name, checkFunction) {
    this._ensureInitialized();
    return this.monitoringEngine.registerHealthCheck(name, checkFunction);
  }

  /**
   * Run specific health check
   */
  async runHealthCheck(name) {
    this._ensureInitialized();
    return await this.monitoringEngine.runHealthCheck(name);
  }

  /**
   * Export monitoring metrics
   */
  exportMonitoringMetrics(format = 'json') {
    this._ensureInitialized();
    return this.monitoringEngine.exportMetrics(format);
  }

  // =============================================================================
  // DATABASE CONNECTORS API
  // =============================================================================

  /**
   * Create external database connection
   */
  async createDatabaseConnection(name, config) {
    this._ensureInitialized();
    return await this.databaseConnectors.createConnection(name, config);
  }

  /**
   * Remove external database connection
   */
  async removeDatabaseConnection(name) {
    this._ensureInitialized();
    return await this.databaseConnectors.removeConnection(name);
  }

  /**
   * Get database connection
   */
  getDatabaseConnection(name) {
    this._ensureInitialized();
    return this.databaseConnectors.getConnection(name);
  }

  /**
   * Get all database connections
   */
  getDatabaseConnections() {
    this._ensureInitialized();
    return this.databaseConnectors.getConnections();
  }

  /**
   * Test database connection
   */
  async testDatabaseConnection(name) {
    this._ensureInitialized();
    return await this.databaseConnectors.testConnection(name);
  }

  /**
   * Execute query on external database
   */
  async executeExternalQuery(connectionName, query, params = []) {
    this._ensureInitialized();
    return await this.databaseConnectors.executeQuery(connectionName, query, params);
  }

  /**
   * Import data from external database
   */
  async importFromExternalDB(connectionName, sourceTable, targetCollection, options = {}) {
    this._ensureInitialized();
    return await this.databaseConnectors.importFromExternalDB(connectionName, sourceTable, targetCollection, options);
  }

  /**
   * Export data to external database
   */
  async exportToExternalDB(connectionName, sourceCollection, targetTable, options = {}) {
    this._ensureInitialized();
    return await this.databaseConnectors.exportToExternalDB(connectionName, sourceCollection, targetTable, options);
  }

  /**
   * Sync with external database
   */
  async syncWithExternalDB(connectionName, mappings, options = {}) {
    this._ensureInitialized();
    return await this.databaseConnectors.syncWithExternalDB(connectionName, mappings, options);
  }

  /**
   * Get external database schema
   */
  async getExternalSchema(connectionName, table) {
    this._ensureInitialized();
    return await this.databaseConnectors.getExternalSchema(connectionName, table);
  }

  /**
   * Get database connectors statistics
   */
  getDatabaseConnectorsStats() {
    this._ensureInitialized();
    return this.databaseConnectors.getStats();
  }

  /**
   * Get supported database drivers
   */
  getSupportedDrivers() {
    this._ensureInitialized();
    return this.databaseConnectors.getSupportedDrivers();
  }

  // =============================================================================
  // GRAPHQL API
  // =============================================================================

  /**
   * Execute GraphQL query
   */
  async executeGraphQLQuery(query, variables = {}, context = {}) {
    this._ensureInitialized();
    return await this.graphqlEngine.executeQuery(query, variables, context);
  }

  /**
   * Get GraphQL schema
   */
  getGraphQLSchema() {
    this._ensureInitialized();
    return this.graphqlEngine.getSchema();
  }

  /**
   * Add custom GraphQL type
   */
  addGraphQLType(name, typeDef) {
    this._ensureInitialized();
    return this.graphqlEngine.addCustomType(name, typeDef);
  }

  /**
   * Add custom GraphQL resolver
   */
  addGraphQLResolver(typeName, fieldName, resolver) {
    this._ensureInitialized();
    return this.graphqlEngine.addCustomResolver(typeName, fieldName, resolver);
  }

  /**
   * Get GraphQL statistics
   */
  getGraphQLStats() {
    this._ensureInitialized();
    return this.graphqlEngine.getStats();
  }

  // =============================================================================
  // REDIS-LIKE CACHE METHODS
  // =============================================================================

  /**
   * Execute Redis-like cache command
   */
  async cacheCommand(command, ...args) {
    this._ensureInitialized();
    if (!this.redisCache || !this.config.redis?.enabled) {
      throw new Error('Redis cache is not enabled');
    }
    return this.redisCache.executeCommand(command, ...args);
  }

  /**
   * Redis GET command
   */
  async cacheGet(key) {
    this._ensureInitialized();
    if (!this.redisCache || !this.config.redis?.enabled) {
      throw new Error('Redis cache is not enabled');
    }
    return this.redisCache.get(key);
  }

  /**
   * Redis SET command
   */
  async cacheSet(key, value, options = {}) {
    this._ensureInitialized();
    return this.redisCache.set(key, value, options);
  }

  /**
   * Redis DEL command
   */
  async cacheDel(...keys) {
    this._ensureInitialized();
    return this.redisCache.del(...keys);
  }

  /**
   * Redis EXISTS command
   */
  async cacheExists(...keys) {
    this._ensureInitialized();
    return this.redisCache.exists(...keys);
  }

  /**
   * Redis EXPIRE command
   */
  async cacheExpire(key, seconds) {
    this._ensureInitialized();
    return this.redisCache.expire(key, seconds);
  }

  /**
   * Redis TTL command
   */
  async cacheTTL(key) {
    this._ensureInitialized();
    return this.redisCache.ttl(key);
  }

  /**
   * Redis INCR command
   */
  async cacheIncr(key) {
    this._ensureInitialized();
    return this.redisCache.incr(key);
  }

  /**
   * Redis DECR command
   */
  async cacheDecr(key) {
    this._ensureInitialized();
    return this.redisCache.decr(key);
  }

  /**
   * Redis LPUSH command
   */
  async cacheLPush(key, ...elements) {
    this._ensureInitialized();
    return this.redisCache.lpush(key, ...elements);
  }

  /**
   * Redis RPUSH command
   */
  async cacheRPush(key, ...elements) {
    this._ensureInitialized();
    return this.redisCache.rpush(key, ...elements);
  }

  /**
   * Redis LPOP command
   */
  async cacheLPop(key) {
    this._ensureInitialized();
    return this.redisCache.lpop(key);
  }

  /**
   * Redis RPOP command
   */
  async cacheRPop(key) {
    this._ensureInitialized();
    return this.redisCache.rpop(key);
  }

  /**
   * Redis LLEN command
   */
  async cacheLLen(key) {
    this._ensureInitialized();
    return thi