bigbasealpha/src/alpha.js

Enterprise-Grade NoSQL Database System with Modular Logger & Offline HSM Security - Complete database platform with professional text-based logging, encryption, caching, indexing, JWT authentication, auto-generated REST API, real-time dashboard, and maste

/*
 * Copyright 2025 ByAlphas
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import { EventEmitter } from 'events';
import { fileURLToPath } from 'url';
import { dirname, join } from 'path';
import { existsSync, mkdirSync } from 'fs';
import crypto from 'crypto';

import { StorageEngine } from './storage/index.js';
import { SecurityManager } from './security/index.js';
import { IndexManager } from './indexing/index.js';
import { CacheManager } from './caching/index.js';
import { PluginManager } from './plugins/index.js';
import { AuditLogger } from './utils/audit.js';
import { ConfigManager } from './utils/config.js';
import { ModularLogger, LoggerPresets } from './utils/logger.js';
import { AuthManager } from './security/auth.js';
import { OfflineHSM } from './security/hsm.js';
import { BackupManager } from './backup/index.js';
import { SearchEngine } from './search/index.js';
import { QueryProfiler } from './profiler/index.js';
import { ETLEngine } from './etl/index.js';
import { StreamingEngine } from './streaming/index.js';
import { AnalyticsEngine } from './analytics/index.js';
import { APIGateway } from './gateway/index.js';
import { MLEngine } from './ml/index.js';
import { ReplicationEngine } from './replication/index.js';
import { MonitoringEngine } from './monitoring/index.js';
import { DatabaseConnectors } from './connectors/index.js';
import { GraphQLEngine } from './graphql/index.js';
import { RedisLikeCache } from './redis/index.js';
import { EventSourcingEngine } from './eventsourcing/index.js';
import { BlockchainEngine } from './blockchain/index.js';
import { DistributedComputingEngine } from './distributed/index.js';
import { StreamProcessor } from './streaming/processor.js';
import TerminalUI from './ui/index.js';
import PerformanceAnalytics from './analytics/performance.js';
import SecurityPrivacySuite from './security/privacy.js';
import CollectionManager from './collections/index.js';
import PerformanceEngine from './performance/index.js';
import AuthenticationManager from './auth/index.js';
import RESTAPIGenerator from './api/rest.js';
import RealtimeDashboard from './dashboard/realtime.js';
import ReplicationManager from './replication/manager.js';

const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);

/**
 * BigBaseAlpha - Professional Grade Custom Database System
 * Built entirely from scratch without external database dependencies
 */
export class BigBaseAlpha extends EventEmitter {
  constructor(options = {}) {
    super();
    
    // =============================================================================
    // v1.5.1 MODULAR LOGGING SYSTEM INITIALIZATION
    // =============================================================================
    
    // Initialize logger with user preferences or enterprise defaults
    const loggerConfig = options.logger || {};
    const loggerPreset = loggerConfig.preset || 'enterprise';
    
    // Apply preset configuration
    let loggerOptions = {};
    if (LoggerPresets[loggerPreset]) {
      loggerOptions = { ...LoggerPresets[loggerPreset] };
    }
    
    // Override with user-specific settings
    loggerOptions = {
      ...loggerOptions,
      silent: options.silent || false,
      prefix: loggerConfig.prefix || 'BigBaseAlpha',
      ...loggerConfig
    };
    
    // Create modular logger instance
    this.logger = new ModularLogger(loggerOptions);
    
    // Maintain backward compatibility
    this.silent = loggerOptions.silent;
    
    // =============================================================================
    // CONFIGURATION SETUP
    // =============================================================================
    
    // Default configuration with modular control
    this.config = {
      path: options.path || './bigbase_data',
      format: options.format || 'json',
      encryption: options.encryption || false,
      compression: options.compression || false,
      maxMemory: options.maxMemory || '512MB',
      backupInterval: options.backupInterval || 3600000,
      indexing: options.indexing !== false,
      caching: options.caching !== false,
      auditLog: options.auditLog !== false,
      ttlCleanup: options.ttlCleanup !== false,
      plugins: options.plugins || [],
      silent: this.silent,
      logger: this.logger,
      
      // v1.5.1 Logger Configuration
      logging: {
        format: loggerOptions.format,
        colors: loggerOptions.colors,
        timestamp: loggerOptions.timestamp,
        colorScheme: loggerOptions.colorScheme,
        ...loggerConfig
      },
      
      // Modular Features (Default: Disabled for Clean Installation)
      modules: {
        // Core features (always enabled)
        authentication: options.modules?.authentication !== false,
        hsm: options.modules?.hsm !== false, // HSM Security
        
        // Advanced features (disabled by default)
        apiGateway: options.modules?.apiGateway || false,
        machineLearning: options.modules?.machineLearning || false,
        replication: options.modules?.replication || false,
        monitoring: options.modules?.monitoring || false,
        databaseConnectors: options.modules?.databaseConnectors || false,
        graphql: options.modules?.graphql || false,
        eventSourcing: options.modules?.eventSourcing || false,
        blockchain: options.modules?.blockchain || false,
        streamProcessor: options.modules?.streamProcessor || false,
        
        // Optional convenience features
        restAPI: options.modules?.restAPI || false,
        realtimeDashboard: options.modules?.realtimeDashboard || false
      },
      
      ...options
    };

    // Initialize managers
    this.configManager = new ConfigManager(this.config);
    this.storage = new StorageEngine(this.config);
    this.security = new SecurityManager({ ...this.config, logger: this.logger });
    this.indexing = new IndexManager(this.config);
    this.cache = new CacheManager(this.config);
    this.plugins = new PluginManager(this.config);
    this.audit = new AuditLogger(this.config);
    this.auth = new AuthManager({ ...this.config, logger: this.logger });
    this.backupManager = new BackupManager({ ...this.config, logger: this.logger });
    this.searchEngine = new SearchEngine({ ...this.config, logger: this.logger });
    this.queryProfiler = new QueryProfiler({ ...this.config, logger: this.logger });
    this.etlEngine = new ETLEngine({ ...this.config, logger: this.logger });
    this.streamingEngine = new StreamingEngine({ ...this.config, logger: this.logger });
    this.analyticsEngine = new AnalyticsEngine({ ...this.config, logger: this.logger });
    
    // Conditional Module Initialization (Only if enabled)
    this.apiGateway = this.config.modules.apiGateway ? new APIGateway(this.config) : null;
    this.mlEngine = this.config.modules.machineLearning ? new MLEngine(this.config) : null;
    this.replicationEngine = this.config.modules.replication ? new ReplicationEngine(this.config) : null;
    this.monitoringEngine = this.config.modules.monitoring ? new MonitoringEngine(this.config) : null;
    this.databaseConnectors = this.config.modules.databaseConnectors ? new DatabaseConnectors(this.config) : null;
    this.graphqlEngine = this.config.modules.graphql ? new GraphQLEngine(this.config) : null;
    this.redisCache = new RedisLikeCache({ ...this.config, logger: this.logger });
    this.eventSourcing = this.config.modules.eventSourcing ? new EventSourcingEngine(this.config) : null;
    this.blockchain = this.config.modules.blockchain ? new BlockchainEngine(this.config) : null;
    this.streamProcessor = this.config.modules.streamProcessor ? new StreamProcessor(this.config) : null;
    
    // Initialize Offline HSM for Maximum Security (100% Offline)
    this.hsm = new OfflineHSM({
      secureStorePath: join(this.config.path, 'hsm'),
      keySize: 256,
      algorithm: 'aes-256-gcm',
      tamperDetection: true,
      autoBackup: true,
      maxKeyAge: 7776000000, // 90 days
      logger: this.logger // v1.5.1 logger integration
    });

    // Initialize Terminal UI and Performance Analytics
    this.ui = new TerminalUI({
      theme: this.config.ui?.theme || 'default',
      colors: this.config.ui?.colors !== false,
      animation: this.config.ui?.animation !== false
    });
    
    this.performance = new PerformanceAnalytics({
      sampleInterval: this.config.performance?.sampleInterval || 1000,
      maxSamples: this.config.performance?.maxSamples || 1000,
      enableProfiling: this.config.performance?.enableProfiling !== false
    });

    // Initialize Security & Privacy Suite
    this.securitySuite = new SecurityPrivacySuite(this, {
      encryption: this.config.security?.encryption || 'AES-256-GCM',
      wipeIterations: this.config.security?.wipeIterations || 3,
      paranoidLogging: this.config.security?.paranoidLogging !== false,
      logger: this.logger
    });

    // Initialize Collection Manager and Performance Engine
    this.collectionManager = new CollectionManager(this, {
      maxCollections: this.config.collections?.maxCollections || 1000,
      autoCreateCollections: this.config.collections?.autoCreateCollections !== false,
      strictMode: this.config.collections?.strictMode || false
    });

    this.performanceEngine = new PerformanceEngine(this, {
      lazyWriteDelay: this.config.performance?.lazyWriteDelay || 5000,
      batchSize: this.config.performance?.batchSize || 100,
      compressionEnabled: this.config.performance?.compressionEnabled !== false
    });

    // Initialize v1.5.0 new features
    this.authManager = new AuthenticationManager(this, {
      jwtSecret: this.config.auth?.jwtSecret,
      tokenExpiry: this.config.auth?.tokenExpiry || '24h',
      requireEmailVerification: this.config.auth?.requireEmailVerification || false,
      maxLoginAttempts: this.config.auth?.maxLoginAttempts || 5
    });

    this.restAPI = new RESTAPIGenerator(this, this.authManager, {
      port: this.config.api?.port || 3001,
      enableSwagger: this.config.api?.enableSwagger !== false,
      enableCors: this.config.api?.enableCors !== false,
      rateLimiting: this.config.api?.rateLimiting !== false
    });

    this.realtimeDashboard = new RealtimeDashboard(this, this.authManager, {
      wsPort: this.config.dashboard?.wsPort || 8080,
      updateInterval: this.config.dashboard?.updateInterval || 1000,
      enableMetrics: this.config.dashboard?.enableMetrics !== false,
      enableAlerts: this.config.dashboard?.enableAlerts !== false
    });

    this.replicationManager = new ReplicationManager(this, {
      role: this.config.replication?.role || 'master',
      port: this.config.replication?.port || 9000,
      slaves: this.config.replication?.slaves || [],
      masterHost: this.config.replication?.masterHost || 'localhost',
      masterPort: this.config.replication?.masterPort || 9000,
      enableCompression: this.config.replication?.enableCompression !== false
    });

    // State management
    this.isInitialized = false;
    this.collections = new Map();
    this.schemas = new Map();
    this.operationLocks = new Map();
    this.lazyWrite = false; // Performance mode flag

    this.stats = {
      totalOperations: 0,
      totalInserts: 0,
      totalReads: 0,
      totalUpdates: 0,
      totalDeletes: 0,
      startTime: null,
      lastBackup: null
    };

    // Bind event handlers
    this._bindEvents();
  }

  /**
   * Initialize the database system
   */
  async init() {
    try {
      this.emit('onInit', this);
      
      // Create data directory if it doesn't exist
      if (!existsSync(this.config.path)) {
        mkdirSync(this.config.path, { recursive: true });
      }

      // Initialize all managers
      await this.configManager.init(this.config);
      await this.security.init();
      await this.storage.init();
      await this.indexing.init();
      await this.cache.init();
      await this.audit.init();
      await this.plugins.init();
      
      // Initialize new managers
      await this.auth.init();
      await this.backupManager.init();
      await this.searchEngine.init();
      await this.queryProfiler.init();
      await this.etlEngine.init();
      
      // Streaming Engine - only init if enabled
      if (this.config.streaming?.enabled !== false) {
        await this.streamingEngine.init();
      }
      
      await this.analyticsEngine.init();
      
      // Conditional Module Initialization
      if (this.config.modules.apiGateway && this.apiGateway) {
        await this.apiGateway.init();
        this._logModule('[SUCCESS] API Gateway initialized on port', this.apiGateway.port);
      }
      
      if (this.config.modules.machineLearning && this.mlEngine) {
        await this.mlEngine.init();
        this._logModule('[SUCCESS] Machine Learning Engine initialized');
      }
      
      if (this.config.modules.replication && this.replicationEngine) {
        await this.replicationEngine.init();
        this._logModule('[PROCESS] Initializing Data Replication Engine...');
      } else {
        this._logModule('[WARN] Replication is disabled in configuration');
      }
      
      if (this.config.modules.monitoring && this.monitoringEngine) {
        await this.monitoringEngine.init();
        this._logModule('[SUCCESS] Advanced Monitoring System initialized');
      } else {
        this._logModule('[PROCESS] Initializing Advanced Monitoring System...');
      }
      
      if (this.config.modules.databaseConnectors && this.databaseConnectors) {
        await this.databaseConnectors.init();
        this._logModule('[SUCCESS] Database Connectors initialized');
      } else {
        this._logModule('[WARN] Database connectors are disabled in configuration');
      }
      
      if (this.config.modules.graphql && this.graphqlEngine) {
        await this.graphqlEngine.init();
        this._logModule('[SUCCESS] GraphQL Engine initialized');
      } else {
        this._logModule('[WARN] GraphQL is disabled in configuration');
      }
      
      // Initialize v1.5.1 features
      this.logger.process('Initializing BigBaseAlpha v1.5.1 features...');
      
      // Initialize Authentication Manager
      await this.authManager._initializeCollections();
      this.logger.success('Authentication system initialized');
      
      // Initialize REST API (but don't start server yet)
      this.logger.success('REST API generator ready');
      
      // Initialize Real-time Dashboard (but don't start WebSocket yet)
      this.logger.success('Real-time Dashboard ready');
      
      // Initialize Replication Manager (but don't start yet)
      this.logger.success('Replication Manager ready');
      
      // Redis Cache - always init (lightweight)
      await this.redisCache.init();
      
      // Event Sourcing - only init if enabled
      if (this.config.modules.eventSourcing && this.eventSourcing) {
        await this.eventSourcing.initialize();
        this._logModule('[SUCCESS] Event Sourcing Engine initialized');
      }
      
      // Blockchain - only init if enabled
      if (this.config.modules.blockchain && this.blockchain) {
        await this.blockchain.initialize();
        this._logModule('[SUCCESS] Blockchain Engine initialized');
      }
      
      this.backupManager.setDatabase(this);
      this.etlEngine.setDatabase(this);
      
      // Set database for optional components only if enabled
      if (this.config.streaming?.enabled !== false) {
        this.streamingEngine.setDatabase(this);
      }
      
      this.analyticsEngine.setDatabase(this);
      
      // Conditional setDatabase calls for optional modules
      if (this.apiGateway) {
        this.apiGateway.setDatabase(this);
      }
      
      if (this.mlEngine) {
        this.mlEngine.setDatabase(this);
      }
      
      if (this.replicationEngine) {
        this.replicationEngine.setDatabase(this);
      }
      
      if (this.monitoringEngine) {
        this.monitoringEngine.setDatabase(this);
      }
      
      if (this.databaseConnectors) {
        this.databaseConnectors.setDatabase(this);
      }
      
      if (this.graphqlEngine) {
        this.graphqlEngine.setDatabase(this);
      }
      
      this.redisCache.setDatabase(this);
      
      if (this.eventSourcing) {
        this.eventSourcing.setDatabase(this);
      }
      
      if (this.blockchain) {
        this.blockchain.setDatabase(this);
      }
      
      if (this.streamProcessor) {
        this.streamProcessor.setDatabase(this);
      }

      // Load existing collections
      await this._loadCollections();

      // Start background tasks
      this._startBackgroundTasks();

      this.isInitialized = true;
      this.stats.startTime = new Date();

      this.audit.log('system', 'init', { 
        config: this.config,
        timestamp: new Date()
      });

      this.emit('initialized', this);
      return this;

    } catch (error) {
      this.emit('error', error);
      throw new Error(`Failed to initialize BigBaseAlpha: ${error.message}`);
    }
  }

  /**
   * Create a new collection with optional schema
   */
  async createCollection(name, schema = null) {
    this._ensureInitialized();
    
    if (this.collections.has(name)) {
      throw new Error(`Collection '${name}' already exists`);
    }

    const collection = {
      name,
      schema,
      created: new Date(),
      documents: new Map(),
      metadata: {
        totalDocuments: 0,
        totalSize: 0,
        lastModified: new Date()
      }
    };

    this.collections.set(name, collection);
    if (schema) {
      this.schemas.set(name, schema);
    }

    // Create storage structure
    await this.storage.createCollection(name);
    
    // Create indexes
    if (this.config.indexing) {
      await this.indexing.createIndexes(name, schema);
    }

    this.audit.log('collection', 'create', { name, schema });
    this.emit('collectionCreated', { name, schema });

    return collection;
  }

  /**
   * Insert a document into a collection
   */
  async insert(collectionName, data) {
    return this._withLock(`insert:${collectionName}`, async () => {
      this._ensureInitialized();
      this._ensureCollection(collectionName);

    const queryId = this._generateId();
    const profile = this.queryProfiler.startQuery(queryId, {
      collection: collectionName,
      operation: 'insert',
      query: { data: this._sanitizeForProfiling(data) }
    });      try {
        // Generate unique ID if not provided
        if (!data._id) {
          data._id = this._generateId();
        }

        // Add metadata
        data._created = new Date();
        data._modified = new Date();

        // Validate against schema if exists
        if (this.schemas.has(collectionName)) {
          this._validateSchema(data, this.schemas.get(collectionName));
        }

        // Encrypt sensitive fields if needed
        if (this.config.encryption) {
          data = await this.security.encryptDocument(data);
        }

        // Store in collection
        const collection = this.collections.get(collectionName);
        collection.documents.set(data._id, data);
        collection.metadata.totalDocuments++;
        collection.metadata.lastModified = new Date();

        // Persist to storage
        await this.storage.insert(collectionName, data);

        // Update indexes
        if (this.config.indexing) {
          await this.indexing.addToIndex(collectionName, data);
        }

        // Index for search
        await this.searchEngine.indexDocument(collectionName, data);

        // Update cache
        if (this.config.caching) {
          this.cache.set(`${collectionName}:${data._id}`, data);
        }

        // Update statistics
        this.stats.totalOperations++;
        this.stats.totalInserts++;

        // Emit events
        this.emit('document:inserted', { collectionName, document: data });

        // Audit log
        if (this.config.auditLog) {
          await this.audit.log('INSERT', { collection: collectionName, document: data._id });
        }

        this.queryProfiler.endQuery(queryId, data);
        return data;
      } catch (error) {
        this.queryProfiler.endQuery(queryId, null, error);
        throw error;
      }
    });
  }

  /**
   * Find a document by ID
   */
  async findById(collectionName, id) {
    this._ensureInitialized();
    this._ensureCollection(collectionName);

    // Check cache first
    if (this.config.caching) {
      const cached = this.cache.get(`${collectionName}:${id}`);
      if (cached) {
        this.stats.totalReads++;
        return this._decryptDocument(cached);
      }
    }

    // Check in-memory collection
    const collection = this.collections.get(collectionName);
    if (collection.documents.has(id)) {
      const doc = collection.documents.get(id);
      
      // Update cache
      if (this.config.caching) {
        this.cache.set(`${collectionName}:${id}`, doc);
      }

      this.stats.totalReads++;
      return this._decryptDocument(doc);
    }

    // Load from storage
    const doc = await this.storage.findById(collectionName, id);
    if (doc) {
      // Add to in-memory collection
      collection.documents.set(id, doc);
      
      // Update cache
      if (this.config.caching) {
        this.cache.set(`${collectionName}:${id}`, doc);
      }

      this.stats.totalReads++;
      return this._decryptDocument(doc);
    }

    return null;
  }

  /**
   * Update a document
   */
  async update(collectionName, id, updateData) {
    this._ensureInitialized();
    this._ensureCollection(collectionName);

    const existingDoc = await this.findById(collectionName, id);
    if (!existingDoc) {
      throw new Error(`Document with id '${id}' not found in collection '${collectionName}'`);
    }

    // Merge update data
    const updatedDoc = {
      ...existingDoc,
      ...updateData,
      _id: id, // Preserve ID
      _created: existingDoc._created, // Preserve creation date
      _modified: new Date()
    };

    // Validate against schema if exists
    if (this.schemas.has(collectionName)) {
      this._validateSchema(updatedDoc, this.schemas.get(collectionName));
    }

    // Encrypt if needed
    let docToStore = updatedDoc;
    if (this.config.encryption) {
      docToStore = await this.security.encryptDocument(updatedDoc);
    }

    // Update in collection
    const collection = this.collections.get(collectionName);
    collection.documents.set(id, docToStore);
    collection.metadata.lastModified = new Date();

    // Persist to storage
    await this.storage.update(collectionName, id, docToStore);

    // Update indexes
    if (this.config.indexing) {
      await this.indexing.updateIndex(collectionName, existingDoc, updatedDoc);
    }

    // Update cache
    if (this.config.caching) {
      this.cache.set(`${collectionName}:${id}`, docToStore);
    }

    this.stats.totalOperations++;
    this.stats.totalUpdates++;

    this.audit.log('document', 'update', { 
      collection: collectionName, 
      id,
      changes: Object.keys(updateData)
    });

    this.emit('documentUpdated', { 
      collection: collectionName, 
      id, 
      document: updatedDoc,
      changes: updateData
    });

    return updatedDoc;
  }

  /**
   * Delete a document
   */
  async delete(collectionName, id) {
    this._ensureInitialized();
    this._ensureCollection(collectionName);

    const doc = await this.findById(collectionName, id);
    if (!doc) {
      return false;
    }

    // Remove from collection
    const collection = this.collections.get(collectionName);
    collection.documents.delete(id);
    collection.metadata.totalDocuments--;
    collection.metadata.lastModified = new Date();

    // Remove from storage
    await this.storage.delete(collectionName, id);

    // Remove from indexes
    if (this.config.indexing) {
      await this.indexing.removeFromIndex(collectionName, doc);
    }

    // Remove from cache
    if (this.config.caching) {
      this.cache.delete(`${collectionName}:${id}`);
    }

    this.stats.totalOperations++;
    this.stats.totalDeletes++;

    this.audit.log('document', 'delete', { 
      collection: collectionName, 
      id 
    });

    this.emit('onDelete', collectionName, id);
    this.emit('documentDeleted', { collection: collectionName, id });

    return true;
  }

  /**
   * Find one document matching the query
   */
  async findOne(collectionName, query = {}) {
    this._ensureInitialized();
    this._ensureCollection(collectionName);
    
    // Use query with limit 1
    const results = await this.query(collectionName, {
      where: query,
      limit: 1
    });

    return results.length > 0 ? results[0] : null;
  }

  /**
   * Find multiple documents
   */
  async find(collectionName, query = {}, options = {}) {
    return this.query(collectionName, {
      where: query,
      ...options
    });
  }

  /**
   * Advanced query with filtering, sorting, and pagination
   */
  async query(collectionName, options = {}) {
    this._ensureInitialized();
    this._ensureCollection(collectionName);

    const queryId = this._generateId();
    const profile = this.queryProfiler.startQuery(queryId, {
      collection: collectionName,
      operation: 'query',
      query: this._sanitizeForProfiling(options.where || {}),
      options: { ...options, where: undefined } // Don't duplicate where clause
    });

    try {
      const {
        where = {},
        sort = {},
        limit = null,
        offset = 0,
        select = null
      } = options;

      let results = [];

      // Try to use indexes if available and query has conditions
      let useIndex = false;
      if (this.config.indexing && Object.keys(where).length > 0) {
        const indexResults = await this.indexing.query(collectionName, where);
        
        if (indexResults.length > 0) {
          results = indexResults;
          useIndex = true;
        }
      }

      // If index query failed or no index available, do full scan
      if (!useIndex) {
        const collection = this.collections.get(collectionName);
        results = Array.from(collection.documents.values());
      }
      
      // Apply where clause (only if not already filtered by index)
      if (Object.keys(where).length > 0 && !useIndex) {
        results = results.filter(doc => this._matchesQuery(doc, where));
      }

      // Apply sorting
      if (Object.keys(sort).length > 0) {
        results = this._sortResults(results, sort);
      }

      // Apply pagination
      if (offset > 0) {
        results = results.slice(offset);
      }
      if (limit !== null) {
        results = results.slice(0, limit);
      }

      // Apply field selection
      if (select) {
        results = results.map(doc => this._selectFields(doc, select));
      }

      // Decrypt documents
      results = await Promise.all(
        results.map(doc => this._decryptDocument(doc))
      );

      this.stats.totalReads++;
      this.queryProfiler.endQuery(queryId, results);
      
      return results;
    } catch (error) {
      this.queryProfiler.endQuery(queryId, null, error);
      throw error;
    }
  }

  /**
   * Get database statistics
   */
  getStats() {
    const uptime = this.stats.startTime ? 
      Date.now() - this.stats.startTime.getTime() : 0;

    return {
      ...this.stats,
      uptime,
      collections: this.collections.size,
      memoryUsage: process.memoryUsage(),
      cacheStats: this.cache.getStats(),
      storageStats: this.storage.getStats()
    };
  }

  /**
   * Get all collection names
   */
  getCollections() {
    return Array.from(this.collections.keys());
  }

  /**
   * List all collection names (alias for getCollections)
   */
  listCollections() {
    return this.getCollections();
  }

  /**
   * Get a collection object for chaining operations
   */
  collection(name) {
    this._ensureCollection(name);
    return {
      find: (query = {}, options = {}) => this.find(name, query, options),
      findOne: (query = {}) => this.findOne(name, query),
      insert: (data) => this.insert(name, data),
      update: (id, data) => this.update(name, id, data),
      delete: (id) => this.delete(name, id),
      count: (query = {}) => {
        const collection = this.collections.get(name);
        if (!query || Object.keys(query).length === 0) {
          return collection.documents.size;
        }
        // Count with filter
        let count = 0;
        for (const doc of collection.documents.values()) {
          if (this._matchesQuery(doc, query)) count++;
        }
        return count;
      }
    };
  }

  /**
   * Backup database
   */
  async backup(path = null) {
    this._ensureInitialized();
    
    const backupPath = path || `./backups/backup_${Date.now()}.bba`;
    await this.storage.backup(backupPath);
    
    this.stats.lastBackup = new Date();
    this.audit.log('system', 'backup', { path: backupPath });
    this.emit('onBackup', backupPath);
    
    return backupPath;
  }

  /**
   * Full-text search across collections
   */
  async search(collectionName, query, options = {}) {
    this._ensureInitialized();
    this._ensureCollection(collectionName);

    const queryId = this._generateId();
    const profile = this.queryProfiler.startQuery(queryId, {
      collection: collectionName,
      operation: 'search',
      query: { searchQuery: query, options }
    });

    try {
      const results = await this.searchEngine.search(collectionName, query, options);
      
      this.queryProfiler.endQuery(queryId, results);
      this.stats.totalReads++;
      
      return results;
    } catch (error) {
      this.queryProfiler.endQuery(queryId, null, error);
      throw error;
    }
  }

  /**
   * Get search suggestions/autocomplete
   */
  async suggest(collectionName, partial, options = {}) {
    this._ensureInitialized();
    this._ensureCollection(collectionName);
    
    return await this.searchEngine.suggest(collectionName, partial, options);
  }

  /**
   * Index a document for full-text search
   */
  async indexForSearch(collectionName, document, searchableFields = null) {
    this._ensureInitialized();
    this._ensureCollection(collectionName);
    
    await this.searchEngine.indexDocument(collectionName, document, searchableFields);
  }

  /**
   * Get search engine statistics
   */
  getSearchStats(collectionName = null) {
    this._ensureInitialized();
    return this.searchEngine.getStats(collectionName);
  }

  /**
   * Get query profiler statistics
   */
  getProfilerStats(options = {}) {
    this._ensureInitialized();
    return this.queryProfiler.getStats(options);
  }

  /**
   * Get slow queries from profiler
   */
  getSlowQueries(options = {}) {
    this._ensureInitialized();
    return this.queryProfiler.getSlowQueries(options);
  }

  /**
   * Analyze query patterns and get recommendations
   */
  analyzeQueryPatterns(options = {}) {
    this._ensureInitialized();
    return this.queryProfiler.analyzePatterns(options);
  }

  /**
   * Get real-time query metrics
   */
  getRealTimeQueryMetrics() {
    this._ensureInitialized();
    return this.queryProfiler.getRealTimeMetrics();
  }

  /**
   * Export query profiles
   */
  async exportQueryProfiles(format = 'json', options = {}) {
    this._ensureInitialized();
    return await this.queryProfiler.exportProfiles(format, options);
  }

  // === ETL & DATA PIPELINE METHODS ===

  /**
   * Create a new ETL pipeline
   */
  async createETLPipeline(config) {
    this._ensureInitialized();
    return await this.etlEngine.createPipeline(config);
  }

  /**
   * Execute an ETL pipeline
   */
  async executeETLPipeline(pipelineId, options = {}) {
    this._ensureInitialized();
    return await this.etlEngine.executePipeline(pipelineId, options);
  }

  /**
   * Get all ETL pipelines
   */
  getETLPipelines() {
    this._ensureInitialized();
    return this.etlEngine.getPipelines();
  }

  /**
   * Get specific ETL pipeline
   */
  getETLPipeline(id) {
    this._ensureInitialized();
    return this.etlEngine.getPipeline(id);
  }

  /**
   * Get active ETL pipelines
   */
  getActiveETLPipelines() {
    this._ensureInitialized();
    return this.etlEngine.getActivePipelines();
  }

  /**
   * Get ETL job history
   */
  getETLJobHistory(limit = 50) {
    this._ensureInitialized();
    return this.etlEngine.getJobHistory(limit);
  }

  /**
   * Get ETL statistics
   */
  getETLStats() {
    this._ensureInitialized();
    return this.etlEngine.getStats();
  }

  /**
   * Import data from CSV file
   */
  async importFromCSV(collectionName, filePath, options = {}) {
    this._ensureInitialized();
    
    const pipeline = await this.createETLPipeline({
      name: `CSV Import - ${collectionName}`,
      description: `Import data from ${filePath} to ${collectionName}`,
      source: {
        type: 'csv',
        path: filePath
      },
      destination: {
        type: 'collection',
        collection: collectionName
      },
      transformations: options.transformations || [],
      validations: options.validations || []
    });

    return await this.executeETLPipeline(pipeline.id);
  }

  /**
   * Export data to CSV file
   */
  async exportToCSV(collectionName, filePath, options = {}) {
    this._ensureInitialized();
    
    const pipeline = await this.createETLPipeline({
      name: `CSV Export - ${collectionName}`,
      description: `Export data from ${collectionName} to ${filePath}`,
      source: {
        type: 'collection',
        collection: collectionName,
        query: options.query || {},
        options: options.queryOptions || {}
      },
      destination: {
        type: 'csv',
        path: filePath
      },
      transformations: options.transformations || []
    });

    return await this.executeETLPipeline(pipeline.id);
  }

  /**
   * Close database connection
   */
  async close() {
    if (!this.isInitialized) {
      return;
    }

    // Stop background tasks
    this._stopBackgroundTasks();

    // Close all managers
    await this.storage.close();
    await this.cache.close();
    await this.indexing.close();
    await this.plugins.close();
    await this.searchEngine.close();
    await this.queryProfiler.close();
    await this.etlEngine.close();

    // Stop backup manager
    if (this.backupManager) {
      this.backupManager.destroy();
    }

    this.isInitialized = false;
    this.emit('closed');
    
    this.logger.process('Closing BigBaseAlpha database...');
    this.logger.process('Closing collection manager...');
    this.logger.success('BigBaseAlpha database closed successfully');
    
    // Force process exit after a brief delay to ensure cleanup
    if (!this.config.silent) {
      setTimeout(() => {
        if (typeof process !== 'undefined' && process.exit) {
          process.exit(0);
        }
      }, 100);
    }
  }

  // Private methods
  _ensureInitialized() {
    if (!this.isInitialized) {
      throw new Error('Database not initialized. Call init() first.');
    }
  }

  _ensureCollection(name) {
    if (!this.collections.has(name)) {
      throw new Error(`Collection '${name}' does not exist. Create it first.`);
    }
  }

  _generateId() {
    return `${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
  }

  _sanitizeForProfiling(data) {
    // Remove sensitive data for profiling logs
    if (!data || typeof data !== 'object') return data;
    
    const sanitized = JSON.parse(JSON.stringify(data));
    const sensitiveFields = ['password', 'token', 'secret', 'key', 'auth'];
    
    function sanitizeObject(obj) {
      if (typeof obj !== 'object' || obj === null) return;
      
      for (const key of Object.keys(obj)) {
        if (sensitiveFields.some(field => key.toLowerCase().includes(field))) {
          obj[key] = '[SANITIZED]';
        } else if (typeof obj[key] === 'object') {
          sanitizeObject(obj[key]);
        }
      }
    }
    
    sanitizeObject(sanitized);
    return sanitized;
  }

  _validateSchema(data, schema) {
    // Basic schema validation
    for (const [field, rules] of Object.entries(schema)) {
      if (rules.required && !(field in data)) {
        throw new Error(`Required field '${field}' is missing`);
      }
      
      if (field in data && rules.type && typeof data[field] !== rules.type) {
        throw new Error(`Field '${field}' must be of type '${rules.type}'`);
      }
    }
  }

  async _decryptDocument(doc) {
    if (this.config.encryption && doc) {
      return await this.security.decryptDocument(doc);
    }
    return doc;
  }

  _matchesQuery(doc, query) {
    // Handle logical operators first
    if (query.$and) {
      return query.$and.every(condition => this._matchesQuery(doc, condition));
    }
    
    if (query.$or) {
      return query.$or.some(condition => this._matchesQuery(doc, condition));
    }
    
    if (query.$not) {
      return !this._matchesQuery(doc, query.$not);
    }

    // Simple query matching - can be extended
    for (const [field, condition] of Object.entries(query)) {
      // Skip logical operators (already handled above)
      if (field.startsWith('$')) continue;
      
      if (typeof condition === 'object' && condition !== null) {
        // Handle operators like $gt, $lt, $eq, etc.
        for (const [op, value] of Object.entries(condition)) {
          switch (op) {
            case '$eq':
              if (doc[field] !== value) return false;
              break;
            case '$gt':
              if (doc[field] <= value) return false;
              break;
            case '$lt':
              if (doc[field] >= value) return false;
              break;
            case '$gte':
              if (doc[field] < value) return false;
              break;
            case '$lte':
              if (doc[field] > value) return false;
              break;
            case '$in':
              if (!Array.isArray(value) || !value.includes(doc[field])) return false;
              break;
            case '$nin':
              if (Array.isArray(value) && value.includes(doc[field])) return false;
              break;
            case '$regex':
              if (typeof doc[field] !== 'string') return false;
              const regex = new RegExp(value.source || value, value.flags || 'i');
              if (!regex.test(doc[field])) return false;
              break;
            case '$exists':
              if ((doc[field] !== undefined) !== value) return false;
              break;
            case '$ne':
              if (doc[field] === value) return false;
              break;
            default:
              throw new Error(`Unknown operator: ${op}`);
          }
        }
      } else {
        // Direct equality
        if (doc[field] !== condition) return false;
      }
    }
    return true;
  }

  _sortResults(results, sort) {
    return results.sort((a, b) => {
      for (const [field, direction] of Object.entries(sort)) {
        const aVal = a[field];
        const bVal = b[field];
        
        if (aVal < bVal) return direction === 1 ? -1 : 1;
        if (aVal > bVal) return direction === 1 ? 1 : -1;
      }
      return 0;
    });
  }

  _selectFields(doc, fields) {
    if (Array.isArray(fields)) {
      const selected = {};
      fields.forEach(field => {
        if (field in doc) {
          selected[field] = doc[field];
        }
      });
      return selected;
    }
    return doc;
  }

  async _loadCollections() {
    const collections = await this.storage.listCollections();
    for (const collectionName of collections) {
      const collection = {
        name: collectionName,
        schema: null,
        created: new Date(),
        documents: new Map(),
        metadata: {
          totalDocuments: 0,
          totalSize: 0,
          lastModified: new Date()
        }
      };
      
      // Load existing documents from storage
      try {
        const documents = await this.storage.listDocuments(collectionName);
        for (const doc of documents) {
          collection.documents.set(doc._id, doc);
          collection.metadata.totalDocuments++;
        }
        this.logger.success(`Loaded ${collection.metadata.totalDocuments} documents from '${collectionName}'`);
      } catch (error) {
        this.logger.warn(`Could not load documents from '${collectionName}':`, error.message);
      }
      
      this.collections.set(collectionName, collection);
    }
  }

  _bindEvents() {
    // Plugin event forwarding
    this.on('onInit', (...args) => this.plugins.emit('onInit', ...args));
    this.on('onWrite', (...args) => this.plugins.emit('onWrite', ...args));
    this.on('onDelete', (...args) => this.plugins.emit('onDelete', ...args));
    this.on('onBackup', (...args) => this.plugins.emit('onBackup', ...args));
  }

  _startBackgroundTasks() {
    // Auto backup
    if (this.config.backupInterval > 0) {
      this.backupTimer = setInterval(() => {
        this.backup().catch(err => this.emit('error', err));
      }, this.config.backupInterval);
    }

    // TTL cleanup
    if (this.config.ttlCleanup) {
      this.ttlTimer = setInterval(() => {
        this._cleanupExpiredDocuments().catch(err => this.emit('error', err));
      }, 60000); // Every minute
    }

    // Cache cleanup
    if (this.config.caching) {
      this.cacheTimer = setInterval(() => {
        this.cache.cleanup();
      }, 300000); // Every 5 minutes
    }
  }

  _stopBackgroundTasks() {
    // Clear all timers and intervals
    if (this.backupTimer) {
      clearInterval(this.backupTimer);
      this.backupTimer = null;
    }
    if (this.ttlTimer) {
      clearInterval(this.ttlTimer);
      this.ttlTimer = null;
    }
    if (this.cacheTimer) {
      clearInterval(this.cacheTimer);
      this.cacheTimer = null;
    }
    
    // Clear any other background timers
    if (this.performanceTimer) {
      clearInterval(this.performanceTimer);
      this.performanceTimer = null;
    }
    if (this.healthCheckTimer) {
      clearInterval(this.healthCheckTimer);
      this.healthCheckTimer = null;
    }
    if (this.metricsTimer) {
      clearInterval(this.metricsTimer);
      this.metricsTimer = null;
    }
    
    this.logger.success('All background tasks stopped');
  }

  async _cleanupExpiredDocuments() {
    // Clean up documents with TTL
    for (const [collectionName, collection] of this.collections) {
      const expired = [];
      for (const [id, doc] of collection.documents) {
        if (doc._ttl && new Date() > new Date(doc._ttl)) {
          expired.push(id);
        }
      }
      
      for (const id of expired) {
        await this.delete(collectionName, id);
      }
    }
  }

  /**
   * Concurrent operation management
   */
  async _acquireLock(resource) {
    if (!this.operationLocks.has(resource)) {
      this.operationLocks.set(resource, Promise.resolve());
    }
    
    const currentLock = this.operationLocks.get(resource);
    let resolveLock;
    
    const newLock = new Promise(resolve => {
      resolveLock = resolve;
    });
    
    this.operationLocks.set(resource, newLock);
    
    await currentLock;
    return resolveLock;
  }

  async _withLock(resource, operation) {
    const releaseLock = await this._acquireLock(resource);
    try {
      return await operation();
    } finally {
      releaseLock();
    }
  }

  // ===== REAL-TIME STREAMING METHODS =====

  /**
   * Get streaming statistics
   */
  getStreamingStats() {
    this._ensureInitialized();
    return this.streamingEngine.getStats();
  }

  /**
   * Get active streaming connections
   */
  getStreamingConnections() {
    this._ensureInitialized();
    return this.streamingEngine.getConnections();
  }

  /**
   * Create data stream
   */
  createDataStream(name, source, options = {}) {
    this._ensureInitialized();
    return this.streamingEngine.createStream(name, source, options);
  }

  /**
   * Broadcast event to all streaming connections
   */
  broadcastEvent(eventType, data) {
    this._ensureInitialized();
    this.streamingEngine._broadcastEvent(eventType, data);
  }

  // ===== ADVANCED ANALYTICS METHODS =====

  /**
   * Generate analytics report
   */
  async generateAnalyticsReport(type, options = {}) {
    this._ensureInitialized();
    return await this.analyticsEngine.generateReport(type, options);
  }

  /**
   * Detect anomalies in collection data
   */
  async detectAnomalies(collection, field, options = {}) {
    this._ensureInitialized();
    return await this.analyticsEngine.detectAnomalies(collection, field, options);
  }

  /**
   * Generate predictions
   */
  async generatePredictions(type, data, options = {}) {
    this._ensureInitialized();
    return await this.analyticsEngine.generatePredictions(type, data, options);
  }

  /**
   * Get insights for collection
   */
  async getCollectionInsights(collection, options = {}) {
    this._ensureInitialized();
    return await this.analyticsEngine.getInsights(collection, options);
  }

  /**
   * Create analytics dashboard
   */
  createAnalyticsDashboard(name, config) {
    this._ensureInitialized();
    return this.analyticsEngine.createDashboard(name, config);
  }

  /**
   * Get analytics statistics
   */
  getAnalyticsStats() {
    this._ensureInitialized();
    return this.analyticsEngine.getStats();
  }

  // ===== API GATEWAY & MICROSERVICES METHODS =====

  /**
   * Register microservice
   */
  registerMicroservice(name, config) {
    this._ensureInitialized();
    return this.apiGateway.registerService(name, config);
  }

  /**
   * Register API route
   */
  registerAPIRoute(path, config) {
    this._ensureInitialized();
    return this.apiGateway.registerRoute(path, config);
  }

  /**
   * Generate API key
   */
  generateAPIKey(name, permissions = []) {
    this._ensureInitialized();
    return this.apiGateway.generateAPIKey(name, permissions);
  }

  /**
   * Get gateway statistics
   */
  getGatewayStats() {
    this._ensureInitialized();
    return this.apiGateway.getStats();
  }

  /**
   * Get registered services
   */
  getRegisteredServices() {
    this._ensureInitialized();
    return this.apiGateway.getServices();
  }

  // ===== MACHINE LEARNING METHODS =====

  /**
   * Create and train ML model
   */
  async createMLModel(name, algorithm, dataset, options = {}) {
    this._ensureInitialized();
    return await this.mlEngine.createModel(name, algorithm, dataset, options);
  }

  /**
   * Make ML prediction
   */
  async predict(modelId, input, options = {}) {
    this._ensureInitialized();
    return await this.mlEngine.predict(modelId, input, options);
  }

  /**
   * Detect patterns in collection
   */
  async detectDataPatterns(collection, options = {}) {
    this._ensureInitialized();
    return await this.mlEngine.detectPatterns(collection, options);
  }

  /**
   * Generate ML recommendations
   */
  async generateMLRecommendations(type, context, options = {}) {
    this._ensureInitialized();
    return await this.mlEngine.generateRecommendations(type, context, options);
  }

  /**
   * Analyze sentiment of text
   */
  analyzeSentiment(text) {
    this._ensureInitialized();
    return this.mlEngine.analyzeSentiment(text);
  }

  /**
   * Get ML statistics
   */
  getMLStats() {
    this._ensureInitialized();
    return this.mlEngine.getStats();
  }

  // ===== DATA REPLICATION METHODS =====

  /**
   * Get replication status
   */
  getReplicationStatus() {
    this._ensureInitialized();
    return this.replicationEngine.getReplicationStatus();
  }

  /**
   * Get connected replication nodes
   */
  getReplicationNodes() {
    this._ensureInitialized();
    return this.replicationEngine.ge