jexidb/src/Database.mjs

JexiDB is a pure JS NPM library for managing data on disk efficiently, without the need for a server.

import { EventEmitter } from 'events'
import IndexManager from './managers/IndexManager.mjs'
import Serializer from './Serializer.mjs'
import { Mutex } from 'async-mutex'
import fs from 'fs'
import readline from 'readline'
import pRetry from 'p-retry'
import { OperationQueue } from './OperationQueue.mjs'

/**
 * IterateEntry class for intuitive API with automatic change detection
 * Uses native JavaScript setters for maximum performance
 */
class IterateEntry {
  constructor(entry, originalRecord) {
    this._entry = entry
    this._originalRecord = originalRecord
    this._modified = false
    this._markedForDeletion = false
  }
  
  // Generic getter that returns values from the original entry
  get(property) {
    return this._entry[property]
  }
  
  // Generic setter that sets values in the original entry
  set(property, value) {
    this._entry[property] = value
    this._modified = true
  }
  
  // Delete method for intuitive deletion
  delete() {
    this._markedForDeletion = true
    return true
  }
  
  // Getter for the underlying entry (for compatibility)
  get value() {
    return this._entry
  }
  
  // Check if entry was modified
  get isModified() {
    return this._modified
  }
  
  // Check if entry is marked for deletion
  get isMarkedForDeletion() {
    return this._markedForDeletion
  }
  
  // Proxy all property access to the underlying entry
  get [Symbol.toPrimitive]() {
    return this._entry
  }
  
  // Handle property access dynamically
  get [Symbol.toStringTag]() {
    return 'IterateEntry'
  }
}

// Import managers
import FileHandler from './FileHandler.mjs'
import { QueryManager } from './managers/QueryManager.mjs'
import { ConcurrencyManager } from './managers/ConcurrencyManager.mjs'
import { StatisticsManager } from './managers/StatisticsManager.mjs'
import StreamingProcessor from './managers/StreamingProcessor.mjs'
import TermManager from './managers/TermManager.mjs'

/**
 * InsertSession - Simple batch insertion without memory duplication
 */
class InsertSession {
  constructor(database, sessionOptions = {}) {
    this.database = database
    this.batchSize = sessionOptions.batchSize || 100
    this.enableAutoSave = sessionOptions.enableAutoSave !== undefined ? sessionOptions.enableAutoSave : true
    this.totalInserted = 0
    this.flushing = false
    this.batches = [] // Array of batches to avoid slice() in flush()
    this.currentBatch = [] // Current batch being filled
    this.sessionId = Math.random().toString(36).substr(2, 9)
    
    // Track pending auto-flush operations
    this.pendingAutoFlushes = new Set()
    
    // Register this session as active
    this.database.activeInsertSessions.add(this)
  }

  async add(record) {
    // CRITICAL FIX: Remove the committed check to allow auto-reusability
    // The session should be able to handle multiple commits
    
    if (this.database.destroyed) {
      throw new Error('Database is destroyed')
    }
    
    // Process record
    const finalRecord = { ...record }
    const id = finalRecord.id || this.database.generateId()
    finalRecord.id = id
    
    // Add to current batch
    this.currentBatch.push(finalRecord)
    this.totalInserted++
    
    // If batch is full, move it to batches array and trigger auto-flush
    if (this.currentBatch.length >= this.batchSize) {
      this.batches.push(this.currentBatch)
      this.currentBatch = []
      
      // Auto-flush in background (non-blocking)
      // This ensures batches are flushed automatically without blocking add()
      this.autoFlush().catch(err => {
        // Log error but don't throw - we don't want to break the add() flow
        console.error('Auto-flush error in InsertSession:', err)
      })
    }
    
    return finalRecord
  }

  async autoFlush() {
    // Only flush if not already flushing
    // This method will process all pending batches
    if (this.flushing) return
    
    // Create a promise for this auto-flush operation
    const flushPromise = this._doFlush()
    this.pendingAutoFlushes.add(flushPromise)
    
    // Remove from pending set when complete (success or error)
    flushPromise
      .then(() => {
        this.pendingAutoFlushes.delete(flushPromise)
      })
      .catch((err) => {
        this.pendingAutoFlushes.delete(flushPromise)
        throw err
      })
    
    return flushPromise
  }

  async _doFlush() {
    // Check if database is destroyed or closed before starting
    if (this.database.destroyed || this.database.closed) {
      // Clear batches if database is closed/destroyed
      this.batches = []
      this.currentBatch = []
      return
    }

    // Prevent concurrent flushes - if already flushing, wait for it
    if (this.flushing) {
      // Wait for the current flush to complete
      while (this.flushing) {
        await new Promise(resolve => setTimeout(resolve, 1))
      }
      // After waiting, check if there's anything left to flush
      // If another flush completed everything, we're done
      if (this.batches.length === 0 && this.currentBatch.length === 0) return
      
      // Check again if database was closed during wait
      if (this.database.destroyed || this.database.closed) {
        this.batches = []
        this.currentBatch = []
        return
      }
    }
    
    this.flushing = true

    try {
      // Process continuously until queue is completely empty
      // This handles the case where new data is added during the flush
      while (this.batches.length > 0 || this.currentBatch.length > 0) {
        // Check if database was closed during processing
        if (this.database.destroyed || this.database.closed) {
          // Clear remaining batches
          this.batches = []
          this.currentBatch = []
          return
        }

        // Process all complete batches that exist at this moment
        // Note: new batches may be added to this.batches during this loop
        const batchesToProcess = this.batches.length
        for (let i = 0; i < batchesToProcess; i++) {
          // Check again before each batch
          if (this.database.destroyed || this.database.closed) {
            this.batches = []
            this.currentBatch = []
            return
          }
          
          const batch = this.batches.shift() // Remove from front
          await this.database.insertBatch(batch)
        }

        // Process current batch if it has data
        // Note: new records may be added to currentBatch during processing
        if (this.currentBatch.length > 0) {
          // Check if database was closed
          if (this.database.destroyed || this.database.closed) {
            this.batches = []
            this.currentBatch = []
            return
          }

          // Check if currentBatch reached batchSize during processing
          if (this.currentBatch.length >= this.batchSize) {
            // Move it to batches array and process in next iteration
            this.batches.push(this.currentBatch)
            this.currentBatch = []
            continue
          }
          
          // Process the current batch
          const batchToProcess = this.currentBatch
          this.currentBatch = [] // Clear before processing to allow new adds
          await this.database.insertBatch(batchToProcess)
        }
      }
    } finally {
      this.flushing = false
    }
  }

  async flush() {
    // Wait for any pending auto-flushes to complete first
    await this.waitForAutoFlushes()
    
    // Then do a final flush to ensure everything is processed
    await this._doFlush()
  }

  async waitForAutoFlushes() {
    // Wait for all pending auto-flush operations to complete
    if (this.pendingAutoFlushes.size > 0) {
      await Promise.all(Array.from(this.pendingAutoFlushes))
    }
  }

  async commit() {
    // CRITICAL FIX: Make session auto-reusable by removing committed state
    // Allow multiple commits on the same session
    
    // First, wait for all pending auto-flushes to complete
    await this.waitForAutoFlushes()
    
    // Then flush any remaining data (including currentBatch)
    // This ensures everything is inserted before commit returns
    await this.flush()
    
    // Reset session state for next commit cycle
    const insertedCount = this.totalInserted
    this.totalInserted = 0
    return insertedCount
  }
  
  /**
   * Wait for this session's operations to complete
   */
  async waitForOperations(maxWaitTime = null) {
    const startTime = Date.now()
    const hasTimeout = maxWaitTime !== null && maxWaitTime !== undefined
    
    // Wait for auto-flushes first
    await this.waitForAutoFlushes()
    
    while (this.flushing || this.batches.length > 0 || this.currentBatch.length > 0) {
      // Check timeout only if we have one
      if (hasTimeout && (Date.now() - startTime) >= maxWaitTime) {
        return false
      }
      
      await new Promise(resolve => setTimeout(resolve, 1))
    }
    
    return true
  }
  
  /**
   * Check if this session has pending operations
   */
  hasPendingOperations() {
    return this.pendingAutoFlushes.size > 0 || 
           this.flushing || 
           this.batches.length > 0 || 
           this.currentBatch.length > 0
  }
  
  /**
   * Destroy this session and unregister it
   */
  destroy() {
    // Unregister from database
    this.database.activeInsertSessions.delete(this)
    
    // Clear all data
    this.batches = []
    this.currentBatch = []
    this.totalInserted = 0
    this.flushing = false
    this.pendingAutoFlushes.clear()
  }
}

/**
 * JexiDB - A high-performance, in-memory database with persistence
 * 
 * Features:
 * - In-memory storage with optional persistence
 * - Advanced indexing and querying
 * - Transaction support
 * - Manual save functionality
 * - Recovery mechanisms
 * - Performance optimizations
 */
class Database extends EventEmitter {
  constructor(file, opts = {}) {
    super()
    
    // Generate unique instance ID for debugging
    this.instanceId = Math.random().toString(36).substr(2, 9)
    
    // Initialize state flags
    this.managersInitialized = false
    
    // Track active insert sessions
    this.activeInsertSessions = new Set()
    
    // Set default options
    this.opts = Object.assign({
      // Core options - auto-save removed, user must call save() manually
      // File creation options
      create: opts.create !== false, // Create file if it doesn't exist (default true)
      clear: opts.clear === true, // Clear existing files before loading (default false)
      // Timeout configurations for preventing hangs
      mutexTimeout: opts.mutexTimeout || 15000, // 15 seconds timeout for mutex operations
      maxFlushAttempts: opts.maxFlushAttempts || 50, // Maximum flush attempts before giving up
      // Term mapping options (always enabled and auto-detected from indexes)
      termMappingCleanup: opts.termMappingCleanup !== false, // Clean up orphaned terms on save (enabled by default)
      // Recovery options
      enableRecovery: opts.enableRecovery === true, // Recovery mechanisms disabled by default for large databases
      // Buffer size options for range merging
      maxBufferSize: opts.maxBufferSize || 4 * 1024 * 1024, // 4MB default maximum buffer size for grouped ranges
      // Memory management options (similar to published v1.1.0)
      maxMemoryUsage: opts.maxMemoryUsage || 64 * 1024, // 64KB limit like published version
      maxWriteBufferSize: opts.maxWriteBufferSize || 1000, // Maximum records in writeBuffer
      // Query strategy options
      streamingThreshold: opts.streamingThreshold || 0.8, // Use streaming when limit > 80% of total records
      // Serialization options
      enableArraySerialization: opts.enableArraySerialization !== false, // Enable array serialization by default
      // Index rebuild options
      allowIndexRebuild: opts.allowIndexRebuild === true, // Allow automatic index rebuild when corrupted (default false - throws error)
    }, opts)
    
    // CRITICAL FIX: Initialize AbortController for lifecycle management
    this.abortController = new AbortController()
    this.pendingOperations = new Set()
    this.pendingPromises = new Set()
    this.destroyed = false
    this.destroying = false
    this.closed = false
    this.operationCounter = 0
    
    // CRITICAL FIX: Initialize OperationQueue to prevent race conditions
    this.operationQueue = new OperationQueue(false) // Disable debug mode for queue
    
    // Normalize file path to ensure it ends with .jdb
    this.normalizedFile = this.normalizeFilePath(file)
    
    // Initialize core properties
    this.offsets = [] // Array of byte offsets for each record
    this.indexOffset = 0 // Current position in file for new records
    this.deletedIds = new Set() // Track deleted record IDs
    this.shouldSave = false
    this.isLoading = false
    this.isSaving = false
    this.lastSaveTime = null
    this.initialized = false
    this._offsetRecoveryInProgress = false
    this.writeBufferTotalSize = 0
    
    
    // Initialize managers
    this.initializeManagers()
    
    // Initialize file mutex for thread safety
    this.fileMutex = new Mutex()
    
    // Initialize performance tracking
    this.performanceStats = {
      operations: 0,
      saves: 0,
      loads: 0,
      queryTime: 0,
      saveTime: 0,
      loadTime: 0
    }

    // Initialize integrity correction tracking
    this.integrityCorrections = {
      indexSync: 0,        // index.totalLines vs offsets.length corrections
      indexInconsistency: 0, // Index record count vs offsets mismatch
      writeBufferForced: 0, // WriteBuffer not cleared after save
      indexSaveFailures: 0, // Failed to save index data
      dataIntegrity: 0,    // General data integrity issues
      utf8Recovery: 0,     // UTF-8 decoding failures recovered
      jsonRecovery: 0      // JSON parsing failures recovered
    }


    // Initialize usage stats for QueryManager
    this.usageStats = {
      totalQueries: 0,
      indexedQueries: 0,
      streamingQueries: 0,
      indexedAverageTime: 0,
      streamingAverageTime: 0
    }

    // Note: Validation will be done after configuration conversion in initializeManagers()
  }

  /**
   * Validate field and index configuration
   */
  validateIndexConfiguration() {
    // Validate fields configuration
    if (this.opts.fields && typeof this.opts.fields === 'object') {
      this.validateFieldTypes(this.opts.fields, 'fields')
    }

    // Validate indexes configuration (legacy support)
    if (this.opts.indexes && typeof this.opts.indexes === 'object') {
      this.validateFieldTypes(this.opts.indexes, 'indexes')
    }

    // Validate indexes array (new format) - but only if we have fields
    if (this.opts.originalIndexes && Array.isArray(this.opts.originalIndexes)) {
      if (this.opts.fields) {
        this.validateIndexFields(this.opts.originalIndexes)
      } else if (this.opts.debugMode) {
        console.log('⚠️  Skipping index field validation because no fields configuration was provided')
      }
    }

    if (this.opts.debugMode) {
      const fieldCount = this.opts.fields ? Object.keys(this.opts.fields).length : 0
      const indexCount = Array.isArray(this.opts.indexes) ? this.opts.indexes.length : 
                        (this.opts.indexes && typeof this.opts.indexes === 'object' ? Object.keys(this.opts.indexes).length : 0)
      if (fieldCount > 0 || indexCount > 0) {
        console.log(`✅ Configuration validated: ${fieldCount} fields, ${indexCount} indexes [${this.instanceId}]`)
      }
    }
  }

  /**
   * Validate field types
   */
  validateFieldTypes(fields, configType) {
    const supportedTypes = ['string', 'number', 'boolean', 'array:string', 'array:number', 'array:boolean', 'array', 'object', 'auto']
    const errors = []

    for (const [fieldName, fieldType] of Object.entries(fields)) {
      if (fieldType === 'auto') {
        continue
      }

      // Check if type is supported
      if (!supportedTypes.includes(fieldType)) {
        errors.push(`Unsupported ${configType} type '${fieldType}' for field '${fieldName}'. Supported types: ${supportedTypes.join(', ')}`)
      }

      // Warn about legacy array type but don't error
      if (fieldType === 'array') {
        if (this.opts.debugMode) {
          console.log(`⚠️  Legacy array type '${fieldType}' for field '${fieldName}'. Consider using 'array:string' for better performance.`)
        }
      }

      // Check for common mistakes
      if (fieldType === 'array:') {
        errors.push(`Incomplete array type '${fieldType}' for field '${fieldName}'. Must specify element type after colon: array:string, array:number, or array:boolean`)
      }
    }

    if (errors.length > 0) {
      throw new Error(`${configType.charAt(0).toUpperCase() + configType.slice(1)} configuration errors:\n${errors.map(e => `  - ${e}`).join('\n')}`)
    }
  }

  /**
   * Validate index fields array
   */
  validateIndexFields(indexFields) {
    if (!this.opts.fields) {
      throw new Error('Index fields array requires fields configuration. Use: { fields: {...}, indexes: [...] }')
    }

    const availableFields = Object.keys(this.opts.fields)
    const errors = []

    for (const fieldName of indexFields) {
      if (!availableFields.includes(fieldName)) {
        errors.push(`Index field '${fieldName}' not found in fields configuration. Available fields: ${availableFields.join(', ')}`)
      }
    }

    if (errors.length > 0) {
      throw new Error(`Index configuration errors:\n${errors.map(e => `  - ${e}`).join('\n')}`)
    }
  }

  /**
   * Prepare index configuration for IndexManager
   */
  prepareIndexConfiguration() {
    if (Array.isArray(this.opts.indexes)) {
      const indexedFields = {}
      const originalIndexes = [...this.opts.indexes]
      const hasFieldConfig = this.opts.fields && typeof this.opts.fields === 'object'

      for (const fieldName of this.opts.indexes) {
        if (hasFieldConfig && this.opts.fields[fieldName]) {
          indexedFields[fieldName] = this.opts.fields[fieldName]
        } else {
          indexedFields[fieldName] = 'auto'
        }
      }

      this.opts.originalIndexes = originalIndexes
      this.opts.indexes = indexedFields

      if (this.opts.debugMode) {
        console.log(`🔍 Normalized indexes array to object: ${Object.keys(indexedFields).join(', ')} [${this.instanceId}]`)
      }
    }
    // Legacy format (indexes as object) is already compatible
  }

  /**
   * Initialize all managers
   */
  initializeManagers() {
    // CRITICAL FIX: Prevent double initialization which corrupts term mappings
    if (this.managersInitialized) {
      if (this.opts.debugMode) {
        console.log(`⚠️  initializeManagers() called again - skipping to prevent corruption [${this.instanceId}]`)
      }
      return
    }
    
    // Handle legacy 'schema' option migration
    if (this.opts.schema) {
      // If fields is already provided and valid, ignore schema
      if (this.opts.fields && typeof this.opts.fields === 'object' && Object.keys(this.opts.fields).length > 0) {
        if (this.opts.debugMode) {
          console.log(`⚠️  Both 'schema' and 'fields' options provided. Ignoring 'schema' and using 'fields'. [${this.instanceId}]`)
        }
      } else if (Array.isArray(this.opts.schema)) {
        // Schema as array is no longer supported
        throw new Error('The "schema" option as an array is no longer supported. Please use "fields" as an object instead. Example: { fields: { id: "number", name: "string" } }')
      } else if (typeof this.opts.schema === 'object' && this.opts.schema !== null) {
        // Schema as object - migrate to fields
        this.opts.fields = { ...this.opts.schema }
        if (this.opts.debugMode) {
          console.log(`⚠️  Migrated 'schema' option to 'fields'. Please update your code to use 'fields' instead of 'schema'. [${this.instanceId}]`)
        }
      } else {
        throw new Error('The "schema" option must be an object. Example: { schema: { id: "number", name: "string" } }')
      }
    }
    
    // Validate that fields is provided (mandatory)
    if (!this.opts.fields || typeof this.opts.fields !== 'object' || Object.keys(this.opts.fields).length === 0) {
      throw new Error('The "fields" option is mandatory and must be an object with at least one field definition. Example: { fields: { id: "number", name: "string" } }')
    }
    
    // CRITICAL FIX: Initialize serializer first - this was missing and causing crashes
    this.serializer = new Serializer(this.opts)
    
    // Initialize schema for array-based serialization
    if (this.opts.enableArraySerialization !== false) {
      this.initializeSchema()
    }
    
    // Initialize TermManager - always enabled for optimal performance
    this.termManager = new TermManager()
    
    // Auto-detect term mapping fields from indexes
    const termMappingFields = this.getTermMappingFields()
    this.termManager.termMappingFields = termMappingFields
    this.opts.termMapping = true // Always enable term mapping for optimal performance
    
    // Validation: Ensure all array:string indexed fields are in term mapping fields
    if (this.opts.indexes) {
      const arrayStringFields = []
      for (const [field, type] of Object.entries(this.opts.indexes)) {
        if (type === 'array:string' && !termMappingFields.includes(field)) {
          arrayStringFields.push(field)
        }
      }
      if (arrayStringFields.length > 0) {
        if (this.opts.debugMode) {
          console.warn(`⚠️  Warning: The following array:string indexed fields were not added to term mapping: ${arrayStringFields.join(', ')}. This may impact performance.`)
        }
      }
    }
    
    if (this.opts.debugMode) {
      if (termMappingFields.length > 0) {
        console.log(`🔍 TermManager initialized for fields: ${termMappingFields.join(', ')} [${this.instanceId}]`)
      } else {
        console.log(`🔍 TermManager initialized (no array:string fields detected) [${this.instanceId}]`)
      }
    }
    
    // Prepare index configuration for IndexManager
    this.prepareIndexConfiguration()

    // Validate configuration after conversion
    this.validateIndexConfiguration()

    // Initialize IndexManager with database reference for term mapping
    this.indexManager = new IndexManager(this.opts, null, this)
    if (this.opts.debugMode) {
        console.log(`🔍 IndexManager initialized with fields: ${this.indexManager.indexedFields.join(', ')} [${this.instanceId}]`)
    }
    
    // Mark managers as initialized
    this.managersInitialized = true
    this.indexOffset = 0
    this.writeBuffer = []
    this.writeBufferOffsets = [] // Track offsets for writeBuffer records
    this.writeBufferSizes = []   // Track sizes for writeBuffer records
    this.writeBufferTotalSize = 0
    this.isInsideOperationQueue = false // Flag to prevent deadlock in save() calls
    
    // Initialize other managers
    this.fileHandler = new FileHandler(this.normalizedFile, this.fileMutex, this.opts)
    this.queryManager = new QueryManager(this)
    this.concurrencyManager = new ConcurrencyManager(this.opts)
    this.statisticsManager = new StatisticsManager(this, this.opts)
    this.streamingProcessor = new StreamingProcessor(this.opts)
  }

  /**
   * Get term mapping fields from configuration or indexes (auto-detected)
   * @returns {string[]} Array of field names that use term mapping
   */
  getTermMappingFields() {
    // If termMappingFields is explicitly configured, use it
    if (this.opts.termMappingFields && Array.isArray(this.opts.termMappingFields)) {
      return [...this.opts.termMappingFields]
    }

    // Auto-detect fields that benefit from term mapping from indexes
    if (!this.opts.indexes) return []

    const termMappingFields = []

    for (const [field, type] of Object.entries(this.opts.indexes)) {
      // Fields that should use term mapping (only array fields)
      if (type === 'array:string') {
        termMappingFields.push(field)
      }
    }

    return termMappingFields
  }

  /**
   * CRITICAL FIX: Validate database state before critical operations
   * Prevents crashes from undefined methods and invalid states
   */
  validateState() {
    if (this.destroyed) {
      throw new Error('Database is destroyed')
    }
    
    if (this.closed) {
      throw new Error('Database is closed. Call init() to reopen it.')
    }
    
    // Allow operations during destroying phase for proper cleanup
    
    if (!this.serializer) {
      throw new Error('Database serializer not initialized - this indicates a critical bug')
    }
    
    if (!this.normalizedFile) {
      throw new Error('Database file path not set - this indicates file path management failure')
    }
    
    if (!this.fileHandler) {
      throw new Error('Database file handler not initialized')
    }
    
    if (!this.indexManager) {
      throw new Error('Database index manager not initialized')
    }
    
    return true
  }

  /**
   * CRITICAL FIX: Ensure file path is valid and accessible
   * Prevents file path loss issues mentioned in crash report
   */
  ensureFilePath() {
    if (!this.normalizedFile) {
      throw new Error('Database file path is missing after initialization - this indicates a critical file path management failure')
    }
    return this.normalizedFile
  }

  /**
   * Normalize file path to ensure it ends with .jdb
   */
  normalizeFilePath(file) {
    if (!file) return null
    return file.endsWith('.jdb') ? file : `${file}.jdb`
  }

  /**
   * Initialize the database
   */
  async initialize() {
    // Check if database is destroyed first (before checking initialized)
    if (this.destroyed) {
      throw new Error('Cannot initialize destroyed database. Use a new instance instead.')
    }
    
    if (this.initialized) return
    
    // Prevent concurrent initialization - wait for ongoing init to complete
    if (this.isLoading) {
      if (this.opts.debugMode) {
        console.log('🔄 init() already in progress - waiting for completion')
      }
      // Wait for ongoing initialization to complete
      while (this.isLoading) {
        await new Promise(resolve => setTimeout(resolve, 10))
      }
      // Check if initialization completed successfully
      if (this.initialized) {
        if (this.opts.debugMode) {
          console.log('✅ Concurrent init() completed - database is now initialized')
        }
        return
      }
      // If we get here, initialization failed - we can try again
    }
    
    try {
      this.isLoading = true
      
      // Reset closed state when reinitializing
      this.closed = false

      // Initialize managers (protected against double initialization)
      this.initializeManagers()
      
      // Handle clear option - delete existing files before loading
      if (this.opts.clear && this.normalizedFile) {
        await this.clearExistingFiles()
      }
      
      // Check file existence and handle create option
      if (this.normalizedFile) {
        const fileExists = await this.fileHandler.exists()
        
        if (!fileExists) {
          if (!this.opts.create) {
            throw new Error(`Database file '${this.normalizedFile}' does not exist and create option is disabled`)
          }
          // File will be created when first data is written
        } else {
          // Load existing data if file exists
          await this.load()
        }
      }

      // CRITICAL INTEGRITY CHECK: Ensure IndexManager is consistent with loaded offsets
      // This must happen immediately after load() to prevent any subsequent operations from seeing inconsistent state
      if (this.indexManager && this.offsets && this.offsets.length > 0) {
        const currentTotalLines = this.indexManager.totalLines || 0
        if (currentTotalLines !== this.offsets.length) {
          this.indexManager.setTotalLines(this.offsets.length)
          if (this.opts.debugMode) {
            console.log(`🔧 Post-load integrity sync: IndexManager totalLines ${currentTotalLines} → ${this.offsets.length}`)
          }
        }
      }

      // Manual save is now the default behavior

      // CRITICAL FIX: Ensure IndexManager totalLines is consistent with offsets
      // This prevents data integrity issues when database is initialized without existing data
      if (this.indexManager && this.offsets) {
        this.indexManager.setTotalLines(this.offsets.length)
        if (this.opts.debugMode) {
          console.log(`🔧 Initialized index totalLines to ${this.offsets.length}`)
        }
      }

      this.initialized = true
      this.emit('initialized')

      if (this.opts.debugMode) {
        console.log(`✅ Database initialized with ${this.writeBuffer.length} records`)
      }
    } catch (error) {
      console.error('Failed to initialize database:', error)
      throw error
    } finally {
      this.isLoading = false
    }
  }

  /**
   * Validate that the database is initialized before performing operations
   * @param {string} operation - The operation being attempted
   * @throws {Error} If database is not initialized
   */
  _validateInitialization(operation) {
    if (this.destroyed) {
      throw new Error(`❌ Cannot perform '${operation}' on a destroyed database. Create a new instance instead.`)
    }
    
    if (this.closed) {
      throw new Error(`❌ Database is closed. Call 'await db.init()' to reopen it before performing '${operation}' operations.`)
    }
    
    if (!this.initialized) {
      const errorMessage = `❌ Database not initialized. Call 'await db.init()' before performing '${operation}' operations.\n\n` +
                          `Example:\n` +
                          `  const db = new Database('./myfile.jdb')\n` +
                          `  await db.init()  // ← Required before any operations\n` +
                          `  await db.insert({ name: 'test' })  // ← Now you can use database operations\n\n` +
                          `File: ${this.normalizedFile || 'unknown'}`
      
      throw new Error(errorMessage)
    }
  }

  /**
   * Clear existing database files (.jdb and .idx.jdb)
   */
  async clearExistingFiles() {
    if (!this.normalizedFile) return
    
    try {
      // Clear main database file
      if (await this.fileHandler.exists()) {
        await this.fileHandler.delete()
        if (this.opts.debugMode) {
          console.log(`🗑️ Cleared database file: ${this.normalizedFile}`)
        }
      }
      
      // Clear index file
      const idxPath = this.normalizedFile.replace('.jdb', '.idx.jdb')
      const idxFileHandler = new FileHandler(idxPath, this.fileMutex, this.opts)
      if (await idxFileHandler.exists()) {
        await idxFileHandler.delete()
        if (this.opts.debugMode) {
          console.log(`🗑️ Cleared index file: ${idxPath}`)
        }
      }
      
      // Reset internal state
      this.offsets = []
      this.indexOffset = 0
      this.deletedIds.clear()
      this.shouldSave = false
      
      // Create empty files to ensure they exist
      await this.fileHandler.writeAll('')
      await idxFileHandler.writeAll('')
      
      if (this.opts.debugMode) {
        console.log('🗑️ Database cleared successfully')
      }
    } catch (error) {
      console.error('Failed to clear existing files:', error)
      throw error
    }
  }

  /**
   * Load data from file
   */
  async load() {
    if (!this.normalizedFile) return
    
    
    try {
      const startTime = Date.now()
      this.isLoading = true
      
      // Don't load the entire file - just initialize empty state
      // The actual record count will come from loaded offsets
      this.writeBuffer = [] // writeBuffer is only for new unsaved records
      this.writeBufferOffsets = []
      this.writeBufferSizes = []
      this.writeBufferTotalSize = 0
      
      // recordCount will be determined from loaded offsets
      // If no offsets were loaded, we'll count records only if needed
      
      // Load index data if available (always try to load offsets, even without indexed fields)
      if (this.indexManager) {
        const idxPath = this.normalizedFile.replace('.jdb', '.idx.jdb')
        try {
          const idxFileHandler = new FileHandler(idxPath, this.fileMutex, this.opts)
          
          // Check if file exists BEFORE trying to read it
          const fileExists = await idxFileHandler.exists()
          if (!fileExists) {
            // File doesn't exist - this will be handled by catch block
            throw new Error('Index file does not exist')
          }
          
          const idxData = await idxFileHandler.readAll()
          
          // If file exists but is empty or has no content, treat as corrupted
          if (!idxData || !idxData.trim()) {
            // File exists but is empty - treat as corrupted
            const fileExists = await this.fileHandler.exists()
            if (fileExists) {
              const stats = await this.fileHandler.getFileStats()
              if (stats && stats.size > 0) {
                // Data file has content but index is empty - corrupted
                if (!this.opts.allowIndexRebuild) {
                  throw new Error(
                    `Index file is corrupted: ${idxPath} exists but contains no index data, ` +
                    `while the data file has ${stats.size} bytes. ` +
                    `Set allowIndexRebuild: true to automatically rebuild the index, ` +
                    `or manually fix/delete the corrupted index file.`
                  )
                }
                // Schedule rebuild if allowed
                if (this.opts.debugMode) {
                  console.log(`⚠️ Index file exists but is empty while data file has ${stats.size} bytes - scheduling rebuild`)
                }
                this._scheduleIndexRebuild()
                // Continue execution - rebuild will happen on first query
                // Don't return - let the code continue to load other things if needed
              }
            }
            // If data file is also empty, just continue (no error needed)
            // Don't return - let the code continue to load other things if needed
          } else {
            // File has content - parse and load it
            const parsedIdxData = JSON.parse(idxData)
            
            // Always load offsets if available (even without indexed fields)
            if (parsedIdxData.offsets && Array.isArray(parsedIdxData.offsets)) {
              this.offsets = parsedIdxData.offsets
              // CRITICAL FIX: Update IndexManager totalLines to match offsets length
              // This ensures queries and length property work correctly even if offsets are reset later
              if (this.indexManager) {
                this.indexManager.setTotalLines(this.offsets.length)
                if (this.opts.debugMode) {
                  console.log(`📂 Loaded ${this.offsets.length} offsets from ${idxPath}, synced IndexManager totalLines`)
                }
              }
            }
            
            // Load indexOffset for proper range calculations
            if (parsedIdxData.indexOffset !== undefined) {
              this.indexOffset = parsedIdxData.indexOffset
              if (this.opts.debugMode) {
                console.log(`📂 Loaded indexOffset: ${this.indexOffset} from ${idxPath}`)
              }
            }
            
            // Load configuration from .idx file if database exists
            // CRITICAL: Load config FIRST so indexes are available for term mapping detection
            if (parsedIdxData.config) {
              const config = parsedIdxData.config
              
              // Override constructor options with saved configuration
              if (config.fields) {
                this.opts.fields = config.fields
                if (this.opts.debugMode) {
                  console.log(`📂 Loaded fields config from ${idxPath}:`, Object.keys(config.fields))
                }
              }
              
              if (config.indexes) {
                this.opts.indexes = config.indexes
                if (this.indexManager) {
                  this.indexManager.setIndexesConfig(config.indexes)
                }
                if (this.opts.debugMode) {
                  console.log(`📂 Loaded indexes config from ${idxPath}:`, Object.keys(config.indexes))
                }
              }
              
              // CRITICAL FIX: Update term mapping fields AFTER loading indexes from config
              // This ensures termManager knows which fields use term mapping
              // (getTermMappingFields() was called during init() before indexes were loaded)
              if (this.termManager && config.indexes) {
                const termMappingFields = this.getTermMappingFields()
                this.termManager.termMappingFields = termMappingFields
                if (this.opts.debugMode && termMappingFields.length > 0) {
                  console.log(`🔍 Updated term mapping fields after loading indexes: ${termMappingFields.join(', ')}`)
                }
              }
            }
            
            // Load term mapping data from .idx file if it exists
            // CRITICAL: Load termMapping even if index is empty (terms are needed for queries)
            // NOTE: termMappingFields should already be set above from config.indexes
            if (parsedIdxData.termMapping && this.termManager && this.termManager.termMappingFields && this.termManager.termMappingFields.length > 0) {
              await this.termManager.loadTerms(parsedIdxData.termMapping)
              if (this.opts.debugMode) {
                console.log(`📂 Loaded term mapping from ${idxPath} (${Object.keys(parsedIdxData.termMapping).length} terms)`)
              }
            }
            
            // Load index data only if available and we have indexed fields
            if (parsedIdxData && parsedIdxData.index && this.indexManager.indexedFields && this.indexManager.indexedFields.length > 0) {
              this.indexManager.load(parsedIdxData.index)
              
              if (this.opts.debugMode) {
                console.log(`📂 Loaded index data from ${idxPath}`)
              }
              
              // Check if loaded index is actually empty (corrupted)
              let hasAnyIndexData = false
              for (const field of this.indexManager.indexedFields) {
                if (this.indexManager.hasUsableIndexData(field)) {
                  hasAnyIndexData = true
                  break
                }
              }
              
              if (this.opts.debugMode) {
                console.log(`📊 Index check: hasAnyIndexData=${hasAnyIndexData}, indexedFields=${this.indexManager.indexedFields.join(',')}`)
              }
              
              // Schedule rebuild if index is empty AND file exists with data
              if (!hasAnyIndexData) {
                // Check if the actual .jdb file has data
                const fileExists = await this.fileHandler.exists()
                if (this.opts.debugMode) {
                  console.log(`📊 File check: exists=${fileExists}`)
                }
                if (fileExists) {
                  const stats = await this.fileHandler.getFileStats()
                  if (this.opts.debugMode) {
                    console.log(`📊 File stats: size=${stats?.size}`)
                  }
                  if (stats && stats.size > 0) {
                    // File has data but index is empty - corrupted index detected
                    if (!this.opts.allowIndexRebuild) {
                      const idxPath = this.normalizedFile.replace('.jdb', '.idx.jdb')
                      throw new Error(
                        `Index file is corrupted: ${idxPath} exists but contains no index data, ` +
                        `while the data file has ${stats.size} bytes. ` +
                        `Set allowIndexRebuild: true to automatically rebuild the index, ` +
                        `or manually fix/delete the corrupted index file.`
                      )
                    }
                    // Schedule rebuild if allowed
                    if (this.opts.debugMode) {
                      console.log(`⚠️ Index loaded but empty while file has ${stats.size} bytes - scheduling rebuild`)
                    }
                    this._scheduleIndexRebuild()
                  }
                }
              }
            }
            
            // Continue with remaining config loading
            if (parsedIdxData.config) {
              const config = parsedIdxData.config
              
              if (config.originalIndexes) {
                this.opts.originalIndexes = config.originalIndexes
                if (this.opts.debugMode) {
                  console.log(`📂 Loaded originalIndexes config from ${idxPath}:`, config.originalIndexes.length, 'indexes')
                }
              }
              
              // Reinitialize schema from saved configuration (only if fields not provided)
              // Note: fields option takes precedence over saved schema
              if (!this.opts.fields && config.schema && this.serializer) {
                this.serializer.initializeSchema(config.schema)
                if (this.opts.debugMode) {
                  console.log(`📂 Loaded schema from ${idxPath}:`, config.schema.join(', '))
                }
              } else if (this.opts.fields && this.serializer) {
                // Use fields option instead of saved schema
                const fieldNames = Object.keys(this.opts.fields)
                if (fieldNames.length > 0) {
                  this.serializer.initializeSchema(fieldNames)
                  if (this.opts.debugMode) {
                    console.log(`📂 Schema initialized from fields option:`, fieldNames.join(', '))
                  }
                }
              }
            }
          }
        } catch (idxError) {
          // Index file doesn't exist or is corrupted, rebuild from data
          // BUT: if error is about rebuild being disabled, re-throw it immediately
          if (idxError.message && (idxError.message.includes('allowIndexRebuild') || idxError.message.includes('corrupted'))) {
            throw idxError
          }
          
          // If error is "Index file does not exist", check if we should throw or rebuild
          if (idxError.message && idxError.message.includes('does not exist')) {
            // Check if the actual .jdb file has data that needs indexing
            try {
              const fileExists = await this.fileHandler.exists()
              if (fileExists) {
                const stats = await this.fileHandler.getFileStats()
                if (stats && stats.size > 0) {
                  // File has data but index is missing
                  if (!this.opts.allowIndexRebuild) {
                    const idxPath = this.normalizedFile.replace('.jdb', '.idx.jdb')
                    throw new Error(
                      `Index file is missing or corrupted: ${idxPath} does not exist or is invalid, ` +
                      `while the data file has ${stats.size} bytes. ` +
                      `Set allowIndexRebuild: true to automatically rebuild the index, ` +
                      `or manually create/fix the index file.`
                    )
                  }
                  // Schedule rebuild if allowed
                  if (this.opts.debugMode) {
                    console.log(`⚠️ .jdb file has ${stats.size} bytes but index missing - scheduling rebuild`)
                  }
                  this._scheduleIndexRebuild()
                  return // Exit early
                }
              }
            } catch (statsError) {
              if (this.opts.debugMode) {
                console.log('⚠️ Could not check file stats:', statsError.message)
              }
              // Re-throw if it's our error about rebuild being disabled
              if (statsError.message && statsError.message.includes('allowIndexRebuild')) {
                throw statsError
              }
            }
            // If no data file or empty, just continue (no error needed)
            return
          }
          
          if (this.opts.debugMode) {
            console.log('📂 No index file found or corrupted, checking if rebuild is needed...')
          }
          
          // Check if the actual .jdb file has data that needs indexing
          try {
            const fileExists = await this.fileHandler.exists()
            if (fileExists) {
              const stats = await this.fileHandler.getFileStats()
              if (stats && stats.size > 0) {
                // File has data but index is missing or corrupted
                if (!this.opts.allowIndexRebuild) {
                  const idxPath = this.normalizedFile.replace('.jdb', '.idx.jdb')
                  throw new Error(
                    `Index file is missing or corrupted: ${idxPath} does not exist or is invalid, ` +
                    `while the data file has ${stats.size} bytes. ` +
                    `Set allowIndexRebuild: true to automatically rebuild the index, ` +
                    `or manually create/fix the index file.`
                  )
                }
                // Schedule rebuild if allowed
                if (this.opts.debugMode) {
                  console.log(`⚠️ .jdb file has ${stats.size} bytes but index missing - scheduling rebuild`)
                }
                this._scheduleIndexRebuild()
              }
            }
          } catch (statsError) {
            if (this.opts.debugMode) {
              console.log('⚠️ Could not check file stats:', statsError.message)
            }
            // Re-throw if it's our error about rebuild being disabled
            if (statsError.message && statsError.message.includes('allowIndexRebuild')) {
              throw statsError
            }
          }
        }
      } else {
        // No indexed fields, no need to rebuild indexes
      }
      
      this.performanceStats.loads++
      this.performanceStats.loadTime += Date.now() - startTime
      this.emit('loaded', this.writeBuffer.length)
    } catch (error) {
      console.error('Failed to load database:', error)
      throw error
    } finally {
      this.isLoading = false
    }
  }


  /**
   * Save data to file
   * @param {boolean} inQueue - Whether to execute within the operation queue (default: false)
   */
  async save(inQueue = false) {
    this._validateInitialization('save')
    
    if (this.opts.debugMode) {
      console.log(`💾 save() called: writeBuffer.length=${this.writeBuffer.length}, offsets.length=${this.offsets.length}`)
    }
    
    // CRITICAL FIX: Wait for all active insert sessions to complete their auto-flushes
    // This prevents race conditions where save() writes data while auto-flushes are still adding to writeBuffer
    if (this.activeInsertSessions && this.activeInsertSessions.size > 0) {
      if (this.opts.debugMode) {
        console.log(`⏳ save(): Waiting for ${this.activeInsertSessions.size} active insert sessions to complete auto-flushes`)
      }
      
      const sessionPromises = Array.from(this.activeInsertSessions).map(session => 
        session.waitForAutoFlushes().catch(err => {
          if (this.opts.debugMode) {
            console.warn(`⚠️ save(): Error waiting for insert session: ${err.message}`)
          }
        })
      )
      
      await Promise.all(sessionPromises)
      
      if (this.opts.debugMode) {
        console.log(`✅ save(): All insert sessions completed auto-flushes`)
      }
    }
    
    // Auto-save removed - no need to pause anything
    
    try {
      // CRITICAL FIX: Wait for any ongoing save operations to complete
      if (this.isSaving) {
        if (this.opts.debugMode) {
          console.log('💾 save(): waiting for previous save to complete')
        }
        // Wait for previous save to complete
        while (this.isSaving) {
          await new Promise(resolve => setTimeout(resolve, 10))
        }
        
        // Check if data changed since the previous save completed
        const hasDataToSave = this.writeBuffer.length > 0 || this.deletedIds.size > 0
        const needsStructureCreation = this.indexManager && this.indexManager.indexedFields && this.indexManager.indexedFields.length > 0
        
        if (!hasDataToSave && !needsStructureCreation) {
          if (this.opts.debugMode) {
            console.log('💾 Save: No new data to save since previous save comp