jexidb
Version:
JexiDB is a pure JS NPM library for managing data on disk efficiently, without the need for a server.
1,361 lines (1,190 loc) • 88.9 kB
JavaScript
import { Mutex } from 'async-mutex'
import { normalizeCriteriaOperators } from '../utils/operatorNormalizer.mjs'
export default class IndexManager {
constructor(opts, databaseMutex = null, database = null) {
this.opts = Object.assign({}, opts)
this.index = Object.assign({data: {}}, this.opts.index)
this.totalLines = 0
this.rangeThreshold = 10 // Sensible threshold: 10+ consecutive numbers justify ranges
this.binarySearchThreshold = 32 // Much higher for better performance
this.database = database // Reference to database for term manager access
this.indexLoaded = false
this.indexIdleUnloadMs = typeof this.opts.indexIdleUnloadMs === 'number' ? this.opts.indexIdleUnloadMs : 30000
this._idleUnloadTimer = null
this._indexLoadPromise = null
// CRITICAL: Use database mutex to prevent deadlocks
// If no database mutex provided, create a local one (for backward compatibility)
this.mutex = databaseMutex || new Mutex()
this.indexedFields = []
this.setIndexesConfig(this.opts.indexes)
}
setTotalLines(total) {
this.totalLines = total
}
/**
* Update indexes configuration and ensure internal structures stay in sync
* @param {Object|Array<string>} indexes
*/
setIndexesConfig(indexes) {
if (!indexes) {
this.opts.indexes = undefined
this.indexedFields = []
return
}
if (Array.isArray(indexes)) {
const fields = indexes.map(field => String(field))
this.indexedFields = fields
const normalizedConfig = {}
for (const field of fields) {
const existingConfig = (!Array.isArray(this.opts.indexes) && typeof this.opts.indexes === 'object') ? this.opts.indexes[field] : undefined
normalizedConfig[field] = existingConfig ?? 'auto'
if (!this.index.data[field]) {
this.index.data[field] = {}
}
}
this.opts.indexes = normalizedConfig
return
}
if (typeof indexes === 'object') {
this.opts.indexes = Object.assign({}, indexes)
this.indexedFields = Object.keys(this.opts.indexes)
for (const field of this.indexedFields) {
if (!this.index.data[field]) {
this.index.data[field] = {}
}
}
}
}
_clearIdleUnloadTimer() {
if (this._idleUnloadTimer) {
clearTimeout(this._idleUnloadTimer)
this._idleUnloadTimer = null
}
}
cancelIdleUnloadTimer() {
this._clearIdleUnloadTimer()
}
markIndexUsed() {
if (!this.indexLoaded) return
this._clearIdleUnloadTimer()
if (this.indexIdleUnloadMs > 0) {
const timer = setTimeout(() => {
try {
this.unload()
} catch (error) {
if (this.opts.debugMode) {
console.error('⚠️ IndexManager.markIndexUsed: Failed to unload index', error)
}
}
}, this.indexIdleUnloadMs)
if (timer && typeof timer.unref === 'function') {
timer.unref()
}
this._idleUnloadTimer = timer
}
}
unload() {
if (!this.indexLoaded) return
if (this.opts.debugMode) {
console.log(`🧹 IndexManager.unload: Unloading index data after ${this.indexIdleUnloadMs}ms idle`)
}
this._clearIdleUnloadTimer()
const newData = {}
for (const field of this.indexedFields) {
newData[field] = {}
}
this.index.data = newData
this.indexLoaded = false
}
/**
* Check if a field is configured as an index
* @param {string} field - Field name
* @returns {boolean}
*/
isFieldIndexed(field) {
if (!field) return false
if (!Array.isArray(this.indexedFields)) {
return false
}
return this.indexedFields.includes(field)
}
/**
* Determine whether the index has usable data for a given field
* @param {string} field - Field name
* @returns {boolean}
*/
hasUsableIndexData(field) {
if (!field) return false
const fieldData = this.index?.data?.[field]
if (!fieldData || typeof fieldData !== 'object') {
return false
}
for (const key in fieldData) {
if (!Object.prototype.hasOwnProperty.call(fieldData, key)) continue
const entry = fieldData[key]
if (!entry) continue
if (entry.set && typeof entry.set.size === 'number' && entry.set.size > 0) {
return true
}
if (Array.isArray(entry.ranges) && entry.ranges.length > 0) {
const hasRangeData = entry.ranges.some(range => {
if (range === null || typeof range === 'undefined') {
return false
}
if (typeof range === 'object') {
const count = typeof range.count === 'number' ? range.count : 0
return count > 0
}
// When ranges are stored as individual numbers
return true
})
if (hasRangeData) {
return true
}
}
}
return false
}
// Ultra-fast range conversion - only for very large datasets
_toRanges(numbers) {
if (numbers.length === 0) return []
if (numbers.length < this.rangeThreshold) return numbers // Keep as-is for small arrays
const sorted = numbers.sort((a, b) => a - b) // Sort in-place
const ranges = []
let start = sorted[0]
let count = 1
for (let i = 1; i < sorted.length; i++) {
if (sorted[i] === sorted[i-1] + 1) {
count++
} else {
// End of consecutive sequence
if (count >= this.rangeThreshold) {
ranges.push({start, count})
} else {
// Add individual numbers for small sequences
for (let j = start; j < start + count; j++) {
ranges.push(j)
}
}
start = sorted[i]
count = 1
}
}
// Handle last sequence
if (count >= this.rangeThreshold) {
ranges.push({start, count})
} else {
for (let j = start; j < start + count; j++) {
ranges.push(j)
}
}
return ranges
}
// Ultra-fast range expansion
_fromRanges(ranges) {
if (!ranges || ranges.length === 0) return []
const numbers = []
for (const item of ranges) {
if (typeof item === 'object' && item.start !== undefined) {
// It's a range - use direct loop for maximum speed
const end = item.start + item.count
for (let i = item.start; i < end; i++) {
numbers.push(i)
}
} else {
// It's an individual number
numbers.push(item)
}
}
return numbers
}
// Ultra-fast lookup - optimized for Set operations
_hasLineNumber(hybridData, lineNumber) {
if (!hybridData) return false
// Check in Set first (O(1)) - most common case
if (hybridData.set && hybridData.set.has(lineNumber)) {
return true
}
// Check in ranges only if necessary
if (hybridData.ranges && hybridData.ranges.length > 0) {
return this._searchInRanges(hybridData.ranges, lineNumber)
}
return false
}
// Optimized search strategy
_searchInRanges(ranges, lineNumber) {
if (ranges.length < this.binarySearchThreshold) {
// Linear search for small ranges
return this._linearSearchRanges(ranges, lineNumber)
} else {
// Binary search for large ranges
return this._binarySearchRanges(ranges, lineNumber)
}
}
// Ultra-fast linear search
_linearSearchRanges(ranges, lineNumber) {
for (const item of ranges) {
if (typeof item === 'object' && item.start !== undefined) {
// It's a range
if (lineNumber >= item.start && lineNumber < item.start + item.count) {
return true
}
} else if (item === lineNumber) {
// It's an individual number
return true
}
}
return false
}
// Optimized binary search
_binarySearchRanges(ranges, lineNumber) {
let left = 0
let right = ranges.length - 1
while (left <= right) {
const mid = Math.floor((left + right) / 2)
const range = ranges[mid]
if (typeof range === 'object' && range.start !== undefined) {
// It's a range
if (lineNumber >= range.start && lineNumber < range.start + range.count) {
return true
} else if (lineNumber < range.start) {
right = mid - 1
} else {
left = mid + 1
}
} else {
// It's an individual number
if (range === lineNumber) {
return true
} else if (range < lineNumber) {
left = mid + 1
} else {
right = mid - 1
}
}
}
return false
}
// Ultra-fast add operation - minimal overhead
_addLineNumber(hybridData, lineNumber) {
// Initialize structure if needed
if (!hybridData) {
hybridData = { set: new Set(), ranges: [] }
}
// Add to Set directly (fastest path)
if (!hybridData.set) {
hybridData.set = new Set()
}
hybridData.set.add(lineNumber)
// Optimize to ranges when Set gets reasonably large
if (hybridData.set.size >= this.rangeThreshold * 2) { // 20 elements
if (this.opts.debugMode) {
console.log(`🔧 Triggering range optimization: Set size ${hybridData.set.size} >= threshold ${this.rangeThreshold * 2}`)
}
this._optimizeToRanges(hybridData)
}
return hybridData
}
// Ultra-fast remove operation
_removeLineNumber(hybridData, lineNumber) {
if (!hybridData) {
return hybridData
}
// Remove from Set (fast path)
if (hybridData.set) {
hybridData.set.delete(lineNumber)
}
// Remove from ranges (less common)
if (hybridData.ranges) {
hybridData.ranges = this._removeFromRanges(hybridData.ranges, lineNumber)
}
return hybridData
}
// Optimized range removal
_removeFromRanges(ranges, lineNumber) {
if (!ranges || ranges.length === 0) return ranges
const newRanges = []
for (const item of ranges) {
if (typeof item === 'object' && item.start !== undefined) {
// It's a range
if (lineNumber >= item.start && lineNumber < item.start + item.count) {
// Split range if needed
if (lineNumber === item.start) {
// Remove first element
if (item.count > 1) {
newRanges.push({ start: item.start + 1, count: item.count - 1 })
}
} else if (lineNumber === item.start + item.count - 1) {
// Remove last element
if (item.count > 1) {
newRanges.push({ start: item.start, count: item.count - 1 })
}
} else {
// Remove from middle - split into two ranges
const beforeCount = lineNumber - item.start
const afterCount = item.count - beforeCount - 1
if (beforeCount >= this.rangeThreshold) {
newRanges.push({ start: item.start, count: beforeCount })
} else {
// Add individual numbers for small sequences
for (let i = item.start; i < lineNumber; i++) {
newRanges.push(i)
}
}
if (afterCount >= this.rangeThreshold) {
newRanges.push({ start: lineNumber + 1, count: afterCount })
} else {
// Add individual numbers for small sequences
for (let i = lineNumber + 1; i < item.start + item.count; i++) {
newRanges.push(i)
}
}
}
} else {
newRanges.push(item)
}
} else if (item !== lineNumber) {
// It's an individual number
newRanges.push(item)
}
}
return newRanges
}
// Ultra-lazy range conversion - only when absolutely necessary
_optimizeToRanges(hybridData) {
if (!hybridData.set || hybridData.set.size === 0) {
return
}
if (this.opts.debugMode) {
console.log(`🔧 Starting range optimization for Set with ${hybridData.set.size} elements`)
}
// Only convert if we have enough data to make it worthwhile
if (hybridData.set.size < this.rangeThreshold) {
return
}
// Convert Set to array and find consecutive sequences
const numbers = Array.from(hybridData.set).sort((a, b) => a - b)
const ranges = []
let start = numbers[0]
let count = 1
for (let i = 1; i < numbers.length; i++) {
if (numbers[i] === numbers[i-1] + 1) {
count++
} else {
// End of consecutive sequence
if (count >= this.rangeThreshold) {
ranges.push({start, count})
// Remove these numbers from Set
for (let j = start; j < start + count; j++) {
hybridData.set.delete(j)
}
}
start = numbers[i]
count = 1
}
}
// Handle last sequence
if (count >= this.rangeThreshold) {
ranges.push({start, count})
for (let j = start; j < start + count; j++) {
hybridData.set.delete(j)
}
}
// Add new ranges to existing ranges
if (ranges.length > 0) {
if (!hybridData.ranges) {
hybridData.ranges = []
}
hybridData.ranges.push(...ranges)
// Keep ranges sorted for efficient binary search
hybridData.ranges.sort((a, b) => {
const aStart = typeof a === 'object' ? a.start : a
const bStart = typeof b === 'object' ? b.start : b
return aStart - bStart
})
}
}
// Ultra-fast get all line numbers
_getAllLineNumbers(hybridData) {
if (!hybridData) return []
// Use generator for lazy evaluation and better memory efficiency
return Array.from(this._getAllLineNumbersGenerator(hybridData))
}
// OPTIMIZATION: Generator-based approach for better memory efficiency
*_getAllLineNumbersGenerator(hybridData) {
const normalizeLineNumber = (value) => {
if (typeof value === 'number') {
return value
}
if (typeof value === 'string') {
const parsed = Number(value)
return Number.isNaN(parsed) ? value : parsed
}
if (typeof value === 'bigint') {
const maxSafe = BigInt(Number.MAX_SAFE_INTEGER)
return value <= maxSafe ? Number(value) : value
}
return value
}
// Yield from Set (fastest path)
if (hybridData.set) {
for (const num of hybridData.set) {
yield normalizeLineNumber(num)
}
}
// Yield from ranges (optimized)
if (hybridData.ranges) {
for (const item of hybridData.ranges) {
if (typeof item === 'object' && item.start !== undefined) {
// It's a range - use direct loop for better performance
const end = item.start + item.count
for (let i = item.start; i < end; i++) {
yield normalizeLineNumber(i)
}
} else {
// It's an individual number
yield normalizeLineNumber(item)
}
}
}
}
// OPTIMIZATION 6: Ultra-fast add operation with incremental index updates
async add(row, lineNumber) {
this.markIndexUsed()
if (typeof row !== 'object' || !row) {
throw new Error('Invalid \'row\' parameter, it must be an object')
}
if (typeof lineNumber !== 'number') {
throw new Error('Invalid line number')
}
// OPTIMIZATION 6: Use direct field access with minimal operations
const data = this.index.data
// OPTIMIZATION 6: Pre-allocate field structures for better performance
const fields = Object.keys(this.opts.indexes || {})
for (const field of fields) {
// PERFORMANCE: Check if this is a term mapping field once
const isTermMappingField = this.database?.termManager &&
this.database.termManager.termMappingFields &&
this.database.termManager.termMappingFields.includes(field)
// CRITICAL FIX: For term mapping fields, prefer ${field}Ids if available
// Records processed by processTermMapping have term IDs in ${field}Ids
// Records loaded from file have term IDs directly in ${field} (after restoreTermIdsAfterDeserialization)
let value
if (isTermMappingField) {
const termIdsField = `${field}Ids`
const termIds = row[termIdsField]
if (termIds && Array.isArray(termIds) && termIds.length > 0) {
// Use term IDs from ${field}Ids (preferred - from processTermMapping)
value = termIds
} else {
// Fallback: use field directly (for records loaded from file that have term IDs in field)
value = row[field]
}
} else {
value = row[field]
}
if (value !== undefined && value !== null) {
// OPTIMIZATION 6: Initialize field structure if it doesn't exist
if (!data[field]) {
data[field] = {}
}
const values = Array.isArray(value) ? value : [value]
for (const val of values) {
let key
if (isTermMappingField && typeof val === 'number') {
// For term mapping fields, values are already term IDs
key = String(val)
} else if (isTermMappingField && typeof val === 'string') {
// Fallback: convert string to term ID
// CRITICAL: During indexing (add), we should use getTermId() to create IDs if needed
// This is different from queries where we use getTermIdWithoutIncrement() to avoid creating new IDs
const termId = this.database.termManager.getTermId(val)
key = String(termId)
} else {
// For non-term-mapping fields (including array:number), use values directly
key = String(val)
}
// OPTIMIZATION 6: Use direct assignment for better performance
if (!data[field][key]) {
data[field][key] = { set: new Set(), ranges: [] }
}
// OPTIMIZATION 6: Direct Set operation - fastest possible
data[field][key].set.add(lineNumber)
// OPTIMIZATION 6: Lazy range optimization - only when beneficial
if (data[field][key].set.size >= this.rangeThreshold * 3) {
this._optimizeToRanges(data[field][key])
}
}
}
}
}
/**
* OPTIMIZATION 6: Add multiple records to the index in batch with optimized operations
* @param {Array} records - Records to add
* @param {number} startLineNumber - Starting line number
*/
async addBatch(records, startLineNumber) {
this.markIndexUsed()
if (!records || !records.length) return
// OPTIMIZATION 6: Pre-allocate index structures for better performance
const data = this.index.data
const fields = Object.keys(this.opts.indexes || {})
for (const field of fields) {
if (!data[field]) {
data[field] = {}
}
}
// OPTIMIZATION 6: Use Map for batch processing to reduce lookups
const fieldUpdates = new Map()
// OPTIMIZATION 6: Process all records in batch with optimized data structures
for (let i = 0; i < records.length; i++) {
const row = records[i]
const lineNumber = startLineNumber + i
for (const field of fields) {
// PERFORMANCE: Check if this is a term mapping field once
const isTermMappingField = this.database?.termManager &&
this.database.termManager.termMappingFields &&
this.database.termManager.termMappingFields.includes(field)
// CRITICAL FIX: For term mapping fields, prefer ${field}Ids if available
// Records processed by processTermMapping have term IDs in ${field}Ids
// Records loaded from file have term IDs directly in ${field} (after restoreTermIdsAfterDeserialization)
let value
if (isTermMappingField) {
const termIdsField = `${field}Ids`
const termIds = row[termIdsField]
if (termIds && Array.isArray(termIds) && termIds.length > 0) {
// Use term IDs from ${field}Ids (preferred - from processTermMapping)
value = termIds
} else {
// Fallback: use field directly (for records loaded from file that have term IDs in field)
value = row[field]
}
} else {
value = row[field]
}
if (value !== undefined && value !== null) {
const values = Array.isArray(value) ? value : [value]
for (const val of values) {
let key
if (isTermMappingField && typeof val === 'number') {
// For term mapping fields, values are already term IDs
key = String(val)
} else if (isTermMappingField && typeof val === 'string') {
// Fallback: convert string to term ID
// CRITICAL: During indexing (addBatch), we should use getTermId() to create IDs if needed
// This is different from queries where we use getTermIdWithoutIncrement() to avoid creating new IDs
const termId = this.database.termManager.getTermId(val)
key = String(termId)
} else {
// For non-term-mapping fields (including array:number), use values directly
key = String(val)
}
// OPTIMIZATION 6: Use Map for efficient batch updates
if (!fieldUpdates.has(field)) {
fieldUpdates.set(field, new Map())
}
const fieldMap = fieldUpdates.get(field)
if (!fieldMap.has(key)) {
fieldMap.set(key, new Set())
}
fieldMap.get(key).add(lineNumber)
}
}
}
}
// OPTIMIZATION 6: Apply all updates in batch for better performance
for (const [field, fieldMap] of fieldUpdates) {
for (const [key, lineNumbers] of fieldMap) {
if (!data[field][key]) {
data[field][key] = { set: new Set(), ranges: [] }
}
// OPTIMIZATION 6: Add all line numbers at once
for (const lineNumber of lineNumbers) {
data[field][key].set.add(lineNumber)
}
// OPTIMIZATION 6: Lazy range optimization - only when beneficial
if (data[field][key].set.size >= this.rangeThreshold * 3) {
this._optimizeToRanges(data[field][key])
}
}
}
}
// Ultra-fast dry remove
dryRemove(ln) {
const data = this.index.data
for (const field in data) {
for (const value in data[field]) {
// Direct Set operation - fastest possible
if (data[field][value].set) {
data[field][value].set.delete(ln)
}
if (data[field][value].ranges) {
data[field][value].ranges = this._removeFromRanges(data[field][value].ranges, ln)
}
// Remove empty entries
if ((!data[field][value].set || data[field][value].set.size === 0) &&
(!data[field][value].ranges || data[field][value].ranges.length === 0)) {
delete data[field][value]
}
}
}
}
// Cleanup method to free memory
cleanup() {
this._clearIdleUnloadTimer()
const data = this.index.data
for (const field in data) {
for (const value in data[field]) {
if (data[field][value].set) {
if (typeof data[field][value].set.clearAll === 'function') {
data[field][value].set.clearAll()
} else if (typeof data[field][value].set.clear === 'function') {
data[field][value].set.clear()
}
}
if (data[field][value].ranges) {
data[field][value].ranges.length = 0
}
}
// Clear the entire field
data[field] = {}
}
// Clear all data
this.index.data = {}
this.totalLines = 0
}
// Clear all indexes
clear() {
this._clearIdleUnloadTimer()
this.index.data = {}
this.totalLines = 0
}
// Update a record in the index
async update(oldRecord, newRecord, lineNumber = null) {
this.markIndexUsed()
if (!oldRecord || !newRecord) return
// Remove old record by ID
await this.remove(oldRecord)
// Add new record with provided line number or use hash of the ID
const actualLineNumber = lineNumber !== null ? lineNumber : this._getIdAsNumber(newRecord.id)
await this.add(newRecord, actualLineNumber)
}
// Convert string ID to number for line number
_getIdAsNumber(id) {
if (typeof id === 'number') return id
if (typeof id === 'string') {
// Simple hash function to convert string to number
let hash = 0
for (let i = 0; i < id.length; i++) {
const char = id.charCodeAt(i)
hash = ((hash << 5) - hash) + char
hash = hash & hash // Convert to 32-bit integer
}
return Math.abs(hash)
}
return 0
}
// Remove a record from the index
async remove(record) {
this.markIndexUsed()
if (!record) return
// If record is an array of line numbers, use the original method
if (Array.isArray(record)) {
return this._removeLineNumbers(record)
}
// If record is an object, remove by record data
if (typeof record === 'object' && record.id) {
return await this._removeRecord(record)
}
}
// Remove a specific record from the index
async _removeRecord(record) {
if (!record) return
const data = this.index.data
const database = this.database
const persistedCount = Array.isArray(database?.offsets) ? database.offsets.length : 0
const lineMatchCache = new Map()
const doesLineNumberBelongToRecord = async (lineNumber) => {
if (lineMatchCache.has(lineNumber)) {
return lineMatchCache.get(lineNumber)
}
let belongs = false
try {
if (lineNumber >= persistedCount) {
const writeBufferIndex = lineNumber - persistedCount
const candidate = database?.writeBuffer?.[writeBufferIndex]
belongs = !!candidate && candidate.id === record.id
} else if (lineNumber >= 0) {
const range = database?.locate?.(lineNumber)
if (range && database.fileHandler && database.serializer) {
const [start, end] = range
const buffer = await database.fileHandler.readRange(start, end)
if (buffer && buffer.length > 0) {
let line = buffer.toString('utf8')
if (line) {
line = line.trim()
if (line.length > 0) {
const storedRecord = database.serializer.deserialize(line)
belongs = storedRecord && storedRecord.id === record.id
}
}
}
}
}
} catch (error) {
belongs = false
}
lineMatchCache.set(lineNumber, belongs)
return belongs
}
for (const field in data) {
if (record[field] !== undefined && record[field] !== null) {
const values = Array.isArray(record[field]) ? record[field] : [record[field]]
for (const val of values) {
let key
// Check if this is a term mapping field (array:string fields only)
const isTermMappingField = this.database?.termManager &&
this.database.termManager.termMappingFields &&
this.database.termManager.termMappingFields.includes(field)
if (isTermMappingField && typeof val === 'number') {
// For term mapping fields (array:string), the values are already term IDs
key = String(val)
if (this.database.opts.debugMode) {
console.log(`🔍 IndexManager._removeRecord: Using term ID ${val} directly for field "${field}"`)
}
} else if (isTermMappingField && typeof val === 'string') {
// For term mapping fields (array:string), convert string to term ID
const termId = this.database.termManager.getTermIdWithoutIncrement(val)
key = String(termId)
if (this.database.opts.debugMode) {
console.log(`🔍 IndexManager._removeRecord: Using term ID ${termId} for term "${val}"`)
}
} else {
// For non-term-mapping fields (including array:number), use values directly
key = String(val)
if (this.database?.opts?.debugMode) {
console.log(`🔍 IndexManager._removeRecord: Using value "${val}" directly for field "${field}"`)
}
}
// Note: TermManager notification is handled by Database.mjs
// to avoid double decrementation during updates
const indexEntry = data[field][key]
if (indexEntry) {
const lineNumbers = this._getAllLineNumbers(indexEntry)
const filteredLineNumbers = []
for (const lineNumber of lineNumbers) {
if (!(await doesLineNumberBelongToRecord(lineNumber))) {
filteredLineNumbers.push(lineNumber)
}
}
if (filteredLineNumbers.length === 0) {
delete data[field][key]
} else {
// Rebuild the index value with filtered line numbers
data[field][key].set = new Set(filteredLineNumbers)
data[field][key].ranges = []
}
}
}
}
}
}
// Ultra-fast remove with batch processing (renamed from remove)
_removeLineNumbers(lineNumbers) {
if (!lineNumbers || lineNumbers.length === 0) return
lineNumbers.sort((a, b) => a - b) // Sort ascending for efficient processing
const data = this.index.data
for (const field in data) {
for (const value in data[field]) {
const numbers = this._getAllLineNumbers(data[field][value])
const newNumbers = []
for (const ln of numbers) {
let offset = 0
for (const lineNumber of lineNumbers) {
if (lineNumber < ln) {
offset++
} else if (lineNumber === ln) {
offset = -1 // Mark for removal
break
}
}
if (offset >= 0) {
newNumbers.push(ln - offset) // Update the value
}
}
if (newNumbers.length > 0) {
// Rebuild hybrid structure with new numbers
data[field][value] = { set: new Set(), ranges: [] }
for (const num of newNumbers) {
data[field][value] = this._addLineNumber(data[field][value], num)
}
} else {
delete data[field][value]
}
}
}
}
// Ultra-fast replace with batch processing
replace(map) {
if (!map || map.size === 0) return
const data = this.index.data
for (const field in data) {
for (const value in data[field]) {
const numbers = this._getAllLineNumbers(data[field][value])
const newNumbers = []
for (const lineNumber of numbers) {
if (map.has(lineNumber)) {
newNumbers.push(map.get(lineNumber))
} else {
newNumbers.push(lineNumber)
}
}
// Rebuild hybrid structure with new numbers
data[field][value] = { set: new Set(), ranges: [] }
for (const num of newNumbers) {
data[field][value] = this._addLineNumber(data[field][value], num)
}
}
}
}
// Ultra-fast query with early exit and smart processing
query(criteria, options = {}) {
this.markIndexUsed()
if (typeof options === 'boolean') {
options = { matchAny: options };
}
const { matchAny = false, caseInsensitive = false } = options;
if (!criteria) {
// Return all line numbers when no criteria provided
return new Set(Array.from({ length: this.totalLines || 0 }, (_, i) => i));
}
// Handle $not operator
if (criteria.$not && typeof criteria.$not === 'object') {
// Get all possible line numbers from database offsets or totalLines
const totalRecords = this.database?.offsets?.length || this.totalLines || 0;
const allLines = new Set(Array.from({ length: totalRecords }, (_, i) => i));
// Get line numbers matching the $not condition
const notLines = this.query(criteria.$not, options);
// Return complement (all lines except those matching $not condition)
const result = new Set([...allLines].filter(x => !notLines.has(x)));
// If there are other conditions besides $not, we need to intersect with them
const otherCriteria = { ...criteria };
delete otherCriteria.$not;
if (Object.keys(otherCriteria).length > 0) {
const otherResults = this.query(otherCriteria, options);
return new Set([...result].filter(x => otherResults.has(x)));
}
return result;
}
// Handle $and queries with parallel processing optimization
if (criteria.$and && Array.isArray(criteria.$and)) {
// OPTIMIZATION: Process conditions in parallel for better performance
if (criteria.$and.length > 1) {
// Process all conditions in parallel (synchronous since query is not async)
const conditionResults = criteria.$and.map(andCondition =>
this.query(andCondition, options)
);
// Intersect all results for AND logic
let result = conditionResults[0];
for (let i = 1; i < conditionResults.length; i++) {
result = new Set([...result].filter(x => conditionResults[i].has(x)));
}
// IMPORTANT: Check if there are other fields besides $and at the root level
// If so, we need to intersect with them too
const otherCriteria = { ...criteria };
delete otherCriteria.$and;
if (Object.keys(otherCriteria).length > 0) {
const otherResults = this.query(otherCriteria, options);
result = new Set([...result].filter(x => otherResults.has(x)));
}
return result || new Set();
} else {
// Single condition - check for other criteria at root level
const andResult = this.query(criteria.$and[0], options);
const otherCriteria = { ...criteria };
delete otherCriteria.$and;
if (Object.keys(otherCriteria).length > 0) {
const otherResults = this.query(otherCriteria, options);
return new Set([...andResult].filter(x => otherResults.has(x)));
}
return andResult;
}
}
const fields = Object.keys(criteria);
if (!fields.length) {
// Return all line numbers when criteria is empty object
return new Set(Array.from({ length: this.totalLines || 0 }, (_, i) => i));
}
let matchingLines = matchAny ? new Set() : null;
const data = this.index.data
for (const field of fields) {
// Skip logical operators - they are handled separately
if (field.startsWith('$')) continue;
if (typeof data[field] === 'undefined') continue;
const originalCriteriaValue = criteria[field];
const criteriaValue = normalizeCriteriaOperators(originalCriteriaValue, { target: 'legacy', preserveOriginal: true });
let lineNumbersForField = new Set();
const isNumericField = this.opts.indexes[field] === 'number';
// Handle RegExp values directly (MUST check before object check since RegExp is an object)
if (criteriaValue instanceof RegExp) {
// RegExp cannot be efficiently queried using indices - fall back to streaming
// This will be handled by the QueryManager's streaming strategy
continue;
}
if (typeof criteriaValue === 'object' && !Array.isArray(criteriaValue) && criteriaValue !== null) {
const fieldIndex = data[field];
// Handle $in operator for array queries
if (criteriaValue.$in !== undefined && criteriaValue.$in !== null) {
const inValues = Array.isArray(criteriaValue.$in) ? criteriaValue.$in : [criteriaValue.$in];
// PERFORMANCE: Cache term mapping field check once
const isTermMappingField = this.database?.termManager &&
this.database.termManager.termMappingFields &&
this.database.termManager.termMappingFields.includes(field)
// PERFORMANCE: Track if any term was found and matched
let foundAnyMatch = false
for (const inValue of inValues) {
// SPACE OPTIMIZATION: Convert search term to term ID for lookup
let searchTermId
if (isTermMappingField && typeof inValue === 'number') {
// For term mapping fields (array:string), the search value is already a term ID
searchTermId = String(inValue)
} else if (isTermMappingField && typeof inValue === 'string') {
// For term mapping fields (array:string), convert string to term ID
const termId = this.database?.termManager?.getTermIdWithoutIncrement(String(inValue))
if (termId === undefined) {
// Term not found in termManager - skip this search value
// This means the term was never saved to the database
if (this.opts?.debugMode) {
console.log(`⚠️ Term "${inValue}" not found in termManager for field "${field}" - skipping`)
}
continue // Skip this value, no matches possible
}
searchTermId = String(termId)
} else {
// For non-term-mapping fields (including array:number), use values directly
searchTermId = String(inValue)
}
// PERFORMANCE: Direct lookup instead of iteration
let matched = false
if (caseInsensitive && typeof inValue === 'string') {
const searchLower = searchTermId.toLowerCase()
for (const value in fieldIndex) {
if (value.toLowerCase() === searchLower) {
const numbers = this._getAllLineNumbers(fieldIndex[value]);
for (const lineNumber of numbers) {
lineNumbersForField.add(lineNumber);
}
matched = true
foundAnyMatch = true
}
}
} else {
const indexData = fieldIndex[searchTermId]
if (indexData) {
const numbers = this._getAllLineNumbers(indexData);
for (const lineNumber of numbers) {
lineNumbersForField.add(lineNumber);
}
matched = true
foundAnyMatch = true
}
}
}
// CRITICAL FIX: If no matches found at all (all terms were unknown or not in index),
// lineNumbersForField remains empty which is correct (no results for $in)
// This is handled correctly by the caller - empty Set means no matches
}
// Handle $nin operator (not in) - returns complement of $in
else if (criteriaValue.$nin !== undefined) {
const ninValues = Array.isArray(criteriaValue.$nin) ? criteriaValue.$nin : [criteriaValue.$nin];
// Get all possible line numbers
const totalRecords = this.database?.offsets?.length || this.totalLines || 0;
const allLines = new Set(Array.from({ length: totalRecords }, (_, i) => i));
// Get line numbers that match any of the $nin values
const matchingLines = new Set();
// PERFORMANCE: Cache term mapping field check once
const isTermMappingField = this.database?.termManager &&
this.database.termManager.termMappingFields &&
this.database.termManager.termMappingFields.includes(field)
for (const ninValue of ninValues) {
// SPACE OPTIMIZATION: Convert search term to term ID for lookup
let searchTermId
if (isTermMappingField && typeof ninValue === 'number') {
// For term mapping fields (array:string), the search value is already a term ID
searchTermId = String(ninValue)
} else if (isTermMappingField && typeof ninValue === 'string') {
// For term mapping fields (array:string), convert string to term ID
const termId = this.database?.termManager?.getTermIdWithoutIncrement(String(ninValue))
if (termId === undefined) {
// Term not found - skip this value (can't exclude what doesn't exist)
if (this.opts?.debugMode) {
console.log(`⚠️ Term "${ninValue}" not found in termManager for field "${field}" - skipping`)
}
continue
}
searchTermId = String(termId)
} else {
// For non-term-mapping fields (including array:number), use values directly
searchTermId = String(ninValue)
}
// PERFORMANCE: Direct lookup instead of iteration
if (caseInsensitive && typeof ninValue === 'string') {
const searchLower = searchTermId.toLowerCase()
for (const value in fieldIndex) {
if (value.toLowerCase() === searchLower) {
const numbers = this._getAllLineNumbers(fieldIndex[value]);
for (const lineNumber of numbers) {
matchingLines.add(lineNumber);
}
}
}
} else {
const indexData = fieldIndex[searchTermId]
if (indexData) {
const numbers = this._getAllLineNumbers(indexData);
for (const lineNumber of numbers) {
matchingLines.add(lineNumber);
}
}
}
}
// Return complement: all lines EXCEPT those matching $nin values
lineNumbersForField = new Set([...allLines].filter(x => !matchingLines.has(x)));
}
// Handle $contains operator for array queries
else if (criteriaValue.$contains !== undefined) {
const containsValue = criteriaValue.$contains;
// Handle case-insensitive for $contains
if (caseInsensitive && typeof containsValue === 'string') {
for (const value in fieldIndex) {
if (value.toLowerCase() === containsValue.toLowerCase()) {
const numbers = this._getAllLineNumbers(fieldIndex[value]);
for (const lineNumber of numbers) {
lineNumbersForField.add(lineNumber);
}
}
}
} else {
if (fieldIndex[containsValue]) {
const numbers = this._getAllLineNumbers(fieldIndex[containsValue]);
for (const lineNumber of numbers) {
lineNumbersForField.add(lineNumber);
}
}
}
}
// Handle $all operator for array queries - FIXED FOR TERM MAPPING
else if (criteriaValue.$all !== undefined) {
const allValues = Array.isArray(criteriaValue.$all) ? criteriaValue.$all : [criteriaValue.$all];
const isTermMappingField = this.database?.termManager &&
this.database.termManager.termMappingFields &&
this.database.termManager.termMappingFields.includes(field)
const normalizeValue = (value) => {
if (isTermMappingField) {
if (typeof value === 'number') {
return String(value)
}
if (typeof value === 'string') {
const termId = this.database?.termManager?.getTermIdWithoutIncrement(value)
if (termId !== undefined) {
return String(termId)
}
return null
}
return null
}
return String(value)
}
const normalizedValues = []
for (const value of allValues) {
const normalized = normalizeValue(value)
if (normalized === null) {
// Term not found in term manager, no matches possible
return lineNumbersForField
}
normalizedValues.push(normalized)
}
// Early exit optimization
if (normalizedValues.length === 0) {
// Empty $all matches everything
for (const value in fieldIndex) {
const numbers = this._getAllLineNumbers(fieldIndex[value]);
for (const lineNumber of numbers) {
lineNumbersForField.add(lineNumber);
}
}
} else {
// For term mapping, we need to find records that contain ALL specified terms
// This requires a different approach than simple field matching
// First, get all line numbers that contain each individual term
const termLineNumbers = new Map();
for (const term of normalizedValues) {
if (fieldIndex[term]) {
termLineNumbers.set(term, new Set(this._getAllLineNumbers(fieldIndex[term])));
} else {
// If any term doesn't exist, no records can match $all
termLineNumbers.set(term, new Set());
}
}
// Find intersection of all term line numbers
if (termLineNumbers.size > 0) {
const allTermSets = Array.from(termLineNumbers.values());
let intersection = allTermSets[0];
for (let i = 1; i < allTermSets.length; i++) {
intersection = new Set([...intersection].filter(x => allTermSets[i].has(x)));
}
// Add all line numbers from intersection
for (const lineNumber of intersection) {
lineNumbersForField.add(lineNumber);
}
}
}
}
// Handle other operators
else {
for (const value in fieldIndex) {
let includeValue = true;
if (isNumericField) {
const numericValue = parseFloat(value);
if (!isNaN(numericValue)) {
if (criteriaValue['>'] !== undefined && numericValue <= criteriaValue['>']) {
includeValue = false;
}
if (criteriaValue['>='] !== undefined && numericValue < criteriaValue['>=']) {
includeValue = false;
}
if (criteriaValue['<'] !== undefined && numericValue >= criteriaValue['<']) {
includeValue = false;
}
if (criteriaValue['<='] !== undefined && numericValue > criteriaValue['<=']) {
includeValue = false;
}
if (criteriaValue['!='] !== undefined) {
const excludeValues = Array.isArray(criteriaValue['!='])
? criteriaValue['!=']
: [criteriaValue['!=']];
if (excludeValues.includes(numericValue)) {
includeValue = false;
}
}
}
} else {
if (criteriaValue['contains'] !== undefined && typeof value === 'string') {
const term = String(criteriaValue['contains']);
if (caseInsensitive) {
if (!value.toLowerCase().includes(term.toLowerCase())) {
includeValue = false;
}
} else {
if (!value.includes(term)) {
includeValue = false;
}
}
}
if (criteriaValue['regex'] !== undefined) {
let regex;
if (typeof criteriaValue['regex'] === 'string') {
regex = new RegExp(criteriaValue['regex'], caseInsensitive ? 'i' : '');
} else if (criteriaValue['regex'] instanceof RegExp) {
if (caseInsensitive && !criteriaValue['regex'].ignoreCase) {
const flags = criteriaVa