jexidb
Version:
JexiDB is a pure JS NPM library for managing data on disk efficiently, without the need for a server.
1,441 lines (1,355 loc) • 565 kB
JavaScript
'use strict';
var events = require('events');
var fs = require('fs');
var readline = require('readline');
var path = require('path');
function _OverloadYield(e, d) {
this.v = e, this.k = d;
}
function _asyncIterator(r) {
var n,
t,
o,
e = 2;
for ("undefined" != typeof Symbol && (t = Symbol.asyncIterator, o = Symbol.iterator); e--;) {
if (t && null != (n = r[t])) return n.call(r);
if (o && null != (n = r[o])) return new AsyncFromSyncIterator(n.call(r));
t = "@@asyncIterator", o = "@@iterator";
}
throw new TypeError("Object is not async iterable");
}
function AsyncFromSyncIterator(r) {
function AsyncFromSyncIteratorContinuation(r) {
if (Object(r) !== r) return Promise.reject(new TypeError(r + " is not an object."));
var n = r.done;
return Promise.resolve(r.value).then(function (r) {
return {
value: r,
done: n
};
});
}
return AsyncFromSyncIterator = function (r) {
this.s = r, this.n = r.next;
}, AsyncFromSyncIterator.prototype = {
s: null,
n: null,
next: function () {
return AsyncFromSyncIteratorContinuation(this.n.apply(this.s, arguments));
},
return: function (r) {
var n = this.s.return;
return void 0 === n ? Promise.resolve({
value: r,
done: true
}) : AsyncFromSyncIteratorContinuation(n.apply(this.s, arguments));
},
throw: function (r) {
var n = this.s.return;
return void 0 === n ? Promise.reject(r) : AsyncFromSyncIteratorContinuation(n.apply(this.s, arguments));
}
}, new AsyncFromSyncIterator(r);
}
function _awaitAsyncGenerator(e) {
return new _OverloadYield(e, 0);
}
function _wrapAsyncGenerator(e) {
return function () {
return new AsyncGenerator(e.apply(this, arguments));
};
}
function AsyncGenerator(e) {
var r, t;
function resume(r, t) {
try {
var n = e[r](t),
o = n.value,
u = o instanceof _OverloadYield;
Promise.resolve(u ? o.v : o).then(function (t) {
if (u) {
var i = "return" === r ? "return" : "next";
if (!o.k || t.done) return resume(i, t);
t = e[i](t).value;
}
settle(n.done ? "return" : "normal", t);
}, function (e) {
resume("throw", e);
});
} catch (e) {
settle("throw", e);
}
}
function settle(e, n) {
switch (e) {
case "return":
r.resolve({
value: n,
done: true
});
break;
case "throw":
r.reject(n);
break;
default:
r.resolve({
value: n,
done: false
});
}
(r = r.next) ? resume(r.key, r.arg) : t = null;
}
this._invoke = function (e, n) {
return new Promise(function (o, u) {
var i = {
key: e,
arg: n,
resolve: o,
reject: u,
next: null
};
t ? t = t.next = i : (r = t = i, resume(e, n));
});
}, "function" != typeof e.return && (this.return = void 0);
}
AsyncGenerator.prototype["function" == typeof Symbol && Symbol.asyncIterator || "@@asyncIterator"] = function () {
return this;
}, AsyncGenerator.prototype.next = function (e) {
return this._invoke("next", e);
}, AsyncGenerator.prototype.throw = function (e) {
return this._invoke("throw", e);
}, AsyncGenerator.prototype.return = function (e) {
return this._invoke("return", e);
};
const E_CANCELED = new Error('request for lock canceled');
var __awaiter$2 = function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
class Semaphore {
constructor(_value, _cancelError = E_CANCELED) {
this._value = _value;
this._cancelError = _cancelError;
this._queue = [];
this._weightedWaiters = [];
}
acquire(weight = 1, priority = 0) {
if (weight <= 0)
throw new Error(`invalid weight ${weight}: must be positive`);
return new Promise((resolve, reject) => {
const task = { resolve, reject, weight, priority };
const i = findIndexFromEnd(this._queue, (other) => priority <= other.priority);
if (i === -1 && weight <= this._value) {
// Needs immediate dispatch, skip the queue
this._dispatchItem(task);
}
else {
this._queue.splice(i + 1, 0, task);
}
});
}
runExclusive(callback_1) {
return __awaiter$2(this, arguments, void 0, function* (callback, weight = 1, priority = 0) {
const [value, release] = yield this.acquire(weight, priority);
try {
return yield callback(value);
}
finally {
release();
}
});
}
waitForUnlock(weight = 1, priority = 0) {
if (weight <= 0)
throw new Error(`invalid weight ${weight}: must be positive`);
if (this._couldLockImmediately(weight, priority)) {
return Promise.resolve();
}
else {
return new Promise((resolve) => {
if (!this._weightedWaiters[weight - 1])
this._weightedWaiters[weight - 1] = [];
insertSorted(this._weightedWaiters[weight - 1], { resolve, priority });
});
}
}
isLocked() {
return this._value <= 0;
}
getValue() {
return this._value;
}
setValue(value) {
this._value = value;
this._dispatchQueue();
}
release(weight = 1) {
if (weight <= 0)
throw new Error(`invalid weight ${weight}: must be positive`);
this._value += weight;
this._dispatchQueue();
}
cancel() {
this._queue.forEach((entry) => entry.reject(this._cancelError));
this._queue = [];
}
_dispatchQueue() {
this._drainUnlockWaiters();
while (this._queue.length > 0 && this._queue[0].weight <= this._value) {
this._dispatchItem(this._queue.shift());
this._drainUnlockWaiters();
}
}
_dispatchItem(item) {
const previousValue = this._value;
this._value -= item.weight;
item.resolve([previousValue, this._newReleaser(item.weight)]);
}
_newReleaser(weight) {
let called = false;
return () => {
if (called)
return;
called = true;
this.release(weight);
};
}
_drainUnlockWaiters() {
if (this._queue.length === 0) {
for (let weight = this._value; weight > 0; weight--) {
const waiters = this._weightedWaiters[weight - 1];
if (!waiters)
continue;
waiters.forEach((waiter) => waiter.resolve());
this._weightedWaiters[weight - 1] = [];
}
}
else {
const queuedPriority = this._queue[0].priority;
for (let weight = this._value; weight > 0; weight--) {
const waiters = this._weightedWaiters[weight - 1];
if (!waiters)
continue;
const i = waiters.findIndex((waiter) => waiter.priority <= queuedPriority);
(i === -1 ? waiters : waiters.splice(0, i))
.forEach((waiter => waiter.resolve()));
}
}
}
_couldLockImmediately(weight, priority) {
return (this._queue.length === 0 || this._queue[0].priority < priority) &&
weight <= this._value;
}
}
function insertSorted(a, v) {
const i = findIndexFromEnd(a, (other) => v.priority <= other.priority);
a.splice(i + 1, 0, v);
}
function findIndexFromEnd(a, predicate) {
for (let i = a.length - 1; i >= 0; i--) {
if (predicate(a[i])) {
return i;
}
}
return -1;
}
var __awaiter$1 = function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
class Mutex {
constructor(cancelError) {
this._semaphore = new Semaphore(1, cancelError);
}
acquire() {
return __awaiter$1(this, arguments, void 0, function* (priority = 0) {
const [, releaser] = yield this._semaphore.acquire(1, priority);
return releaser;
});
}
runExclusive(callback, priority = 0) {
return this._semaphore.runExclusive(() => callback(), 1, priority);
}
isLocked() {
return this._semaphore.isLocked();
}
waitForUnlock(priority = 0) {
return this._semaphore.waitForUnlock(1, priority);
}
release() {
if (this._semaphore.isLocked())
this._semaphore.release();
}
cancel() {
return this._semaphore.cancel();
}
}
const aliasToCanonical = {
'>': '$gt',
'>=': '$gte',
'<': '$lt',
'<=': '$lte',
'!=': '$ne',
'=': '$eq',
'==': '$eq',
eq: '$eq',
equals: '$eq',
in: '$in',
nin: '$nin',
regex: '$regex',
contains: '$contains',
all: '$all',
exists: '$exists',
size: '$size',
not: '$not'
};
const canonicalToLegacy = {
'$gt': '>',
'$gte': '>=',
'$lt': '<',
'$lte': '<=',
'$ne': '!=',
'$eq': '=',
'$contains': 'contains',
'$regex': 'regex'
};
/**
* Normalize an operator to its canonical Mongo-style representation (prefixed with $)
* @param {string} operator
* @returns {string}
*/
function normalizeOperator(operator) {
if (typeof operator !== 'string') {
return operator;
}
if (operator.startsWith('$')) {
return operator;
}
if (aliasToCanonical[operator] !== undefined) {
return aliasToCanonical[operator];
}
const lowerCase = operator.toLowerCase();
if (aliasToCanonical[lowerCase] !== undefined) {
return aliasToCanonical[lowerCase];
}
return operator;
}
/**
* Convert an operator to its legacy (non-prefixed) alias when available
* @param {string} operator
* @returns {string}
*/
function operatorToLegacy(operator) {
if (typeof operator !== 'string') {
return operator;
}
const canonical = normalizeOperator(operator);
if (canonicalToLegacy[canonical]) {
return canonicalToLegacy[canonical];
}
return operator;
}
/**
* Normalize operator keys in a criteria object
* @param {Object} criteriaValue
* @param {Object} options
* @param {'canonical'|'legacy'} options.target - Preferred operator style
* @param {boolean} [options.preserveOriginal=false] - Whether to keep the original keys alongside normalized ones
* @returns {Object}
*/
function normalizeCriteriaOperators(criteriaValue, {
target = 'canonical',
preserveOriginal = false
} = {}) {
if (!criteriaValue || typeof criteriaValue !== 'object' || Array.isArray(criteriaValue)) {
return criteriaValue;
}
const normalized = preserveOriginal ? {
...criteriaValue
} : {};
for (const [operator, value] of Object.entries(criteriaValue)) {
const canonical = normalizeOperator(operator);
if (target === 'canonical') {
normalized[canonical] = value;
if (preserveOriginal && canonical !== operator) {
normalized[operator] = value;
}
} else if (target === 'legacy') {
const legacy = operatorToLegacy(operator);
normalized[legacy] = value;
if (preserveOriginal) {
if (legacy !== canonical) {
normalized[canonical] = value;
}
if (operator !== legacy && operator !== canonical) {
normalized[operator] = value;
}
}
}
}
return normalized;
}
class IndexManager {
constructor(opts, databaseMutex = null, database = null) {
this.opts = Object.assign({}, opts);
this.index = Object.assign({
data: {}
}, this.opts.index);
this.totalLines = 0;
this.rangeThreshold = 10; // Sensible threshold: 10+ consecutive numbers justify ranges
this.binarySearchThreshold = 32; // Much higher for better performance
this.database = database; // Reference to database for term manager access
// CRITICAL: Use database mutex to prevent deadlocks
// If no database mutex provided, create a local one (for backward compatibility)
this.mutex = databaseMutex || new Mutex();
this.indexedFields = [];
this.setIndexesConfig(this.opts.indexes);
}
setTotalLines(total) {
this.totalLines = total;
}
/**
* Update indexes configuration and ensure internal structures stay in sync
* @param {Object|Array<string>} indexes
*/
setIndexesConfig(indexes) {
if (!indexes) {
this.opts.indexes = undefined;
this.indexedFields = [];
return;
}
if (Array.isArray(indexes)) {
const fields = indexes.map(field => String(field));
this.indexedFields = fields;
const normalizedConfig = {};
for (const field of fields) {
const existingConfig = !Array.isArray(this.opts.indexes) && typeof this.opts.indexes === 'object' ? this.opts.indexes[field] : undefined;
normalizedConfig[field] = existingConfig ?? 'auto';
if (!this.index.data[field]) {
this.index.data[field] = {};
}
}
this.opts.indexes = normalizedConfig;
return;
}
if (typeof indexes === 'object') {
this.opts.indexes = Object.assign({}, indexes);
this.indexedFields = Object.keys(this.opts.indexes);
for (const field of this.indexedFields) {
if (!this.index.data[field]) {
this.index.data[field] = {};
}
}
}
}
/**
* Check if a field is configured as an index
* @param {string} field - Field name
* @returns {boolean}
*/
isFieldIndexed(field) {
if (!field) return false;
if (!Array.isArray(this.indexedFields)) {
return false;
}
return this.indexedFields.includes(field);
}
/**
* Determine whether the index has usable data for a given field
* @param {string} field - Field name
* @returns {boolean}
*/
hasUsableIndexData(field) {
if (!field) return false;
const fieldData = this.index?.data?.[field];
if (!fieldData || typeof fieldData !== 'object') {
return false;
}
for (const key in fieldData) {
if (!Object.prototype.hasOwnProperty.call(fieldData, key)) continue;
const entry = fieldData[key];
if (!entry) continue;
if (entry.set && typeof entry.set.size === 'number' && entry.set.size > 0) {
return true;
}
if (Array.isArray(entry.ranges) && entry.ranges.length > 0) {
const hasRangeData = entry.ranges.some(range => {
if (range === null || typeof range === 'undefined') {
return false;
}
if (typeof range === 'object') {
const count = typeof range.count === 'number' ? range.count : 0;
return count > 0;
}
// When ranges are stored as individual numbers
return true;
});
if (hasRangeData) {
return true;
}
}
}
return false;
}
// Ultra-fast range conversion - only for very large datasets
_toRanges(numbers) {
if (numbers.length === 0) return [];
if (numbers.length < this.rangeThreshold) return numbers; // Keep as-is for small arrays
const sorted = numbers.sort((a, b) => a - b); // Sort in-place
const ranges = [];
let start = sorted[0];
let count = 1;
for (let i = 1; i < sorted.length; i++) {
if (sorted[i] === sorted[i - 1] + 1) {
count++;
} else {
// End of consecutive sequence
if (count >= this.rangeThreshold) {
ranges.push({
start,
count
});
} else {
// Add individual numbers for small sequences
for (let j = start; j < start + count; j++) {
ranges.push(j);
}
}
start = sorted[i];
count = 1;
}
}
// Handle last sequence
if (count >= this.rangeThreshold) {
ranges.push({
start,
count
});
} else {
for (let j = start; j < start + count; j++) {
ranges.push(j);
}
}
return ranges;
}
// Ultra-fast range expansion
_fromRanges(ranges) {
if (!ranges || ranges.length === 0) return [];
const numbers = [];
for (const item of ranges) {
if (typeof item === 'object' && item.start !== undefined) {
// It's a range - use direct loop for maximum speed
const end = item.start + item.count;
for (let i = item.start; i < end; i++) {
numbers.push(i);
}
} else {
// It's an individual number
numbers.push(item);
}
}
return numbers;
}
// Ultra-fast lookup - optimized for Set operations
_hasLineNumber(hybridData, lineNumber) {
if (!hybridData) return false;
// Check in Set first (O(1)) - most common case
if (hybridData.set && hybridData.set.has(lineNumber)) {
return true;
}
// Check in ranges only if necessary
if (hybridData.ranges && hybridData.ranges.length > 0) {
return this._searchInRanges(hybridData.ranges, lineNumber);
}
return false;
}
// Optimized search strategy
_searchInRanges(ranges, lineNumber) {
if (ranges.length < this.binarySearchThreshold) {
// Linear search for small ranges
return this._linearSearchRanges(ranges, lineNumber);
} else {
// Binary search for large ranges
return this._binarySearchRanges(ranges, lineNumber);
}
}
// Ultra-fast linear search
_linearSearchRanges(ranges, lineNumber) {
for (const item of ranges) {
if (typeof item === 'object' && item.start !== undefined) {
// It's a range
if (lineNumber >= item.start && lineNumber < item.start + item.count) {
return true;
}
} else if (item === lineNumber) {
// It's an individual number
return true;
}
}
return false;
}
// Optimized binary search
_binarySearchRanges(ranges, lineNumber) {
let left = 0;
let right = ranges.length - 1;
while (left <= right) {
const mid = Math.floor((left + right) / 2);
const range = ranges[mid];
if (typeof range === 'object' && range.start !== undefined) {
// It's a range
if (lineNumber >= range.start && lineNumber < range.start + range.count) {
return true;
} else if (lineNumber < range.start) {
right = mid - 1;
} else {
left = mid + 1;
}
} else {
// It's an individual number
if (range === lineNumber) {
return true;
} else if (range < lineNumber) {
left = mid + 1;
} else {
right = mid - 1;
}
}
}
return false;
}
// Ultra-fast add operation - minimal overhead
_addLineNumber(hybridData, lineNumber) {
// Initialize structure if needed
if (!hybridData) {
hybridData = {
set: new Set(),
ranges: []
};
}
// Add to Set directly (fastest path)
if (!hybridData.set) {
hybridData.set = new Set();
}
hybridData.set.add(lineNumber);
// Optimize to ranges when Set gets reasonably large
if (hybridData.set.size >= this.rangeThreshold * 2) {
// 20 elements
if (this.opts.debugMode) {
console.log(`🔧 Triggering range optimization: Set size ${hybridData.set.size} >= threshold ${this.rangeThreshold * 2}`);
}
this._optimizeToRanges(hybridData);
}
return hybridData;
}
// Ultra-fast remove operation
_removeLineNumber(hybridData, lineNumber) {
if (!hybridData) {
return hybridData;
}
// Remove from Set (fast path)
if (hybridData.set) {
hybridData.set.delete(lineNumber);
}
// Remove from ranges (less common)
if (hybridData.ranges) {
hybridData.ranges = this._removeFromRanges(hybridData.ranges, lineNumber);
}
return hybridData;
}
// Optimized range removal
_removeFromRanges(ranges, lineNumber) {
if (!ranges || ranges.length === 0) return ranges;
const newRanges = [];
for (const item of ranges) {
if (typeof item === 'object' && item.start !== undefined) {
// It's a range
if (lineNumber >= item.start && lineNumber < item.start + item.count) {
// Split range if needed
if (lineNumber === item.start) {
// Remove first element
if (item.count > 1) {
newRanges.push({
start: item.start + 1,
count: item.count - 1
});
}
} else if (lineNumber === item.start + item.count - 1) {
// Remove last element
if (item.count > 1) {
newRanges.push({
start: item.start,
count: item.count - 1
});
}
} else {
// Remove from middle - split into two ranges
const beforeCount = lineNumber - item.start;
const afterCount = item.count - beforeCount - 1;
if (beforeCount >= this.rangeThreshold) {
newRanges.push({
start: item.start,
count: beforeCount
});
} else {
// Add individual numbers for small sequences
for (let i = item.start; i < lineNumber; i++) {
newRanges.push(i);
}
}
if (afterCount >= this.rangeThreshold) {
newRanges.push({
start: lineNumber + 1,
count: afterCount
});
} else {
// Add individual numbers for small sequences
for (let i = lineNumber + 1; i < item.start + item.count; i++) {
newRanges.push(i);
}
}
}
} else {
newRanges.push(item);
}
} else if (item !== lineNumber) {
// It's an individual number
newRanges.push(item);
}
}
return newRanges;
}
// Ultra-lazy range conversion - only when absolutely necessary
_optimizeToRanges(hybridData) {
if (!hybridData.set || hybridData.set.size === 0) {
return;
}
if (this.opts.debugMode) {
console.log(`🔧 Starting range optimization for Set with ${hybridData.set.size} elements`);
}
// Only convert if we have enough data to make it worthwhile
if (hybridData.set.size < this.rangeThreshold) {
return;
}
// Convert Set to array and find consecutive sequences
const numbers = Array.from(hybridData.set).sort((a, b) => a - b);
const ranges = [];
let start = numbers[0];
let count = 1;
for (let i = 1; i < numbers.length; i++) {
if (numbers[i] === numbers[i - 1] + 1) {
count++;
} else {
// End of consecutive sequence
if (count >= this.rangeThreshold) {
ranges.push({
start,
count
});
// Remove these numbers from Set
for (let j = start; j < start + count; j++) {
hybridData.set.delete(j);
}
}
start = numbers[i];
count = 1;
}
}
// Handle last sequence
if (count >= this.rangeThreshold) {
ranges.push({
start,
count
});
for (let j = start; j < start + count; j++) {
hybridData.set.delete(j);
}
}
// Add new ranges to existing ranges
if (ranges.length > 0) {
if (!hybridData.ranges) {
hybridData.ranges = [];
}
hybridData.ranges.push(...ranges);
// Keep ranges sorted for efficient binary search
hybridData.ranges.sort((a, b) => {
const aStart = typeof a === 'object' ? a.start : a;
const bStart = typeof b === 'object' ? b.start : b;
return aStart - bStart;
});
}
}
// Ultra-fast get all line numbers
_getAllLineNumbers(hybridData) {
if (!hybridData) return [];
// Use generator for lazy evaluation and better memory efficiency
return Array.from(this._getAllLineNumbersGenerator(hybridData));
}
// OPTIMIZATION: Generator-based approach for better memory efficiency
*_getAllLineNumbersGenerator(hybridData) {
const normalizeLineNumber = value => {
if (typeof value === 'number') {
return value;
}
if (typeof value === 'string') {
const parsed = Number(value);
return Number.isNaN(parsed) ? value : parsed;
}
if (typeof value === 'bigint') {
const maxSafe = BigInt(Number.MAX_SAFE_INTEGER);
return value <= maxSafe ? Number(value) : value;
}
return value;
};
// Yield from Set (fastest path)
if (hybridData.set) {
for (const num of hybridData.set) {
yield normalizeLineNumber(num);
}
}
// Yield from ranges (optimized)
if (hybridData.ranges) {
for (const item of hybridData.ranges) {
if (typeof item === 'object' && item.start !== undefined) {
// It's a range - use direct loop for better performance
const end = item.start + item.count;
for (let i = item.start; i < end; i++) {
yield normalizeLineNumber(i);
}
} else {
// It's an individual number
yield normalizeLineNumber(item);
}
}
}
}
// OPTIMIZATION 6: Ultra-fast add operation with incremental index updates
async add(row, lineNumber) {
if (typeof row !== 'object' || !row) {
throw new Error('Invalid \'row\' parameter, it must be an object');
}
if (typeof lineNumber !== 'number') {
throw new Error('Invalid line number');
}
// OPTIMIZATION 6: Use direct field access with minimal operations
const data = this.index.data;
// OPTIMIZATION 6: Pre-allocate field structures for better performance
const fields = Object.keys(this.opts.indexes || {});
for (const field of fields) {
// PERFORMANCE: Check if this is a term mapping field once
const isTermMappingField = this.database?.termManager && this.database.termManager.termMappingFields && this.database.termManager.termMappingFields.includes(field);
// CRITICAL FIX: For term mapping fields, prefer ${field}Ids if available
// Records processed by processTermMapping have term IDs in ${field}Ids
// Records loaded from file have term IDs directly in ${field} (after restoreTermIdsAfterDeserialization)
let value;
if (isTermMappingField) {
const termIdsField = `${field}Ids`;
const termIds = row[termIdsField];
if (termIds && Array.isArray(termIds) && termIds.length > 0) {
// Use term IDs from ${field}Ids (preferred - from processTermMapping)
value = termIds;
} else {
// Fallback: use field directly (for records loaded from file that have term IDs in field)
value = row[field];
}
} else {
value = row[field];
}
if (value !== undefined && value !== null) {
// OPTIMIZATION 6: Initialize field structure if it doesn't exist
if (!data[field]) {
data[field] = {};
}
const values = Array.isArray(value) ? value : [value];
for (const val of values) {
let key;
if (isTermMappingField && typeof val === 'number') {
// For term mapping fields, values are already term IDs
key = String(val);
} else if (isTermMappingField && typeof val === 'string') {
// Fallback: convert string to term ID
// CRITICAL: During indexing (add), we should use getTermId() to create IDs if needed
// This is different from queries where we use getTermIdWithoutIncrement() to avoid creating new IDs
const termId = this.database.termManager.getTermId(val);
key = String(termId);
} else {
// For non-term-mapping fields (including array:number), use values directly
key = String(val);
}
// OPTIMIZATION 6: Use direct assignment for better performance
if (!data[field][key]) {
data[field][key] = {
set: new Set(),
ranges: []
};
}
// OPTIMIZATION 6: Direct Set operation - fastest possible
data[field][key].set.add(lineNumber);
// OPTIMIZATION 6: Lazy range optimization - only when beneficial
if (data[field][key].set.size >= this.rangeThreshold * 3) {
this._optimizeToRanges(data[field][key]);
}
}
}
}
}
/**
* OPTIMIZATION 6: Add multiple records to the index in batch with optimized operations
* @param {Array} records - Records to add
* @param {number} startLineNumber - Starting line number
*/
async addBatch(records, startLineNumber) {
if (!records || !records.length) return;
// OPTIMIZATION 6: Pre-allocate index structures for better performance
const data = this.index.data;
const fields = Object.keys(this.opts.indexes || {});
for (const field of fields) {
if (!data[field]) {
data[field] = {};
}
}
// OPTIMIZATION 6: Use Map for batch processing to reduce lookups
const fieldUpdates = new Map();
// OPTIMIZATION 6: Process all records in batch with optimized data structures
for (let i = 0; i < records.length; i++) {
const row = records[i];
const lineNumber = startLineNumber + i;
for (const field of fields) {
// PERFORMANCE: Check if this is a term mapping field once
const isTermMappingField = this.database?.termManager && this.database.termManager.termMappingFields && this.database.termManager.termMappingFields.includes(field);
// CRITICAL FIX: For term mapping fields, prefer ${field}Ids if available
// Records processed by processTermMapping have term IDs in ${field}Ids
// Records loaded from file have term IDs directly in ${field} (after restoreTermIdsAfterDeserialization)
let value;
if (isTermMappingField) {
const termIdsField = `${field}Ids`;
const termIds = row[termIdsField];
if (termIds && Array.isArray(termIds) && termIds.length > 0) {
// Use term IDs from ${field}Ids (preferred - from processTermMapping)
value = termIds;
} else {
// Fallback: use field directly (for records loaded from file that have term IDs in field)
value = row[field];
}
} else {
value = row[field];
}
if (value !== undefined && value !== null) {
const values = Array.isArray(value) ? value : [value];
for (const val of values) {
let key;
if (isTermMappingField && typeof val === 'number') {
// For term mapping fields, values are already term IDs
key = String(val);
} else if (isTermMappingField && typeof val === 'string') {
// Fallback: convert string to term ID
// CRITICAL: During indexing (addBatch), we should use getTermId() to create IDs if needed
// This is different from queries where we use getTermIdWithoutIncrement() to avoid creating new IDs
const termId = this.database.termManager.getTermId(val);
key = String(termId);
} else {
// For non-term-mapping fields (including array:number), use values directly
key = String(val);
}
// OPTIMIZATION 6: Use Map for efficient batch updates
if (!fieldUpdates.has(field)) {
fieldUpdates.set(field, new Map());
}
const fieldMap = fieldUpdates.get(field);
if (!fieldMap.has(key)) {
fieldMap.set(key, new Set());
}
fieldMap.get(key).add(lineNumber);
}
}
}
}
// OPTIMIZATION 6: Apply all updates in batch for better performance
for (const [field, fieldMap] of fieldUpdates) {
for (const [key, lineNumbers] of fieldMap) {
if (!data[field][key]) {
data[field][key] = {
set: new Set(),
ranges: []
};
}
// OPTIMIZATION 6: Add all line numbers at once
for (const lineNumber of lineNumbers) {
data[field][key].set.add(lineNumber);
}
// OPTIMIZATION 6: Lazy range optimization - only when beneficial
if (data[field][key].set.size >= this.rangeThreshold * 3) {
this._optimizeToRanges(data[field][key]);
}
}
}
}
// Ultra-fast dry remove
dryRemove(ln) {
const data = this.index.data;
for (const field in data) {
for (const value in data[field]) {
// Direct Set operation - fastest possible
if (data[field][value].set) {
data[field][value].set.delete(ln);
}
if (data[field][value].ranges) {
data[field][value].ranges = this._removeFromRanges(data[field][value].ranges, ln);
}
// Remove empty entries
if ((!data[field][value].set || data[field][value].set.size === 0) && (!data[field][value].ranges || data[field][value].ranges.length === 0)) {
delete data[field][value];
}
}
}
}
// Cleanup method to free memory
cleanup() {
const data = this.index.data;
for (const field in data) {
for (const value in data[field]) {
if (data[field][value].set) {
if (typeof data[field][value].set.clearAll === 'function') {
data[field][value].set.clearAll();
} else if (typeof data[field][value].set.clear === 'function') {
data[field][value].set.clear();
}
}
if (data[field][value].ranges) {
data[field][value].ranges.length = 0;
}
}
// Clear the entire field
data[field] = {};
}
// Clear all data
this.index.data = {};
this.totalLines = 0;
}
// Clear all indexes
clear() {
this.index.data = {};
this.totalLines = 0;
}
// Update a record in the index
async update(oldRecord, newRecord, lineNumber = null) {
if (!oldRecord || !newRecord) return;
// Remove old record by ID
await this.remove(oldRecord);
// Add new record with provided line number or use hash of the ID
const actualLineNumber = lineNumber !== null ? lineNumber : this._getIdAsNumber(newRecord.id);
await this.add(newRecord, actualLineNumber);
}
// Convert string ID to number for line number
_getIdAsNumber(id) {
if (typeof id === 'number') return id;
if (typeof id === 'string') {
// Simple hash function to convert string to number
let hash = 0;
for (let i = 0; i < id.length; i++) {
const char = id.charCodeAt(i);
hash = (hash << 5) - hash + char;
hash = hash & hash; // Convert to 32-bit integer
}
return Math.abs(hash);
}
return 0;
}
// Remove a record from the index
async remove(record) {
if (!record) return;
// If record is an array of line numbers, use the original method
if (Array.isArray(record)) {
return this._removeLineNumbers(record);
}
// If record is an object, remove by record data
if (typeof record === 'object' && record.id) {
return await this._removeRecord(record);
}
}
// Remove a specific record from the index
async _removeRecord(record) {
if (!record) return;
const data = this.index.data;
const database = this.database;
const persistedCount = Array.isArray(database?.offsets) ? database.offsets.length : 0;
const lineMatchCache = new Map();
const doesLineNumberBelongToRecord = async lineNumber => {
if (lineMatchCache.has(lineNumber)) {
return lineMatchCache.get(lineNumber);
}
let belongs = false;
try {
if (lineNumber >= persistedCount) {
const writeBufferIndex = lineNumber - persistedCount;
const candidate = database?.writeBuffer?.[writeBufferIndex];
belongs = !!candidate && candidate.id === record.id;
} else if (lineNumber >= 0) {
const range = database?.locate?.(lineNumber);
if (range && database.fileHandler && database.serializer) {
const [start, end] = range;
const buffer = await database.fileHandler.readRange(start, end);
if (buffer && buffer.length > 0) {
let line = buffer.toString('utf8');
if (line) {
line = line.trim();
if (line.length > 0) {
const storedRecord = database.serializer.deserialize(line);
belongs = storedRecord && storedRecord.id === record.id;
}
}
}
}
}
} catch (error) {
belongs = false;
}
lineMatchCache.set(lineNumber, belongs);
return belongs;
};
for (const field in data) {
if (record[field] !== undefined && record[field] !== null) {
const values = Array.isArray(record[field]) ? record[field] : [record[field]];
for (const val of values) {
let key;
// Check if this is a term mapping field (array:string fields only)
const isTermMappingField = this.database?.termManager && this.database.termManager.termMappingFields && this.database.termManager.termMappingFields.includes(field);
if (isTermMappingField && typeof val === 'number') {
// For term mapping fields (array:string), the values are already term IDs
key = String(val);
if (this.database.opts.debugMode) {
console.log(`🔍 IndexManager._removeRecord: Using term ID ${val} directly for field "${field}"`);
}
} else if (isTermMappingField && typeof val === 'string') {
// For term mapping fields (array:string), convert string to term ID
const termId = this.database.termManager.getTermIdWithoutIncrement(val);
key = String(termId);
if (this.database.opts.debugMode) {
console.log(`🔍 IndexManager._removeRecord: Using term ID ${termId} for term "${val}"`);
}
} else {
// For non-term-mapping fields (including array:number), use values directly
key = String(val);
if (this.database?.opts?.debugMode) {
console.log(`🔍 IndexManager._removeRecord: Using value "${val}" directly for field "${field}"`);
}
}
// Note: TermManager notification is handled by Database.mjs
// to avoid double decrementation during updates
const indexEntry = data[field][key];
if (indexEntry) {
const lineNumbers = this._getAllLineNumbers(indexEntry);
const filteredLineNumbers = [];
for (const lineNumber of lineNumbers) {
if (!(await doesLineNumberBelongToRecord(lineNumber))) {
filteredLineNumbers.push(lineNumber);
}
}
if (filteredLineNumbers.length === 0) {
delete data[field][key];
} else {
// Rebuild the index value with filtered line numbers
data[field][key].set = new Set(filteredLineNumbers);
data[field][key].ranges = [];
}
}
}
}
}
}
// Ultra-fast remove with batch processing (renamed from remove)
_removeLineNumbers(lineNumbers) {
if (!lineNumbers || lineNumbers.length === 0) return;
lineNumbers.sort((a, b) => a - b); // Sort ascending for efficient processing
const data = this.index.data;
for (const field in data) {
for (const value in data[field]) {
const numbers = this._getAllLineNumbers(data[field][value]);
const newNumbers = [];
for (const ln of numbers) {
let offset = 0;
for (const lineNumber of lineNumbers) {
if (lineNumber < ln) {
offset++;
} else if (lineNumber === ln) {
offset = -1; // Mark for removal
break;
}
}
if (offset >= 0) {
newNumbers.push(ln - offset); // Update the value
}
}
if (newNumbers.length > 0) {
// Rebuild hybrid structure with new numbers
data[field][value] = {
set: new Set(),
ranges: []
};
for (const num of newNumbers) {
data[field][value] = this._addLineNumber(data[field][value], num);
}
} else {
delete data[field][value];
}
}
}
}
// Ultra-fast replace with batch processing
replace(map) {
if (!map || map.size === 0) return;
const data = this.index.data;
for (const field in data) {
for (const value in data[field]) {
const numbers = this._getAllLineNumbers(data[field][value]);
const newNumbers = [];
for (const lineNumber of numbers) {
if (map.has(lineNumber)) {
newNumbers.push(map.get(lineNumber));
} else {
newNumbers.push(lineNumber);
}
}
// Rebuild hybrid structure with new numbers
data[field][value] = {
set: new Set(),
ranges: []
};
for (const num of newNumbers) {
data[field][value] = this._addLineNumber(data[field][value], num);
}
}
}
}
// Ultra-fast query with early exit and smart processing
query(criteria, options = {}) {
if (typeof options === 'boolean') {
options = {
matchAny: options
};
}
const {
matchAny = false,
caseInsensitive = false
} = options;
if (!criteria) {
// Return all line numbers when no criteria provided
return new Set(Array.from({
length: this.totalLines || 0
}, (_, i) => i));
}
// Handle $not operator
if (criteria.$not && typeof criteria.$not === 'object') {
// Get all possible line numbers from database offsets or totalLines
const totalRecords = this.database?.offsets?.length || this.totalLines || 0;
const allLines = new Set(Array.from({
length: totalRecords
}, (_, i) => i));
// Get line numbers matching the $not condition
const notLines = this.query(criteria.$not, options);
// Return complement (all lines except those matching $not condition)
const result = new Set([...allLines].filter(x => !notLines.has(x)));
// If there are other conditions besides $not, we need to intersect with them
const otherCriteria = {
...criteria
};
delete otherCriteria.$not;
if (Object.keys(otherCriteria).length > 0) {
const otherResults = this.query(otherCriteria, options);
return new Set([...result].filter(x => otherResults.has(x)));
}
return result;
}
// Handle $and queries with parallel processing optimization
if (criteria.$and && Array.isArray(criteria.$and)) {
// OPTIMIZATION: Process conditions in parallel for better performance
if (criteria.$and.length > 1) {
// Process all conditions in parallel (synchronous since query is not async)
const conditionResults = criteria.$and.map(andCondition => this.query(andCondition, options));
// Intersect all results for AND logic
let result = conditionResults[0];
for (let i = 1; i < conditionResults.length; i++) {
result = new Set([...result].filter(x => conditionResults[i].has(x)));
}
// IMPORTANT: Check if there are other fields besides $and at the root level
// If so, we need to intersect with them too
const otherCriteria = {
...criteria
};
delete otherCriteria.$and;
if (Object.keys(otherCriteria).length > 0) {
const otherResults = this.query(otherCriteria, options);
result = new Set([...result].filter(x => otherResults.has(x)));
}
return result || new Set();
} else {
// Single condition - check for other criteria at root level
const andResult = this.query(criteria.$and[0], options);
const otherCriteria = {
...criteria
};
delete otherCriteria.$and;
if (Object.keys(otherCriteria).length > 0) {
const otherResults = this.query(otherCriteria, options);
return new Set([...andResult].filter(x => otherResults.has(x)));
}
return andResult;
}
}
const fields = Object.keys(criteria);
if (!fields.length) {
// Return all line numbers when criteria is empty object
return new Set(Array.from({
length: this.totalLines || 0
}, (_, i) => i));
}
let matchingLines = matchAny ? new Set() : null;
const data = this.index.data;
for (const field of fields) {
// Skip logical operators - they are handled separately
if (field.startsWith('$')) continue;
if (typeof data[field] === 'undefined') continue;
const originalCriteriaValue = criteria[field];
const criteriaValue = normalizeCriteriaOperators(originalCriteriaValue, {
target: 'legacy',
preserveOriginal: true
});
let lineNumbersForField = new Set();
const isNumericField = this.opts.indexes[field] === 'number';
// Handle RegExp values directly (MUST check before object check since RegExp is an object)
if (criteriaValue instanceof RegExp) {
// RegExp cannot be efficiently queried using indices - fall back to streaming
// This will be handled by the QueryManager's streaming strategy
continue;
}
if (typeof criteriaValue === 'object' && !Array.isArray(criteriaValue) && criteriaValue !== null) {
const fieldIndex = data[field];
// Handle $in operator for array queries
if (criteriaValue.$in !== undefined && criteriaValue.$in !== null) {
const inValues = Array.isArray(criteriaValue.$in) ? criteriaValue.$in : [criteriaValue.$in];
// PERFORMANCE: Cache term mapping field check once
const isTermMappingField = this.database?.termManager && this.database.termManager.termMappingFields && this.database.termManager.termMappingFields.includes(field);
for (const inValue of inValues) {
// SPACE OPTIMIZATION: Convert search term to term ID for lookup
let searchTermId;
if (isTermMappingField && typeof inValue === 'number') {
// For term mapping fields (array:string), the search value is already a term ID
searchTermId = String(inValue);
} else if (isTermMappingField && typeof inValue === 'string') {
// For term mapping fields (array:string), convert string to term ID
const termId = this.database?.termManager?.getTermIdWithoutIncrement(String(inValue));
if (termId === undefined) {
// Term not found in termManager - skip this search value
// This means the term was never saved to the database
if (this.opts?.debugMode) {
console.log(`⚠️ Term "${inValue}" not found in termManager for field "${field}" - skipping`);
}
continue; // S