evelodb
Version:
A high-performance native B-tree database for Node.js. Made by Evelocore.
1,278 lines (1,277 loc) • 68.4 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.eveloDB = exports.QueryResult = void 0;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const bson_1 = require("bson");
const imageProcess_js_1 = __importDefault(require("./imageProcess.js"));
const backup_js_1 = require("./backup.js");
// ─── Windows Safe Rename ───────────────────────────────────────────────────────
function safeRename(oldPath, newPath) {
if (oldPath === newPath)
return;
try {
if (fs.existsSync(newPath)) {
try {
fs.unlinkSync(newPath);
}
catch (e) { /* ignore */ }
}
fs.renameSync(oldPath, newPath);
}
catch (err) {
if (process.platform === 'win32') {
try {
fs.copyFileSync(oldPath, newPath);
fs.unlinkSync(oldPath);
}
catch (e) {
throw err;
}
}
else {
throw err;
}
}
}
// ─── Object Store Handler ───────────────────────────────────────────────────
class ObjectStore {
dbDir;
name;
baseDir;
constructor(dbDir, name) {
this.dbDir = dbDir;
this.name = name;
this.baseDir = `${this.dbDir}/objects`;
if (!fs.existsSync(this.baseDir))
fs.mkdirSync(this.baseDir, { recursive: true });
}
getPath(name) {
return `${this.baseDir}/${name}.objdb`;
}
read() {
if (!this.name)
return null;
const p = this.getPath(this.name);
if (!fs.existsSync(p))
return null;
try {
return bson_1.BSON.deserialize(fs.readFileSync(p));
}
catch {
return null;
}
}
write(data) {
if (!this.name)
return { success: false, err: 'Object name required' };
try {
fs.writeFileSync(this.getPath(this.name), bson_1.BSON.serialize(data));
return { success: true };
}
catch (e) {
return { success: false, err: e.message };
}
}
update(data) {
if (!this.name)
return { success: false, err: 'Object name required' };
const current = this.read() || {};
return this.write({ ...current, ...data });
}
delete() {
if (!this.name)
return { success: false, err: 'Object name required' };
const p = this.getPath(this.name);
if (!fs.existsSync(p))
return { success: false, err: 'Not found' };
try {
fs.unlinkSync(p);
return { success: true };
}
catch (e) {
return { success: false, err: e.message };
}
}
rename(newName) {
if (!this.name || !newName)
return { success: false, err: 'Names required' };
const oldP = this.getPath(this.name);
const newP = this.getPath(newName);
if (!fs.existsSync(oldP))
return { success: false, err: 'Not found' };
try {
safeRename(oldP, newP);
this.name = newName;
return { success: true };
}
catch (e) {
return { success: false, err: e.message };
}
}
list() {
if (!fs.existsSync(this.baseDir))
return [];
return fs.readdirSync(this.baseDir)
.filter(f => f.endsWith('.objdb'))
.map(f => f.replace('.objdb', ''));
}
}
// ─── Default Config ────────────────────────────────────────────────────────────
const defaultConfig = {
directory: './evelodbprime',
maxHandles: 64,
compactThreshold: 0.3,
schema: {},
};
// ─── Helpers ───────────────────────────────────────────────────────────────────
function deepCompare(obj1, obj2) {
if (obj1 === obj2)
return true;
if (obj1 === null || obj2 === null || typeof obj1 !== 'object' || typeof obj2 !== 'object')
return obj1 === obj2;
const isArr1 = Array.isArray(obj1);
const isArr2 = Array.isArray(obj2);
if (isArr1 !== isArr2)
return false;
if (isArr1 && Array.isArray(obj2)) {
if (obj1.length !== obj2.length)
return false;
for (let i = 0; i < obj1.length; i++)
if (!deepCompare(obj1[i], obj2[i]))
return false;
return true;
}
const o1 = obj1;
const o2 = obj2;
const keys1 = Object.keys(o1);
const keys2 = Object.keys(o2);
if (keys1.length !== keys2.length)
return false;
for (const key of keys1)
if (!Object.prototype.hasOwnProperty.call(o2, key) || !deepCompare(o1[key], o2[key]))
return false;
return true;
}
function keyCmp(a, b) {
if (a < b)
return -1;
if (a > b)
return 1;
return 0;
}
class PBTreeNode {
entries;
children;
isLeaf;
constructor(isLeaf) {
this.entries = [];
this.children = [];
this.isLeaf = isLeaf;
}
}
class PersistedBTree {
order;
root;
idxPath;
dirty;
constructor(idxPath, order = 128) {
this.order = order;
this.idxPath = idxPath;
this.dirty = false;
this.root = this.load();
}
serializeNode(node, buf) {
buf.push(node.isLeaf ? 1 : 0);
const kc = node.entries.length;
buf.push((kc >> 24) & 0xff, (kc >> 16) & 0xff, (kc >> 8) & 0xff, kc & 0xff);
for (const e of node.entries) {
const keyBuf = Buffer.from(e.key, 'utf8');
const kl = keyBuf.length;
buf.push((kl >> 8) & 0xff, kl & 0xff);
for (let i = 0; i < kl; i++)
buf.push(keyBuf[i]);
const hi = Math.floor(e.offset / 0x100000000);
const lo = e.offset >>> 0;
buf.push((hi >> 24) & 0xff, (hi >> 16) & 0xff, (hi >> 8) & 0xff, hi & 0xff, (lo >> 24) & 0xff, (lo >> 16) & 0xff, (lo >> 8) & 0xff, lo & 0xff);
buf.push((e.len >> 24) & 0xff, (e.len >> 16) & 0xff, (e.len >> 8) & 0xff, e.len & 0xff);
}
const cc = node.children.length;
buf.push((cc >> 24) & 0xff, (cc >> 16) & 0xff, (cc >> 8) & 0xff, cc & 0xff);
for (const child of node.children)
this.serializeNode(child, buf);
}
deserializeNode(buf, pos) {
const isLeaf = buf[pos.offset++] === 1;
const node = new PBTreeNode(isLeaf);
const kc = ((buf[pos.offset] << 24) | (buf[pos.offset + 1] << 16) |
(buf[pos.offset + 2] << 8) | buf[pos.offset + 3]) >>> 0;
pos.offset += 4;
for (let i = 0; i < kc; i++) {
const kl = (buf[pos.offset] << 8) | buf[pos.offset + 1];
pos.offset += 2;
const key = buf.slice(pos.offset, pos.offset + kl).toString('utf8');
pos.offset += kl;
const hi = (buf[pos.offset] * 0x1000000) +
((buf[pos.offset + 1] << 16) | (buf[pos.offset + 2] << 8) | buf[pos.offset + 3]);
const lo = (buf[pos.offset + 4] * 0x1000000) +
((buf[pos.offset + 5] << 16) | (buf[pos.offset + 6] << 8) | buf[pos.offset + 7]);
const offset = hi * 0x100000000 + lo;
pos.offset += 8;
const len = ((buf[pos.offset] << 24) | (buf[pos.offset + 1] << 16) |
(buf[pos.offset + 2] << 8) | buf[pos.offset + 3]) >>> 0;
pos.offset += 4;
node.entries.push({ key, offset, len });
}
const cc = ((buf[pos.offset] << 24) | (buf[pos.offset + 1] << 16) |
(buf[pos.offset + 2] << 8) | buf[pos.offset + 3]) >>> 0;
pos.offset += 4;
for (let i = 0; i < cc; i++)
node.children.push(this.deserializeNode(buf, pos));
return node;
}
load() {
if (!fs.existsSync(this.idxPath))
return new PBTreeNode(true);
try {
const buf = fs.readFileSync(this.idxPath);
if (buf.length < 5)
return new PBTreeNode(true);
const pos = { offset: 0 };
this.order = ((buf[0] << 24) | (buf[1] << 16) | (buf[2] << 8) | buf[3]) >>> 0;
pos.offset = 4;
return this.deserializeNode(buf, pos);
}
catch (err) {
const corrupt = this.idxPath + '.corrupt.' + Date.now();
try {
safeRename(this.idxPath, corrupt);
}
catch { /* ignore */ }
console.error(`[eveloDB] WARNING: corrupt index "${this.idxPath}" renamed to "${corrupt}". ` +
`Index will be rebuilt from scratch. Error: ${err.message}`);
return new PBTreeNode(true);
}
}
flush() {
if (!this.dirty)
return;
const arr = [];
arr.push((this.order >> 24) & 0xff, (this.order >> 16) & 0xff, (this.order >> 8) & 0xff, this.order & 0xff);
this.serializeNode(this.root, arr);
const tmp = this.idxPath + '.tmp';
fs.writeFileSync(tmp, Buffer.from(arr));
safeRename(tmp, this.idxPath);
this.dirty = false;
}
find(key) {
return this.findInNode(this.root, key);
}
findInNode(node, key) {
let lo = 0, hi = node.entries.length - 1;
while (lo <= hi) {
const mid = (lo + hi) >>> 1;
const cmp = keyCmp(node.entries[mid].key, key);
if (cmp === 0)
return node.entries[mid];
if (cmp < 0)
lo = mid + 1;
else
hi = mid - 1;
}
if (node.isLeaf)
return null;
return this.findInNode(node.children[lo], key);
}
findAll(key) {
const results = [];
this.traverseRange(this.root, key, key, results);
return results;
}
traverseRange(node, low, high, result) {
let i = 0;
while (i < node.entries.length && keyCmp(node.entries[i].key, low) < 0)
i++;
if (!node.isLeaf)
this.traverseRange(node.children[i], low, high, result);
while (i < node.entries.length) {
const cmpHigh = keyCmp(node.entries[i].key, high);
if (cmpHigh > 0)
break;
result.push(node.entries[i]);
i++;
if (!node.isLeaf)
this.traverseRange(node.children[i], low, high, result);
}
}
insert(entry) {
this.dirty = true;
if (this.root.entries.length === this.order - 1) {
const newRoot = new PBTreeNode(false);
newRoot.children.push(this.root);
this.splitChild(newRoot, 0);
this.root = newRoot;
}
this.insertNonFull(this.root, entry);
}
update(entry, oldOffset) {
this.dirty = true;
return this.updateInNode(this.root, entry, oldOffset);
}
updateInNode(node, entry, oldOffset) {
let lo = 0, hi = node.entries.length - 1;
while (lo <= hi) {
const mid = (lo + hi) >>> 1;
const cmp = keyCmp(node.entries[mid].key, entry.key);
if (cmp === 0) {
if (oldOffset === undefined || node.entries[mid].offset === oldOffset) {
node.entries[mid] = entry;
return true;
}
// If offset doesn't match, check other entries with same key in this node
// Search left
for (let j = mid - 1; j >= 0 && keyCmp(node.entries[j].key, entry.key) === 0; j--) {
if (node.entries[j].offset === oldOffset) {
node.entries[j] = entry;
return true;
}
}
// Search right
for (let j = mid + 1; j < node.entries.length && keyCmp(node.entries[j].key, entry.key) === 0; j++) {
if (node.entries[j].offset === oldOffset) {
node.entries[j] = entry;
return true;
}
}
// If not found in this node, it might be in children (B-trees can have duplicate keys across nodes)
}
if (cmp < 0)
lo = mid + 1;
else
hi = mid - 1;
}
if (node.isLeaf)
return false;
// Check the child that could contain this key
return this.updateInNode(node.children[lo], entry, oldOffset);
}
delete(key, offset) {
this.dirty = true;
this.deleteFromNode(this.root, key, offset);
if (!this.root.isLeaf && this.root.entries.length === 0 && this.root.children.length > 0)
this.root = this.root.children[0];
}
deleteFromNode(node, key, offset) {
let lo = 0, hi = node.entries.length - 1, idx = -1;
while (lo <= hi) {
const mid = (lo + hi) >>> 1;
const cmp = keyCmp(node.entries[mid].key, key);
if (cmp === 0) {
if (offset === undefined || node.entries[mid].offset === offset) {
idx = mid;
break;
}
// Duplicate key, search adjacent entries in the same node
let found = false;
for (let j = mid - 1; j >= 0 && keyCmp(node.entries[j].key, key) === 0; j--) {
if (node.entries[j].offset === offset) {
idx = j;
found = true;
break;
}
}
if (!found) {
for (let j = mid + 1; j < node.entries.length && keyCmp(node.entries[j].key, key) === 0; j++) {
if (node.entries[j].offset === offset) {
idx = j;
found = true;
break;
}
}
}
if (found)
break;
}
if (cmp < 0)
lo = mid + 1;
else
hi = mid - 1;
}
if (idx !== -1) {
if (node.isLeaf) {
node.entries.splice(idx, 1);
return true;
}
else {
const pred = this.getRightmost(node.children[idx]);
node.entries[idx] = pred;
this.deleteFromNode(node.children[idx], pred.key, pred.offset);
this.fixChild(node, idx);
return true;
}
}
else {
if (node.isLeaf)
return false;
const res = this.deleteFromNode(node.children[lo], key, offset);
if (res)
this.fixChild(node, lo);
return res;
}
}
getRightmost(node) {
if (node.isLeaf)
return node.entries[node.entries.length - 1];
return this.getRightmost(node.children[node.children.length - 1]);
}
fixChild(parent, ci) {
const minKeys = Math.floor((this.order - 1) / 2);
const child = parent.children[ci];
if (child.entries.length >= minKeys)
return;
const leftSib = ci > 0 ? parent.children[ci - 1] : null;
const rightSib = ci < parent.children.length - 1 ? parent.children[ci + 1] : null;
if (leftSib && leftSib.entries.length > minKeys) {
child.entries.unshift(parent.entries[ci - 1]);
parent.entries[ci - 1] = leftSib.entries.pop();
if (!leftSib.isLeaf)
child.children.unshift(leftSib.children.pop());
}
else if (rightSib && rightSib.entries.length > minKeys) {
child.entries.push(parent.entries[ci]);
parent.entries[ci] = rightSib.entries.shift();
if (!rightSib.isLeaf)
child.children.push(rightSib.children.shift());
}
else {
if (leftSib) {
leftSib.entries.push(parent.entries.splice(ci - 1, 1)[0], ...child.entries);
if (!child.isLeaf)
leftSib.children.push(...child.children);
parent.children.splice(ci, 1);
}
else if (rightSib) {
child.entries.push(parent.entries.splice(ci, 1)[0], ...rightSib.entries);
if (!rightSib.isLeaf)
child.children.push(...rightSib.children);
parent.children.splice(ci + 1, 1);
}
}
}
insertNonFull(node, entry) {
let i = node.entries.length - 1;
if (node.isLeaf) {
node.entries.push(null);
while (i >= 0 && keyCmp(entry.key, node.entries[i].key) < 0) {
node.entries[i + 1] = node.entries[i];
i--;
}
node.entries[i + 1] = entry;
}
else {
while (i >= 0 && keyCmp(entry.key, node.entries[i].key) < 0)
i--;
i++;
if (node.children[i].entries.length === this.order - 1) {
this.splitChild(node, i);
if (keyCmp(entry.key, node.entries[i].key) > 0)
i++;
}
this.insertNonFull(node.children[i], entry);
}
}
splitChild(parent, i) {
const mid = Math.floor((this.order - 1) / 2);
const child = parent.children[i];
const sibling = new PBTreeNode(child.isLeaf);
parent.entries.splice(i, 0, child.entries[mid]);
parent.children.splice(i + 1, 0, sibling);
sibling.entries = child.entries.splice(mid + 1);
child.entries.splice(mid);
if (!child.isLeaf)
sibling.children = child.children.splice(mid + 1);
}
allEntries() {
const result = [];
this.traverseNode(this.root, result);
return result;
}
traverseNode(node, result) {
for (let i = 0; i < node.entries.length; i++) {
if (!node.isLeaf)
this.traverseNode(node.children[i], result);
result.push(node.entries[i]);
}
if (!node.isLeaf && node.children.length > node.entries.length)
this.traverseNode(node.children[node.entries.length], result);
}
size() { return this.allEntries().length; }
}
// ─── BSON Page Store ───────────────────────────────────────────────────────────
const FLAG_LIVE = 0x01;
const FLAG_TOMBSTONE = 0x00;
const HEADER_SIZE = 5;
class BSONPageStore {
dataPath;
walPath;
fd = null;
fileSize = 0;
tombstoneCount = 0;
liveCount = 0;
constructor(dataPath) {
this.dataPath = dataPath;
this.walPath = dataPath + '.wal';
this.open();
this.replayWAL();
}
open() {
const exists = fs.existsSync(this.dataPath);
this.fd = fs.openSync(this.dataPath, exists ? 'r+' : 'w+');
this.fileSize = fs.fstatSync(this.fd).size;
}
close() {
if (this.fd !== null) {
fs.closeSync(this.fd);
this.fd = null;
}
}
writeWAL(type, offset, data) {
const header = Buffer.allocUnsafe(13);
header[0] = type;
const hi = Math.floor(offset / 0x100000000);
const lo = offset >>> 0;
header.writeUInt32BE(hi, 1);
header.writeUInt32BE(lo, 5);
header.writeUInt32BE(data.length, 9);
fs.appendFileSync(this.walPath, Buffer.concat([header, data]));
}
replayWAL() {
if (!fs.existsSync(this.walPath))
return;
try {
const buf = fs.readFileSync(this.walPath);
let pos = 0;
while (pos + 13 <= buf.length) {
const type = buf[pos];
const hi = buf.readUInt32BE(pos + 1);
const lo = buf.readUInt32BE(pos + 5);
const offset = hi * 0x100000000 + lo;
const len = buf.readUInt32BE(pos + 9);
pos += 13;
if (pos + len > buf.length)
break;
const data = buf.slice(pos, pos + len);
pos += len;
if (this.fd === null)
continue;
if (type === 0x01) {
fs.writeSync(this.fd, data, 0, data.length, offset);
if (offset + data.length > this.fileSize)
this.fileSize = offset + data.length;
this.liveCount++;
}
else if (type === 0x02) {
const flagBuf = Buffer.from([FLAG_TOMBSTONE]);
fs.writeSync(this.fd, flagBuf, 0, 1, offset + 4);
this.tombstoneCount++;
this.liveCount = Math.max(0, this.liveCount - 1);
}
}
}
catch { /* skip */ }
try {
fs.writeFileSync(this.walPath, Buffer.alloc(0));
}
catch { /* ignore */ }
}
append(doc) {
const bsonDoc = bson_1.BSON.serialize(doc);
const header = Buffer.allocUnsafe(HEADER_SIZE);
header.writeUInt32LE(bsonDoc.length, 0);
header[4] = FLAG_LIVE;
const record = Buffer.concat([header, bsonDoc]);
const offset = this.fileSize;
this.writeWAL(0x01, offset, record);
if (this.fd !== null)
fs.writeSync(this.fd, record, 0, record.length, offset);
this.fileSize += record.length;
this.liveCount++;
return { offset, len: record.length };
}
read(offset, len) {
if (this.fd === null || offset < 0 || offset + len > this.fileSize)
return null;
const buf = Buffer.allocUnsafe(len);
fs.readSync(this.fd, buf, 0, len, offset);
if (buf[4] !== FLAG_LIVE)
return null;
const bodyLen = buf.readUInt32LE(0);
if (bodyLen + HEADER_SIZE > len)
return null;
try {
return bson_1.BSON.deserialize(buf.slice(HEADER_SIZE, HEADER_SIZE + bodyLen));
}
catch {
return null;
}
}
tombstone(offset) {
this.writeWAL(0x02, offset, Buffer.alloc(0));
if (this.fd !== null) {
const flagBuf = Buffer.from([FLAG_TOMBSTONE]);
fs.writeSync(this.fd, flagBuf, 0, 1, offset + 4);
}
this.tombstoneCount++;
this.liveCount = Math.max(0, this.liveCount - 1);
}
compact(liveEntries) {
const tmpPath = this.dataPath + '.compact.tmp';
const tmpFd = fs.openSync(tmpPath, 'w');
const remap = new Map();
let writePos = 0;
for (const entry of liveEntries) {
const doc = this.read(entry.offset, entry.len);
if (!doc)
continue;
const bsonDoc = bson_1.BSON.serialize(doc);
const header = Buffer.allocUnsafe(HEADER_SIZE);
header.writeUInt32LE(bsonDoc.length, 0);
header[4] = FLAG_LIVE;
const record = Buffer.concat([header, bsonDoc]);
fs.writeSync(tmpFd, record, 0, record.length, writePos);
remap.set(entry.offset, { offset: writePos, len: record.length });
writePos += record.length;
}
fs.closeSync(tmpFd);
this.close();
try {
fs.writeFileSync(this.walPath, Buffer.alloc(0));
}
catch { /* ignore */ }
safeRename(tmpPath, this.dataPath);
this.open();
this.tombstoneCount = 0;
this.liveCount = liveEntries.length;
return remap;
}
getFileSize() { return this.fileSize; }
*scan() {
if (this.fd === null)
return;
let pos = 0;
this.liveCount = 0;
this.tombstoneCount = 0;
while (pos + HEADER_SIZE <= this.fileSize) {
const header = Buffer.allocUnsafe(HEADER_SIZE);
fs.readSync(this.fd, header, 0, HEADER_SIZE, pos);
const bodyLen = header.readUInt32LE(0);
const flags = header[4];
const len = HEADER_SIZE + bodyLen;
if (pos + len > this.fileSize)
break;
if (flags === FLAG_LIVE) {
const doc = this.read(pos, len);
if (doc) {
this.liveCount++;
yield { offset: pos, len, doc };
}
}
else {
this.tombstoneCount++;
}
pos += len;
}
}
}
// ─── QueryResult ───────────────────────────────────────────────────────────────
class QueryResult {
data;
err;
constructor(data, err) {
if (err) {
this.err = err;
this.data = [];
}
else {
this.data = Array.isArray(data) ? data : [];
}
}
getList(offset = 0, limit = 10) {
if (this.err)
return [];
return this.data.slice(offset, offset + limit);
}
count() {
if (this.err)
return 0;
return this.data.length;
}
sort(compareFn) {
if (this.err)
return this;
return new QueryResult([...this.data].sort(compareFn));
}
all() {
if (this.err)
return [];
return this.data;
}
}
exports.QueryResult = QueryResult;
// ─── eveloDB ───────────────────────────────────────────────────────────────────
class eveloDB {
config;
handles = new Map();
locks = new Map();
backupManager;
constructor(config = {}) {
this.config = { ...defaultConfig, ...config };
if (!fs.existsSync(this.config.directory))
fs.mkdirSync(this.config.directory, { recursive: true });
this.backupManager = new backup_js_1.BackupManager(this);
}
getBsonPaths(collection) {
const dir = this.config.directory;
if (!fs.existsSync(dir))
fs.mkdirSync(dir, { recursive: true });
return {
dataPath: path.join(dir, `${collection}.db`),
primaryIdxPath: path.join(dir, `${collection}.bidx`),
};
}
getHandle(collection) {
const cached = this.handles.get(collection);
if (cached) {
cached.lastAccess = Date.now();
return cached;
}
if (this.handles.size >= this.config.maxHandles) {
const oldest = [...this.handles.entries()].sort((a, b) => a[1].lastAccess - b[1].lastAccess)[0];
if (oldest) {
oldest[1].store.close();
oldest[1].primaryIndex.flush();
this.handles.delete(oldest[0]);
}
}
const { dataPath, primaryIdxPath } = this.getBsonPaths(collection);
const schema = this.config.schema?.[collection];
const secondaryIndexes = new Map();
if (schema?.indexes) {
for (const field of schema.indexes) {
if (field === '_id')
continue;
const idxPath = path.join(this.config.directory, `${collection}.${field}.bidx`);
secondaryIndexes.set(field, new PersistedBTree(idxPath, 128));
}
}
const handle = {
store: new BSONPageStore(dataPath),
primaryIndex: new PersistedBTree(primaryIdxPath, 128),
secondaryIndexes,
lastAccess: Date.now(),
};
this.handles.set(collection, handle);
return handle;
}
evictLRU() {
let oldest = null, oldestTime = Infinity;
for (const [name, h] of this.handles) {
if (h.lastAccess < oldestTime) {
oldestTime = h.lastAccess;
oldest = name;
}
}
if (oldest) {
const h = this.handles.get(oldest);
h.primaryIndex.flush();
for (const idx of h.secondaryIndexes.values())
idx.flush();
h.store.close();
this.handles.delete(oldest);
}
}
flushHandle(collection) {
const h = this.handles.get(collection);
if (h) {
h.primaryIndex.flush();
for (const idx of h.secondaryIndexes.values())
idx.flush();
}
}
closeHandle(collection) {
const h = this.handles.get(collection);
if (h) {
h.primaryIndex.flush();
for (const idx of h.secondaryIndexes.values())
idx.flush();
h.store.close();
this.handles.delete(collection);
}
}
closeAll() {
for (const [, h] of this.handles) {
h.primaryIndex.flush();
for (const idx of h.secondaryIndexes.values())
idx.flush();
h.store.close();
}
this.handles.clear();
}
/**
* Lists all collection names currently known to the database directory
* by scanning for `.db` files on disk.
*/
listCollections() {
const dir = this.config.directory;
if (!fs.existsSync(dir))
return [];
const files = fs.readdirSync(dir);
const collections = new Set();
for (const file of files) {
if (file.endsWith('.db')) {
collections.add(file.slice(0, -3));
}
}
return [...collections];
}
/**
* Re-initializes the database instance by closing all open handles,
* clearing in-memory state, and re-syncing with the physical files on disk.
* Call this after external modifications to the database files.
*/
reInit() {
// Flush and close all open handles
this.closeAll();
// Clear any pending transaction locks
this.locks.clear();
// Ensure the directory still exists
if (!fs.existsSync(this.config.directory)) {
fs.mkdirSync(this.config.directory, { recursive: true });
}
}
generateUniqueId() {
return new bson_1.ObjectId().toHexString();
}
getObjectIdKey(collection) {
return this.config.schema?.[collection]?.objectIdKey || '_id';
}
mapInput(collection, data) {
const key = this.getObjectIdKey(collection);
if (key === '_id' || !data)
return data;
const mapped = { ...data };
if (key in mapped) {
mapped._id = mapped[key];
delete mapped[key];
}
return mapped;
}
mapOutput(collection, data) {
const key = this.getObjectIdKey(collection);
if (key === '_id' || !data || typeof data !== 'object')
return data;
const mapped = { ...data };
if ('_id' in mapped) {
mapped[key] = mapped._id;
delete mapped._id;
}
return mapped;
}
matchesConditions(item, conditions) {
return Object.entries(conditions).every(([key, value]) => {
const fieldValue = item[key];
if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
const keys = Object.keys(value);
if (keys.length > 0 && keys.every(k => k.startsWith('$'))) {
const cond = value;
return Object.entries(cond).every(([op, condVal]) => {
switch (op) {
case '$eq': return deepCompare(fieldValue, condVal);
case '$ne': return !deepCompare(fieldValue, condVal);
case '$gt': return fieldValue > condVal;
case '$gte': return fieldValue >= condVal;
case '$lt': return fieldValue < condVal;
case '$lte': return fieldValue <= condVal;
case '$in': return Array.isArray(condVal) && condVal.some(v => deepCompare(fieldValue, v));
case '$nin': return Array.isArray(condVal) && !condVal.some(v => deepCompare(fieldValue, v));
case '$regex': {
const flags = typeof cond.$options === 'string' ? cond.$options : 'i';
return new RegExp(condVal, flags).test(String(fieldValue));
}
default: return false;
}
});
}
}
return deepCompare(fieldValue, value);
});
}
isDuplicateInArray(candidates, newDoc, excludeId) {
return candidates.some(existing => {
if (excludeId && String(existing._id) === excludeId)
return false;
const existingKeys = Object.keys(existing).filter(k => k !== '_id' && k !== '_createdAt' && k !== '_modifiedAt');
const newKeys = Object.keys(newDoc).filter(k => k !== '_id' && k !== '_createdAt' && k !== '_modifiedAt');
if (existingKeys.length !== newKeys.length)
return false;
return newKeys.every(k => deepCompare(existing[k], newDoc[k])) &&
existingKeys.every(k => deepCompare(existing[k], newDoc[k]));
});
}
maybeCompact(collection) {
const h = this.handles.get(collection);
if (!h)
return;
const total = h.store.liveCount + h.store.tombstoneCount;
if (total === 0)
return;
if (h.store.tombstoneCount / total >= this.config.compactThreshold)
this.compact(collection);
}
buildFingerprintSet(records) {
const set = new Set();
for (const r of records) {
const copy = { ...r };
delete copy._id;
delete copy._createdAt;
delete copy._modifiedAt;
set.add(JSON.stringify(copy, Object.keys(copy).sort()));
}
return set;
}
fingerprintOf(doc) {
const copy = { ...doc };
delete copy._id;
delete copy._createdAt;
delete copy._modifiedAt;
return JSON.stringify(copy, Object.keys(copy).sort());
}
validateSchema(collection, doc, schemaOverride) {
const collSchema = this.config.schema?.[collection];
const fields = schemaOverride || collSchema?.fields;
if (!fields)
return { valid: true };
for (const [field, config] of Object.entries(fields)) {
const val = doc[field];
if (config.required && (val === undefined || val === null))
return { valid: false, err: `'${field}' is required` };
if (val !== undefined && val !== null) {
if (typeof config.type === 'object' && !Array.isArray(config.type)) {
// Recursive check for objects
if (typeof val !== 'object' || Array.isArray(val))
return { valid: false, err: `'${field}' must be an object` };
const res = this.validateSchema(collection, val, config.type);
if (!res.valid)
return { valid: false, err: `${field}.${res.err.replace(/'/g, '')}` };
}
else {
let typeValid = false;
if (config.type === String)
typeValid = typeof val === 'string';
else if (config.type === Number)
typeValid = typeof val === 'number';
else if (config.type === Array)
typeValid = Array.isArray(val);
else if (config.type === Object)
typeValid = typeof val === 'object' && !Array.isArray(val);
else if (config.type === Boolean)
typeValid = typeof val === 'boolean';
if (!typeValid)
return { valid: false, err: `'${field}' must be of type ${config.type.name}` };
if (config.min !== undefined || config.max !== undefined) {
let compareVal;
if (typeof val === 'string' || Array.isArray(val))
compareVal = val.length;
else if (typeof val === 'number')
compareVal = val;
if (compareVal !== undefined) {
if (config.min !== undefined && compareVal < config.min)
return { valid: false, err: `'${field}' is below minimum (${config.min})` };
if (config.max !== undefined && compareVal > config.max)
return { valid: false, err: `'${field}' exceeds maximum (${config.max})` };
}
}
}
}
}
// Check for unknown fields (only if not a recursive call for nested object)
if (!schemaOverride) {
const schemaKeys = new Set(Object.keys(fields));
const internalKeys = ['_id', '_createdAt', '_modifiedAt', this.getObjectIdKey(collection)];
for (const key of Object.keys(doc)) {
if (!schemaKeys.has(key) && !internalKeys.includes(key)) {
return { valid: false, err: `Field '${key}' is not defined in schema` };
}
}
}
else {
// For nested objects, we also check for unknown fields
const schemaKeys = new Set(Object.keys(fields));
for (const key of Object.keys(doc)) {
if (!schemaKeys.has(key)) {
return { valid: false, err: `Field '${key}' is not defined in schema` };
}
}
}
return { valid: true };
}
create(collection, data) {
if (!collection || !data || typeof data !== 'object')
return { err: 'Invalid request' };
// Strict Collection Check
if (this.config.schema && Object.keys(this.config.schema).length > 0 && !this.config.schema[collection]) {
return { err: `Collection '${collection}' is not defined in schema`, code: 'COLLECTION_NOT_DEFINED' };
}
const idKey = this.getObjectIdKey(collection);
const forbidden = ['_id', '_createdAt', '_modifiedAt', idKey];
for (const key of forbidden)
if (key)
delete data[key];
const mappedData = this.mapInput(collection, data);
const schemaRes = this.validateSchema(collection, mappedData);
if (!schemaRes.valid)
return { err: schemaRes.err, code: 'SCHEMA_VALIDATION_FAILED' };
const h = this.getHandle(collection);
const doc = { ...mappedData };
const now = new Date().toISOString();
if (!doc._id)
doc._id = this.generateUniqueId();
if (!doc._createdAt)
doc._createdAt = now;
doc._modifiedAt = now;
const pk = String(doc._id);
if (h.primaryIndex.find(pk))
return { err: 'Duplicate primary key', code: 'DUPLICATE_KEY' };
// Unique Keys Check
const collSchema = this.config.schema?.[collection];
if (collSchema?.uniqueKeys) {
for (const field of collSchema.uniqueKeys) {
const val = String(doc[field]);
if (h.secondaryIndexes.has(field)) {
if (h.secondaryIndexes.get(field).find(val))
return { err: `Duplicate unique key: ${field}`, code: 'DUPLICATE_UNIQUE_KEY' };
}
else {
// Fallback to scan if not indexed but unique
const exists = this.findOne(collection, { [field]: doc[field] });
if (exists)
return { err: `Duplicate unique key: ${field}`, code: 'DUPLICATE_UNIQUE_KEY' };
}
}
}
const noRepeat = collSchema?.noRepeat !== false;
if (noRepeat) {
const fingerprints = this.buildFingerprintSet(this.allInternal(collection));
if (fingerprints.has(this.fingerprintOf(doc)))
return { err: 'Duplicate data', code: 'DUPLICATE_DATA' };
}
const { offset, len } = h.store.append(doc);
h.primaryIndex.insert({ key: pk, offset, len });
// Update Secondary Indexes
for (const [field, idx] of h.secondaryIndexes) {
if (doc[field] !== undefined)
idx.insert({ key: String(doc[field]), offset, len });
}
this.flushHandle(collection);
const result = { success: true };
result[idKey] = doc._id;
return result;
}
delete(collection, conditions) {
if (!collection || !conditions)
return { err: 'Invalid request' };
// Strict Collection Check
if (this.config.schema && Object.keys(this.config.schema).length > 0 && !this.config.schema[collection]) {
return { err: `Collection '${collection}' is not defined in schema`, code: 'COLLECTION_NOT_DEFINED' };
}
const mappedConditions = this.mapInput(collection, conditions);
const h = this.getHandle(collection);
let deletedCount = 0;
const toDelete = this.find(collection, mappedConditions, true).all();
for (const doc of toDelete) {
const pk = String(doc._id);
const entry = h.primaryIndex.find(pk);
if (entry) {
h.store.tombstone(entry.offset);
h.primaryIndex.delete(pk);
// Delete from Secondary Indexes
for (const [field, idx] of h.secondaryIndexes) {
if (doc[field] !== undefined)
idx.delete(String(doc[field]), entry.offset);
}
deletedCount++;
}
}
this.flushHandle(collection);
this.maybeCompact(collection);
return { success: true, deletedCount };
}
find(collection, conditions, raw = false) {
if (!collection || !conditions)
return new QueryResult(null, 'Invalid request');
// Strict Collection Check
if (this.config.schema && Object.keys(this.config.schema).length > 0 && !this.config.schema[collection]) {
return new QueryResult(null, `Collection '${collection}' is not defined in schema`);
}
const mappedConditions = this.mapInput(collection, conditions);
const h = this.getHandle(collection);
const condEntries = Object.entries(mappedConditions);
// Try to use indexes
if (condEntries.length > 0) {
for (const [field, val] of condEntries) {
if (typeof val === 'object' && val !== null)
continue;
if (field === '_id') {
const entry = h.primaryIndex.find(String(val));
if (entry) {
const doc = h.store.read(entry.offset, entry.len);
if (doc && this.matchesConditions(doc, mappedConditions)) {
return new QueryResult([(raw ? doc : this.mapOutput(collection, doc))]);
}
}
return new QueryResult([]);
}
const sIdx = h.secondaryIndexes.get(field);
if (sIdx) {
const entries = sIdx.findAll(String(val));
if (entries.length > 0) {
const results = [];
for (const entry of entries) {
const doc = h.store.read(entry.offset, entry.len);
if (doc && this.matchesConditions(doc, mappedConditions)) {
results.push((raw ? doc : this.mapOutput(collection, doc)));
}
}
return new QueryResult(results);
}
}
}
}
const results = [];
for (const e of h.primaryIndex.allEntries()) {
const doc = h.store.read(e.offset, e.len);
if (doc && this.matchesConditions(doc, mappedConditions)) {
results.push((raw ? doc : this.mapOutput(collection, doc)));
}
}
return new QueryResult(results);
}
findOne(collection, conditions) {
const res = this.find(collection, conditions).all();
return Array.isArray(res) && res.length > 0 ? res[0] : null;
}
get(collection) {
if (!collection)
return new QueryResult(null, 'collection required!');
// Strict Collection Check
if (this.config.schema && Object.keys(this.config.schema).length > 0 && !this.config.schema[collection]) {
return new QueryResult(null, `Collection '${collection}' is not defined in schema`);
}
const h = this.getHandle(collection), results = [];
for (const e of h.primaryIndex.allEntries()) {
const doc = h.store.read(e.offset, e.len);
if (doc)
results.push(this.mapOutput(collection, doc));
}
return new QueryResult(results);
}
update(collection, conditions, newData) {
return this.edit(collection, conditions, newData);
}
inject(collection, data, options = {}) {
if (!collection || !Array.isArray(data))
return { err: 'Invalid request' };
const method = options.method || 'overwrite';
const idKey = this.getObjectIdKey(collection);
const collSchema = this.config.schema?.[collection];
const isNoRepeat = collSchema?.noRepeat !== false;
// 1. Strict Collection Check
if (this.config.schema && Object.keys(this.config.schema).length > 0 && !this.config.schema[collection]) {
return { err: `Collection '${collection}' is not defined in schema`, code: 'COLLECTION_NOT_DEFINED' };
}
// 2. Validation & Pre-processing
const processedData = [];
for (const item of data) {
if (!item._id && !item[idKey])
return { err: `Record missing ID field (${idKey})`, code: 'MISSING_ID' };
if (!item._createdAt)
return { err: "Record missing _createdAt", code: 'MISSING_CREATED_AT' };
if (!item._modifiedAt)
return { err: "Record missing _modifiedAt", code: 'MISSING_MODIFIED_AT' };
const internal = this.mapInput(collection, item);
const schemaRes = this.validateSchema(collection, internal);
if (!schemaRes.valid)
return { err: `Validation failed for record: ${schemaRes.err}`, code: 'SCHEMA_VALIDATION_FAILED' };
processedData.push(internal);
}
// 3. Handle Duplicate Checks (noRepeat)
if (isNoRepeat) {
const fingerprints = new Set();
for (const doc of processedData) {
const fp = this.fingerprintOf(doc);
if (fingerprints.has(fp))
return { err: 'Duplicate data found in injection payload', code: 'DUPLICATE_DATA' };
fingerprints.add(fp);
}
if (method === 'merge') {
const existingFingerprints = this.buildFingerprintSet(this.allInternal(collection));
for (const doc of processedData) {
if (existingFingerprints.has(this.fingerprintOf(doc))) {
return { err: 'Injection contains records that already exist in the collection', code: 'DUPLICATE_DATA' };
}
}
}
}
// 4. Execute Injection
if (method === 'overwrite') {
this.drop(collection);
}
const h = this.getHandle(collection);
let successCount = 0;
for (const doc of processedData) {
const pk = String(doc._id);
if (method === 'merge' && h.primaryIndex.find(pk)) {
return { err: `Conflict: ID ${pk} already exists`, code: 'ID_CONFLICT' };
}
const { offset, len } = h.store.append(doc);
h.primaryIndex.insert({ key: pk, offset, len });
for (const [field, idx] of h.secondaryIndexes) {
if (doc[field] !== undefined)
idx.insert({ key: String(doc[field]), offset, len });
}