nodedb-json/dist/esm/database.js

A lightweight JSON-based database for Node.js with TypeScript support, indexing, and complex query capabilities

import * as fs from 'fs';
import _ from 'lodash';
import { Collection } from './collection/collection.js';
import { mergeIndexDefinitions, validateIndexDefinition } from './indexer/index-codec.js';
import { assertUniqueIndexesForKey, assertUniqueIndexForDefinition, buildIndex, getItemIndexByField, getItemIndexesByField, hasIndexDefinition, hasIndexOnField } from './indexer/index-manager.js';
import { matchesConditions } from './query/matcher.js';
import { QueryEngine } from './query/query-engine.js';
import { applySort } from './query/sorter.js';
import { FileLock } from './storage/file-lock.js';
import { JsonFileStorage } from './storage/json-file-storage.js';
/**
 * A class to manage JSON-based database operations.
 */
export class NodedbJson {
    /**
     * Creates an instance of NodedbJson.
     * @param {string} filePath - The path to the JSON file.
     * @param {DbOptions} [options] - Database options.
     */
    constructor(filePath, options = {}) {
        this._pendingChanges = 0;
        this._indexes = {};
        this._indexDefinitions = {};
        this.closed = false;
        this.filePath = filePath;
        this.options = {
            autoSave: true,
            createIfNotExists: true,
            defaultValue: {},
            enableIndexing: true,
            autoIndex: true,
            indexValueMode: 'strict',
            persistIndexes: true,
            atomicWrites: true,
            backupOnWrite: true,
            fileLock: true,
            ...options
        };
        this.indexMetaPath = this.options.indexMetaPath || `${this.filePath}.meta.json`;
        this.backupPath = this.options.backupPath || `${this.filePath}.bak`;
        this.tempPath = this.options.tempPath || `${this.filePath}.tmp`;
        this.lockPath = this.options.lockPath || `${this.filePath}.lock`;
        this.fileLock = new FileLock({
            enabled: this.options.fileLock !== false,
            lockPath: this.lockPath,
            staleLockMs: this.options.staleLockMs
        });
        this.fileLock.acquire();
        try {
            this.storage = this._createStorage(this.filePath, this.tempPath, this.backupPath, this.options.defaultValue || {}, this.options.createIfNotExists !== false);
            this.metaStorage = this._createStorage(this.indexMetaPath, `${this.indexMetaPath}.tmp`, `${this.indexMetaPath}.bak`, { indexes: {} }, false);
            this.data = this.readJSONFile();
            this._indexDefinitions = this._loadIndexDefinitions();
            // 自动创建索引
            if (this.options.enableIndexing && this.options.autoIndex) {
                this._rebuildAllIndexes();
            }
        }
        catch (error) {
            this.fileLock.release();
            throw error;
        }
    }
    readJSONFile() {
        this._assertOpen();
        return this.storage.read();
    }
    writeJSONFile() {
        this._assertOpen();
        this.storage.write(JSON.stringify(this.data, null, 2));
        this._pendingChanges = 0;
    }
    _createStorage(filePath, tempPath, backupPath, defaultValue, createIfNotExists) {
        return new JsonFileStorage(filePath, {
            atomicWrites: this.options.atomicWrites !== false,
            backupOnWrite: this.options.backupOnWrite !== false,
            backupPath,
            createIfNotExists,
            defaultValue,
            tempPath
        });
    }
    _loadIndexDefinitions() {
        const definitions = {};
        if (this.options.persistIndexes && fs.existsSync(this.indexMetaPath)) {
            const parsedMeta = this.metaStorage.read();
            const persistedIndexes = parsedMeta.indexes || parsedMeta;
            mergeIndexDefinitions(definitions, persistedIndexes);
        }
        if (this.options.indexes) {
            mergeIndexDefinitions(definitions, this.options.indexes);
        }
        return definitions;
    }
    _saveIndexDefinitions() {
        this._assertOpen();
        if (!this.options.persistIndexes) {
            return;
        }
        this.metaStorage.write(JSON.stringify({ indexes: this._indexDefinitions }, null, 2));
    }
    _getReference(key) {
        this._assertOpen();
        return _.get(this.data, key);
    }
    _assertOpen() {
        if (this.closed) {
            throw new Error('Database instance is closed.');
        }
    }
    /**
     * Manually save changes to file.
     * @returns {NodedbJson} - The instance of the database for chaining.
     */
    save() {
        this._assertOpen();
        if (this._pendingChanges > 0) {
            this.writeJSONFile();
        }
        return this;
    }
    /**
     * Writes pending changes to disk and returns this instance.
     */
    flush() {
        this._assertOpen();
        this.writeJSONFile();
        return this;
    }
    /**
     * Reloads data and persisted index definitions from disk.
     */
    reload() {
        this._assertOpen();
        this.data = this.readJSONFile();
        this._indexDefinitions = this._loadIndexDefinitions();
        this._indexes = {};
        this._pendingChanges = 0;
        if (this.options.enableIndexing && this.options.autoIndex) {
            this._rebuildAllIndexes();
        }
        return this;
    }
    /**
     * Flushes pending changes and releases the process-level file lock.
     */
    close(options = {}) {
        if (this.closed) {
            return;
        }
        if (!options.force) {
            this.save();
        }
        this.closed = true;
        this.fileLock.release();
    }
    set(key, value) {
        this._assertOpen();
        // 检查是否需要更新索引
        const oldValue = _.get(this.data, key);
        const needsIndexUpdate = this.options.enableIndexing &&
            (Array.isArray(oldValue) || Array.isArray(value)) &&
            this._hasIndexDefinition(key);
        if (this.options.enableIndexing && Array.isArray(value)) {
            this._assertUniqueIndexesForKey(key, value);
        }
        _.set(this.data, key, value);
        this._pendingChanges++;
        // 如果修改了带索引的数组，重建索引
        if (needsIndexUpdate) {
            this._rebuildIndexesForKey(key);
        }
        if (this.options.autoSave) {
            this.writeJSONFile();
        }
        return this;
    }
    get(key) {
        return _.cloneDeep(this._getReference(key));
    }
    getUnsafeReference(key) {
        return this._getReference(key);
    }
    mutate(key, mutator) {
        const value = this._getReference(key);
        if (value === undefined) {
            throw new Error(`Key "${key}" does not exist.`);
        }
        const nextValue = _.cloneDeep(value);
        mutator(nextValue);
        if (this.options.enableIndexing && Array.isArray(nextValue)) {
            this._assertUniqueIndexesForKey(key, nextValue);
        }
        _.set(this.data, key, nextValue);
        this._pendingChanges++;
        if (this.options.enableIndexing && this._hasIndexDefinition(key)) {
            this._rebuildIndexesForKey(key);
        }
        if (this.options.autoSave) {
            this.writeJSONFile();
        }
        return this;
    }
    /**
     * Checks if a key exists in the JSON data.
     * @param {string} key - The key to check.
     * @returns {boolean} - True if the key exists, otherwise false.
     */
    has(key) {
        this._assertOpen();
        return _.has(this.data, key);
    }
    update(key, predicateOrUpdater, updater) {
        const data = this._getReference(key);
        if (Array.isArray(data)) {
            if (!updater) {
                throw new Error(`Updater object must be provided for array updates.`);
            }
            // 如果使用索引进行更新
            if (typeof predicateOrUpdater === 'object' &&
                this.options.enableIndexing &&
                this._hasIndexDefinition(key)) {
                // 尝试使用索引查找
                const foundIndex = this._findIndexedItemPosition(key, predicateOrUpdater, data);
                if (foundIndex !== -1) {
                    const nextData = _.cloneDeep(data);
                    _.merge(nextData[foundIndex], updater);
                    this._assertUniqueIndexesForKey(key, nextData);
                    _.merge(data[foundIndex], updater);
                    this._pendingChanges++;
                    // 更新索引
                    this._rebuildIndexesForKey(key);
                }
                else {
                    throw new Error(`No item found matching the predicate.`);
                }
            }
            else {
                // 常规查找
                const predicate = typeof predicateOrUpdater === 'function'
                    ? predicateOrUpdater
                    : (item) => matchesConditions(item, predicateOrUpdater);
                const item = _.find(data, predicate);
                if (item) {
                    const nextData = _.cloneDeep(data);
                    const foundIndex = data.indexOf(item);
                    _.merge(nextData[foundIndex], updater);
                    if (this.options.enableIndexing) {
                        this._assertUniqueIndexesForKey(key, nextData);
                    }
                    _.merge(item, updater);
                    this._pendingChanges++;
                    // 如果有索引，需要更新
                    if (this.options.enableIndexing && this._hasIndexDefinition(key)) {
                        this._rebuildIndexesForKey(key);
                    }
                }
                else if (!item) {
                    throw new Error(`No item found matching the predicate.`);
                }
            }
        }
        else if (_.isObject(data)) {
            if (typeof predicateOrUpdater === 'function') {
                _.update(this.data, key, predicateOrUpdater);
            }
            else {
                _.merge(data, predicateOrUpdater);
            }
            this._pendingChanges++;
        }
        else {
            throw new Error(`Key "${key}" does not reference a collection or array.`);
        }
        if (this.options.autoSave) {
            this.writeJSONFile();
        }
        return this;
    }
    delete(key, predicateOrKeys, field = 'id') {
        const data = this._getReference(key);
        const hasIndex = this.options.enableIndexing && this._hasIndexDefinition(key);
        if (Array.isArray(data)) {
            if (typeof predicateOrKeys === 'function') {
                _.remove(data, predicateOrKeys);
            }
            else if (Array.isArray(predicateOrKeys)) {
                // 如果有索引，尝试使用索引删除
                if (hasIndex && this._hasBuiltIndexOnField(key, field)) {
                    const indexes = _.uniq(predicateOrKeys.flatMap(fieldValue => this._getItemIndexesByField(key, field, fieldValue))).sort((a, b) => b - a);
                    for (const index of indexes) {
                        data.splice(index, 1);
                    }
                }
                else {
                    if (this._getIndexValueMode() === 'coerce') {
                        const valuesToDelete = new Set(predicateOrKeys.map(value => String(value)));
                        _.remove(data, (item) => valuesToDelete.has(String(item[field])));
                    }
                    else {
                        _.remove(data, (item) => predicateOrKeys.includes(item[field]));
                    }
                }
            }
            else {
                throw new Error(`Predicate or keys array must be provided for array deletion.`);
            }
        }
        else if (this.has(key)) {
            if (Array.isArray(predicateOrKeys)) {
                predicateOrKeys.forEach(itemKey => {
                    _.unset(data, itemKey);
                });
            }
            else {
                _.unset(this.data, key);
            }
        }
        else {
            throw new Error(`Key "${key}" does not exist.`);
        }
        // 如果有索引，更新索引
        if (hasIndex) {
            this._rebuildIndexesForKey(key);
        }
        this._pendingChanges++;
        if (this.options.autoSave) {
            this.writeJSONFile();
        }
        return this;
    }
    find(key, predicate) {
        const data = this._getReference(key);
        if (Array.isArray(data)) {
            return _.cloneDeep(_.find(data, predicate));
        }
        else if (_.isObject(data)) {
            return _.cloneDeep(_.find(Object.values(data), predicate));
        }
        else {
            throw new Error(`Key "${key}" does not reference a collection or array.`);
        }
    }
    findByField(key, field, value) {
        const data = this._getReference(key);
        if (!Array.isArray(data)) {
            throw new Error(`Key "${key}" does not reference an array.`);
        }
        // 使用索引进行查找
        if (this.options.enableIndexing && this._hasBuiltIndexOnField(key, field)) {
            const index = this._getItemIndexByField(key, field, value);
            return index !== -1 ? _.cloneDeep(data[index]) : undefined;
        }
        // 常规查找
        return _.cloneDeep(_.find(data, item => item[field] === value));
    }
    filter(key, predicate) {
        const data = this._getReference(key);
        if (Array.isArray(data)) {
            return _.cloneDeep(_.filter(data, predicate));
        }
        else if (_.isObject(data)) {
            return _.cloneDeep(_.filter(Object.values(data), predicate));
        }
        else {
            throw new Error(`Key "${key}" does not reference a collection or array.`);
        }
    }
    filterByField(key, field, values) {
        const data = this._getReference(key);
        if (!Array.isArray(data)) {
            throw new Error(`Key "${key}" does not reference an array.`);
        }
        // 使用索引进行过滤
        if (this.options.enableIndexing && this._hasBuiltIndexOnField(key, field)) {
            const result = [];
            for (const value of values) {
                const indexes = this._getItemIndexesByField(key, field, value);
                for (const index of indexes) {
                    result.push(data[index]);
                }
            }
            return _.cloneDeep(result);
        }
        // 常规过滤
        return _.cloneDeep(_.filter(data, item => values.includes(item[field])));
    }
    push(key, value) {
        if (!this.has(key)) {
            if (Array.isArray(value)) {
                this.set(key, value);
            }
            else {
                this.set(key, [value]);
            }
        }
        else {
            const array = this._getReference(key);
            if (Array.isArray(array)) {
                if (this.options.enableIndexing) {
                    const nextArray = Array.isArray(value)
                        ? array.concat(value)
                        : array.concat([value]);
                    this._assertUniqueIndexesForKey(key, nextArray);
                }
                if (Array.isArray(value)) {
                    array.push(...value);
                }
                else {
                    array.push(value);
                }
                this._pendingChanges++;
                // 如果有索引，更新索引
                if (this.options.enableIndexing && this._hasIndexDefinition(key)) {
                    this._rebuildIndexesForKey(key);
                }
                if (this.options.autoSave) {
                    this.writeJSONFile();
                }
            }
            else {
                throw new Error(`Key "${key}" is not an array.`);
            }
        }
        return this;
    }
    /**
     * Executes multiple operations in batch.
     * @param {Array<{method: string, args: any[]}>} operations - Array of operations to execute.
     * @returns {NodedbJson} - The instance of the database for chaining.
     */
    batch(operations) {
        this._assertOpen();
        const originalAutoSave = this.options.autoSave;
        const originalData = _.cloneDeep(this.data);
        const originalIndexes = _.cloneDeep(this._indexes);
        const originalPendingChanges = this._pendingChanges;
        this.options.autoSave = false;
        try {
            operations.forEach(op => {
                const { method, args } = op;
                const batchMethod = this[method];
                if (!NodedbJson.BATCH_ALLOWED_METHODS.has(method) || typeof batchMethod !== 'function') {
                    throw new Error(`Invalid batch method: ${method}`);
                }
                batchMethod.apply(this, args);
            });
            this.writeJSONFile();
        }
        catch (error) {
            this.data = originalData;
            this._indexes = originalIndexes;
            this._pendingChanges = originalPendingChanges;
            throw error;
        }
        finally {
            this.options.autoSave = originalAutoSave;
        }
        return this;
    }
    createIndex(key, indexDefinition) {
        if (!this.options.enableIndexing) {
            throw new Error('Indexing is not enabled. Set enableIndexing option to true.');
        }
        validateIndexDefinition(indexDefinition);
        const data = this._getReference(key);
        if (!Array.isArray(data)) {
            throw new Error(`Cannot create index on non-array data at "${key}"`);
        }
        if (indexDefinition.type === 'unique') {
            this._assertUniqueIndexForDefinition(key, data, indexDefinition);
        }
        const originalDefinitions = _.cloneDeep(this._indexDefinitions);
        const originalIndexes = _.cloneDeep(this._indexes);
        try {
            // 保存索引定义
            if (!this._indexDefinitions[key]) {
                this._indexDefinitions[key] = {};
            }
            this._indexDefinitions[key][indexDefinition.field] = indexDefinition;
            // 创建索引
            this._buildIndex(key, indexDefinition);
            this._saveIndexDefinitions();
        }
        catch (error) {
            this._indexDefinitions = originalDefinitions;
            this._indexes = originalIndexes;
            throw error;
        }
        return this;
    }
    dropIndex(key, field) {
        this._assertOpen();
        if (!this.options.enableIndexing) {
            return this;
        }
        if (this._indexDefinitions[key] && this._indexDefinitions[key][field]) {
            const originalDefinitions = _.cloneDeep(this._indexDefinitions);
            const originalIndexes = _.cloneDeep(this._indexes);
            try {
                delete this._indexDefinitions[key][field];
                // 如果没有索引了，删除整个键
                if (Object.keys(this._indexDefinitions[key]).length === 0) {
                    delete this._indexDefinitions[key];
                }
                // 删除索引数据
                const indexKey = `${key}:${field}`;
                if (this._indexes[indexKey]) {
                    delete this._indexes[indexKey];
                }
                this._saveIndexDefinitions();
            }
            catch (error) {
                this._indexDefinitions = originalDefinitions;
                this._indexes = originalIndexes;
                throw error;
            }
        }
        return this;
    }
    /**
     * 获取所有索引信息
     * @returns {Record<string, Record<string, IndexDefinition>>} - 索引定义
     */
    getIndexes() {
        this._assertOpen();
        return _.cloneDeep(this._indexDefinitions);
    }
    _hasIndexDefinition(key) {
        return hasIndexDefinition(this._indexDefinitions, key);
    }
    _hasIndexOnField(key, field) {
        return hasIndexOnField(this._indexDefinitions, key, field);
    }
    _hasBuiltIndexOnField(key, field) {
        return Object.prototype.hasOwnProperty.call(this._indexes, `${key}:${field}`);
    }
    _rebuildAllIndexes() {
        // 清空所有索引
        this._indexes = {};
        // 重建每个定义的索引
        for (const key in this._indexDefinitions) {
            for (const field in this._indexDefinitions[key]) {
                this._buildIndex(key, this._indexDefinitions[key][field]);
            }
        }
    }
    _rebuildIndexesForKey(key) {
        if (!this._indexDefinitions[key]) {
            return;
        }
        // 重建该路径的所有索引
        for (const field in this._indexDefinitions[key]) {
            this._buildIndex(key, this._indexDefinitions[key][field]);
        }
    }
    _assertUniqueIndexesForKey(key, data) {
        assertUniqueIndexesForKey(this._indexDefinitions, key, data, this._getIndexValueMode());
    }
    _assertUniqueIndexForDefinition(key, data, indexDefinition) {
        assertUniqueIndexForDefinition(key, data, indexDefinition, this._getIndexValueMode());
    }
    _buildIndex(key, indexDef) {
        const data = this._getReference(key);
        if (!Array.isArray(data)) {
            return;
        }
        buildIndex(this._indexes, key, data, indexDef, this._getIndexValueMode());
    }
    _findIndexedItemPosition(key, obj, data) {
        // 检查是否有匹配的索引
        for (const field in obj) {
            if (this._hasBuiltIndexOnField(key, field)) {
                const indexes = this._getItemIndexesByField(key, field, obj[field]);
                const index = indexes.find(itemIndex => matchesConditions(data[itemIndex], obj));
                if (index !== undefined) {
                    return index;
                }
            }
        }
        return -1;
    }
    _getItemIndexByField(key, field, value) {
        return getItemIndexByField(this._indexes, key, field, value, this._getIndexValueMode());
    }
    _getItemIndexesByField(key, field, value) {
        return getItemIndexesByField(this._indexes, key, field, value, this._getIndexValueMode());
    }
    _getIndexValueMode() {
        return this.options.indexValueMode === 'coerce' ? 'coerce' : 'strict';
    }
    query(key, options = {}) {
        const data = this._getReference(key);
        if (!Array.isArray(data)) {
            throw new Error(`Key "${key}" does not reference an array.`);
        }
        const engine = new QueryEngine({
            getItemIndexesByField: (field, value) => this._getItemIndexesByField(key, field, value),
            hasIndexOnField: field => this.options.enableIndexing === true && this._hasBuiltIndexOnField(key, field)
        });
        return engine.execute(data, options);
    }
    _applySort(data, sort) {
        return applySort(data, sort);
    }
    orderBy(key, sort, limit) {
        const data = this._getReference(key);
        if (!Array.isArray(data)) {
            throw new Error(`Key "${key}" does not reference an array.`);
        }
        const sortOptions = Array.isArray(sort) ? sort : [sort];
        let sortedData = _.orderBy(data, sortOptions.map(option => option.field), sortOptions.map(option => option.direction));
        if (limit && limit > 0) {
            sortedData = sortedData.slice(0, limit);
        }
        return _.cloneDeep(sortedData);
    }
    paginate(key, page, pageSize, where) {
        const result = this.query(key, {
            where,
            pagination: { page, pageSize }
        });
        return {
            data: result.data,
            pagination: result.pagination
        };
    }
    aggregate(key, aggregations, where) {
        const result = this.query(key, {
            where,
            aggregation: aggregations
        });
        return result.aggregations || [];
    }
    count(key, where) {
        var _a;
        const result = this.aggregate(key, [{ type: 'count' }], where);
        return ((_a = result[0]) === null || _a === void 0 ? void 0 : _a.value) || 0;
    }
    distinct(key, field) {
        const data = this._getReference(key);
        if (!Array.isArray(data)) {
            throw new Error(`Key "${key}" does not reference an array.`);
        }
        return _.cloneDeep(_.uniqBy(data, field).map(item => _.get(item, field)));
    }
    collection(key) {
        this._assertOpen();
        return new Collection(this, key);
    }
}
NodedbJson.BATCH_ALLOWED_METHODS = new Set([
    'set',
    'update',
    'delete',
    'push',
    'mutate'
]);
export default NodedbJson;
//# sourceMappingURL=database.js.map