mem100x/dist/database.js

⚡ The FASTEST MCP memory server ever built - 66k+ entities/sec with intelligent context detection

"use strict";
/**
 * High-performance SQLite database implementation with better-sqlite3
 * Optimized for speed with prepared statements, indexes, and WAL mode
 */
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.MemoryDatabase = void 0;
const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
const fs_1 = require("fs");
const path_1 = require("path");
const cache_interface_js_1 = require("./utils/cache-interface.js");
const counting_bloom_filter_js_1 = require("./utils/counting-bloom-filter.js");
const compression_js_1 = require("./utils/compression.js");
const database_schema_js_1 = require("./database-schema.js");
const logger_js_1 = require("./utils/logger.js");
const fast_json_js_1 = require("./utils/fast-json.js");
const config_js_1 = require("./config.js");
class MemoryDatabase {
    db;
    dbPath;
    // Performance optimizations
    entityCache;
    searchCache;
    entityBloom;
    compressionEnabled = config_js_1.config.performance.compressionEnabled;
    relationQueryThreshold = config_js_1.config.performance.relationQueryThreshold;
    // Transaction management
    transactionDepth = 0;
    isInTransaction = false;
    // Prepared statements for maximum performance
    statements = {};
    constructor(dbPath) {
        this.dbPath = dbPath;
        // Initialize caches
        this.entityCache = (0, cache_interface_js_1.createStringCache)(config_js_1.config.performance.cacheStrategy, config_js_1.config.performance.entityCacheSize);
        this.searchCache = (0, cache_interface_js_1.createStringCache)(config_js_1.config.performance.cacheStrategy, config_js_1.config.performance.searchCacheSize);
        // Initialize database
        this.initDatabase();
        // Initialize bloom filter
        this.initializeBloomFilter();
    }
    initDatabase() {
        const perf = new logger_js_1.PerformanceTracker('database_initialization', { dbPath: this.dbPath });
        const dir = (0, path_1.dirname)(this.dbPath);
        if (!(0, fs_1.existsSync)(dir)) {
            (0, fs_1.mkdirSync)(dir, { recursive: true });
        }
        this.db = new better_sqlite3_1.default(this.dbPath);
        (0, logger_js_1.logInfo)('Initializing database', { path: this.dbPath });
        // Apply optimizations
        const pragmas = (0, database_schema_js_1.getPragmas)();
        for (const pragma of pragmas) {
            this.db.exec(pragma);
        }
        // Create schema
        this.db.exec((0, database_schema_js_1.getCompleteSchema)());
        // Prepare statements for performance
        this.prepareStatements();
        perf.end({ status: 'success' });
    }
    prepareStatements() {
        this.statements.createEntity = this.db.prepare(`
      INSERT INTO entities (name, entity_type, observations)
      VALUES (?, ?, ?)
      ON CONFLICT(name) DO UPDATE SET
        entity_type = excluded.entity_type,
        observations = excluded.observations,
        updated_at = julianday('now')
    `);
        this.statements.getEntity = this.db.prepare('SELECT * FROM entities WHERE name = ?');
        this.statements.searchEntities = this.db.prepare(`
      SELECT e.* FROM entities e
      JOIN entities_fts ON e.name = entities_fts.name
      WHERE entities_fts MATCH ? ORDER BY rank LIMIT ?
    `);
        this.statements.searchEntitiesLike = this.db.prepare(`
      SELECT * FROM entities 
      WHERE name LIKE ? OR entity_type LIKE ? OR observations LIKE ?
      ORDER BY updated_at DESC LIMIT ?
    `);
        this.statements.createRelation = this.db.prepare(`
      INSERT OR IGNORE INTO relations (from_entity, to_entity, relation_type)
      VALUES (?, ?, ?)
    `);
        this.statements.deleteEntity = this.db.prepare('DELETE FROM entities WHERE name = ?');
        this.statements.deleteRelation = this.db.prepare('DELETE FROM relations WHERE from_entity = ? AND to_entity = ? AND relation_type = ?');
        this.statements.updateObservations = this.db.prepare('UPDATE entities SET observations = ?, updated_at = julianday(\'now\') WHERE name = ?');
        this.statements.getRelationsByEntity = this.db.prepare(`
      SELECT * FROM relations 
      WHERE from_entity = ? OR to_entity = ?
    `);
        this.statements.getEntityStats = this.db.prepare('SELECT entity_type, COUNT(*) as count FROM entities GROUP BY entity_type');
    }
    initializeBloomFilter() {
        const bloomPath = this.dbPath.replace('.db', '.cbloom');
        const loadedFilter = counting_bloom_filter_js_1.CountingBloomFilter.loadFromFileSync(bloomPath);
        if (loadedFilter) {
            this.entityBloom = loadedFilter;
            (0, logger_js_1.logInfo)('Counting Bloom filter loaded from disk', { path: bloomPath });
        }
        else {
            this.entityBloom = new counting_bloom_filter_js_1.CountingBloomFilter(config_js_1.config.bloomFilter.expectedItems, config_js_1.config.bloomFilter.falsePositiveRate);
            this.entityBloom.initSync();
            (0, logger_js_1.logInfo)('New Counting Bloom filter created', {
                expectedItems: config_js_1.config.bloomFilter.expectedItems,
                falsePositiveRate: config_js_1.config.bloomFilter.falsePositiveRate
            });
            this.populateBloomFilter();
            this.saveBloomFilter();
        }
    }
    populateBloomFilter() {
        if (!this.entityBloom)
            return;
        const rows = this.db.prepare('SELECT name FROM entities').all();
        for (const row of rows) {
            this.entityBloom.add(row.name.toLowerCase());
        }
    }
    saveBloomFilter() {
        if (!this.entityBloom)
            return;
        const bloomPath = this.dbPath.replace('.db', '.cbloom');
        try {
            this.entityBloom.saveToFileSync(bloomPath);
            (0, logger_js_1.logDebug)('Counting Bloom filter saved to disk', { path: bloomPath });
        }
        catch (error) {
            (0, logger_js_1.logError)('Failed to save counting bloom filter', error, { path: bloomPath });
        }
    }
    // Transaction helpers
    transaction(fn) {
        return this.db.transaction(fn)();
    }
    createEntities(entities) {
        const perf = new logger_js_1.PerformanceTracker('createEntities', { count: entities.length });
        const created = this.transaction(() => {
            const results = [];
            for (const entity of entities) {
                const observationsData = this.compressionEnabled
                    ? compression_js_1.CompressionUtils.compressObservations(entity.observations)
                    : (0, fast_json_js_1.stringifyObservations)(entity.observations);
                this.statements.createEntity.run(entity.name, entity.entityType, observationsData);
                const result = {
                    type: 'entity',
                    name: entity.name,
                    entityType: entity.entityType,
                    observations: entity.observations
                };
                results.push(result);
                // Update caches
                this.entityCache.set(entity.name.toLowerCase(), result);
                this.entityBloom.add(entity.name.toLowerCase());
            }
            return results;
        });
        // Clear search cache as new entities were added
        this.searchCache.clear();
        perf.end({ created: created.length });
        return created;
    }
    createRelations(relations) {
        const created = [];
        this.transaction(() => {
            for (const relation of relations) {
                const result = this.statements.createRelation.run(relation.from, relation.to, relation.relationType);
                if (result.changes > 0) {
                    created.push({
                        type: 'relation',
                        from: relation.from,
                        to: relation.to,
                        relationType: relation.relationType
                    });
                }
            }
        });
        return created;
    }
    getEntity(name) {
        const lowerName = name.toLowerCase();
        // Check cache first
        const cached = this.entityCache.get(lowerName);
        if (cached)
            return cached;
        // Check bloom filter
        if (!this.entityBloom.contains(lowerName))
            return undefined;
        // Query database
        const row = this.statements.getEntity.get(name);
        if (!row)
            return undefined;
        const entity = {
            type: 'entity',
            name: row.name,
            entityType: row.entity_type,
            observations: this.compressionEnabled
                ? compression_js_1.CompressionUtils.decompressObservations(row.observations)
                : (0, fast_json_js_1.parseObservations)(row.observations)
        };
        // Cache the result
        this.entityCache.set(lowerName, entity);
        return entity;
    }
    searchNodes(options) {
        const { query, limit = 20 } = options;
        const cacheKey = `search:${query}:${limit}`;
        // Check cache
        const cached = this.searchCache.get(cacheKey);
        if (cached)
            return cached;
        // FTS search first
        const ftsQuery = query.split(/\s+/)
            .filter(term => term.length > 0)
            .map(term => `"${term}"*`)
            .join(' OR ');
        let rows = this.statements.searchEntities.all(ftsQuery, limit);
        // Fallback to LIKE search if no FTS results
        if (rows.length === 0) {
            const likePattern = `%${query}%`;
            rows = this.statements.searchEntitiesLike.all(likePattern, likePattern, likePattern, limit);
        }
        // Convert rows to entities
        const entities = rows.map(row => ({
            type: 'entity',
            name: row.name,
            entityType: row.entity_type,
            observations: this.compressionEnabled
                ? compression_js_1.CompressionUtils.decompressObservations(row.observations)
                : (0, fast_json_js_1.parseObservations)(row.observations)
        }));
        // Update entity cache
        entities.forEach(entity => this.entityCache.set(entity.name.toLowerCase(), entity));
        // Get relations for found entities
        const entityNames = entities.map(e => e.name);
        const relations = entityNames.length > 0
            ? this.getRelationsForEntities(entityNames)
            : [];
        const result = { entities, relations };
        this.searchCache.set(cacheKey, result);
        return result;
    }
    readGraph(limit, offset = 0) {
        const totalEntities = this.db.prepare('SELECT COUNT(*) as count FROM entities').get().count;
        const totalRelations = this.db.prepare('SELECT COUNT(*) as count FROM relations').get().count;
        const entityQuery = limit
            ? `SELECT * FROM entities ORDER BY updated_at DESC LIMIT ${limit} OFFSET ${offset}`
            : 'SELECT * FROM entities ORDER BY updated_at DESC';
        const entityRows = this.db.prepare(entityQuery).all();
        const entities = entityRows.map(row => ({
            type: 'entity',
            name: row.name,
            entityType: row.entity_type,
            observations: this.compressionEnabled
                ? compression_js_1.CompressionUtils.decompressObservations(row.observations)
                : (0, fast_json_js_1.parseObservations)(row.observations)
        }));
        const relations = entities.length > 0
            ? this.getRelationsForEntities(entities.map(e => e.name))
            : [];
        return {
            entities,
            relations,
            pagination: limit ? {
                totalEntities,
                totalRelations,
                offset,
                limit,
                hasMore: offset + entities.length < totalEntities
            } : undefined
        };
    }
    getRelationsForEntities(entityNames) {
        if (entityNames.length === 0)
            return [];
        // Use temporary table for large queries
        if (entityNames.length > this.relationQueryThreshold) {
            return this.transaction(() => {
                // Create temp table
                this.db.exec('CREATE TEMP TABLE IF NOT EXISTS temp_entities (name TEXT PRIMARY KEY)');
                const insertStmt = this.db.prepare('INSERT OR IGNORE INTO temp_entities VALUES (?)');
                for (const name of entityNames) {
                    insertStmt.run(name);
                }
                // Query using temp table
                const rows = this.db.prepare(`
          SELECT DISTINCT r.* FROM relations r
          WHERE r.from_entity IN (SELECT name FROM temp_entities)
             OR r.to_entity IN (SELECT name FROM temp_entities)
        `).all();
                // Clean up
                this.db.exec('DROP TABLE temp_entities');
                return rows.map(row => ({
                    type: 'relation',
                    from: row.from_entity,
                    to: row.to_entity,
                    relationType: row.relation_type
                }));
            });
        }
        // Use IN clause for small queries
        const placeholders = entityNames.map(() => '?').join(',');
        const rows = this.db.prepare(`
      SELECT * FROM relations 
      WHERE from_entity IN (${placeholders}) 
         OR to_entity IN (${placeholders})
    `).all(...entityNames, ...entityNames);
        return rows.map(row => ({
            type: 'relation',
            from: row.from_entity,
            to: row.to_entity,
            relationType: row.relation_type
        }));
    }
    addObservations(updates) {
        if (updates.length === 0)
            return;
        this.transaction(() => {
            for (const update of updates) {
                const row = this.statements.getEntity.get(update.entityName);
                if (row) {
                    const existing = this.compressionEnabled
                        ? compression_js_1.CompressionUtils.decompressObservations(row.observations)
                        : (0, fast_json_js_1.parseObservations)(row.observations);
                    const combined = Array.from(new Set([...existing, ...update.contents]));
                    const observationsJson = this.compressionEnabled
                        ? compression_js_1.CompressionUtils.compressObservations(combined)
                        : (0, fast_json_js_1.stringifyObservations)(combined);
                    this.statements.updateObservations.run(observationsJson, update.entityName);
                    this.entityCache.delete(update.entityName.toLowerCase());
                }
            }
        });
        this.searchCache.clear();
    }
    deleteEntities(entityNames) {
        this.transaction(() => {
            for (const name of entityNames) {
                this.statements.deleteEntity.run(name);
                this.entityBloom.remove(name.toLowerCase());
                this.entityCache.delete(name.toLowerCase());
            }
        });
        this.searchCache.clear();
    }
    deleteObservations(deletions) {
        if (deletions.length === 0)
            return;
        this.transaction(() => {
            for (const deletion of deletions) {
                const row = this.statements.getEntity.get(deletion.entityName);
                if (row) {
                    const existing = this.compressionEnabled
                        ? compression_js_1.CompressionUtils.decompressObservations(row.observations)
                        : (0, fast_json_js_1.parseObservations)(row.observations);
                    const toRemove = new Set(deletion.observations);
                    const updated = existing.filter(obs => !toRemove.has(obs));
                    const observationsJson = this.compressionEnabled
                        ? compression_js_1.CompressionUtils.compressObservations(updated)
                        : (0, fast_json_js_1.stringifyObservations)(updated);
                    this.statements.updateObservations.run(observationsJson, deletion.entityName);
                    this.entityCache.delete(deletion.entityName.toLowerCase());
                }
            }
        });
        this.searchCache.clear();
    }
    deleteRelations(relations) {
        this.transaction(() => {
            for (const relation of relations) {
                this.statements.deleteRelation.run(relation.from, relation.to, relation.relationType);
            }
        });
    }
    openNodes(names) {
        if (names.length === 0)
            return { entities: [], relations: [] };
        const entities = [];
        const namesToQuery = [];
        // Check cache and bloom filter
        for (const name of names) {
            const lowerName = name.toLowerCase();
            if (!this.entityBloom.contains(lowerName))
                continue;
            const cached = this.entityCache.get(lowerName);
            if (cached) {
                entities.push(cached);
            }
            else {
                namesToQuery.push(name);
            }
        }
        // Query remaining entities
        if (namesToQuery.length > 0) {
            const placeholders = namesToQuery.map(() => '?').join(',');
            const entityRows = this.db.prepare(`SELECT * FROM entities WHERE name IN (${placeholders})`).all(...namesToQuery);
            for (const row of entityRows) {
                const entity = {
                    type: 'entity',
                    name: row.name,
                    entityType: row.entity_type,
                    observations: this.compressionEnabled
                        ? compression_js_1.CompressionUtils.decompressObservations(row.observations)
                        : (0, fast_json_js_1.parseObservations)(row.observations)
                };
                entities.push(entity);
                this.entityCache.set(row.name.toLowerCase(), entity);
            }
        }
        const foundNames = entities.map(e => e.name);
        const relations = foundNames.length > 0
            ? this.getRelationsForEntities(foundNames)
            : [];
        return { entities, relations };
    }
    getStats() {
        const entityCount = this.db.prepare('SELECT COUNT(*) as count FROM entities').get().count;
        const relationCount = this.db.prepare('SELECT COUNT(*) as count FROM relations').get().count;
        const typeRows = this.statements.getEntityStats.all();
        const entityTypes = typeRows.reduce((acc, row) => {
            acc[row.entity_type] = row.count;
            return acc;
        }, {});
        const dbFileSize = (0, fs_1.statSync)(this.dbPath).size;
        return {
            totalEntities: entityCount,
            totalRelations: relationCount,
            entityTypes,
            databaseSizeKb: Math.round(dbFileSize / 1024),
            cacheStats: {
                entity: this.entityCache.getStats(),
                search: this.searchCache.getStats()
            },
            bloomStats: this.entityBloom.getStats()
        };
    }
    // Utility method for tests that expect async initialization
    async waitForBloomFilter() {
        // Bloom filter is initialized synchronously in constructor,
        // so this just returns immediately
        return Promise.resolve();
    }
    // Transaction operations
    beginTransaction() {
        if (this.isInTransaction) {
            throw new Error('A transaction is already active');
        }
        this.db.prepare('BEGIN').run();
        this.isInTransaction = true;
        this.transactionDepth = 1;
    }
    commitTransaction() {
        if (!this.isInTransaction) {
            throw new Error('No active transaction to commit');
        }
        this.db.prepare('COMMIT').run();
        this.isInTransaction = false;
        this.transactionDepth = 0;
        // Clear caches after successful commit
        this.entityCache.clear();
        this.searchCache.clear();
    }
    rollbackTransaction() {
        if (!this.isInTransaction) {
            throw new Error('No active transaction to rollback');
        }
        this.db.prepare('ROLLBACK').run();
        this.isInTransaction = false;
        this.transactionDepth = 0;
        // Clear caches after rollback
        this.entityCache.clear();
        this.searchCache.clear();
    }
    // Backup operations
    backup(backupPath) {
        // Save bloom filter before backup
        this.saveBloomFilter();
        // Checkpoint WAL to ensure all changes are in main database file
        this.db.pragma('wal_checkpoint(TRUNCATE)');
        (0, fs_1.copyFileSync)(this.dbPath, backupPath);
        const bloomPath = this.dbPath.replace('.db', '.cbloom');
        const backupBloomPath = backupPath.replace('.db', '.cbloom');
        if ((0, fs_1.existsSync)(bloomPath)) {
            (0, fs_1.copyFileSync)(bloomPath, backupBloomPath);
        }
    }
    close() {
        // Rollback any active transaction before closing
        if (this.isInTransaction) {
            try {
                this.db.prepare('ROLLBACK').run();
            }
            catch (error) {
                // Ignore errors during rollback on close
            }
            this.transactionDepth = 0;
            this.isInTransaction = false;
        }
        this.saveBloomFilter();
        this.db.close();
    }
}
exports.MemoryDatabase = MemoryDatabase;
//# sourceMappingURL=database.js.map