@yihuangdb/storage-object/dist/schema-registry-optimized.js

A Node.js storage object layer library using Redis OM

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.OptimizedSchemaRegistry = void 0;
exports.getOptimizedSchemaRegistry = getOptimizedSchemaRegistry;
const storage_1 = require("./storage");
const connection_1 = require("./connection");
const performance_profiler_1 = require("./performance-profiler");
class OptimizedSchemaRegistry {
    static instance;
    registeredSchemas = new Map();
    storageInstances = new Map();
    client;
    options;
    registryKey = 'schema:registry';
    // Performance optimizations
    cache = new Map();
    cacheSize = 100;
    cacheTTL = 60000; // 60 seconds
    accessFrequency = new Map();
    hotSchemas = new Set();
    pendingOperations = new Map();
    cleanupInterval;
    hotSchemaInterval;
    pendingPersistence;
    constructor(options = {}) {
        this.options = {
            enableCache: true,
            cacheSize: 100,
            cacheTTL: 60000,
            enableProfiling: true,
            lazyLoad: true,
            compressionEnabled: false,
            ...options,
        };
        this.cacheSize = options.cacheSize || 100;
        this.cacheTTL = options.cacheTTL || 60000;
        if (options.persistMetadata !== false) {
            this.initializeRedis();
        }
        if (options.enableProfiling) {
            performance_profiler_1.profiler.enable();
            performance_profiler_1.profiler.setSlowThreshold(50); // 50ms for registry operations
        }
        // Periodic cache cleanup
        this.cleanupInterval = setInterval(() => this.cleanupCache(), 30000);
        // Periodic hot schema detection
        this.hotSchemaInterval = setInterval(() => this.detectHotSchemas(), 60000);
    }
    static getInstance(options) {
        if (!OptimizedSchemaRegistry.instance) {
            OptimizedSchemaRegistry.instance = new OptimizedSchemaRegistry(options);
        }
        return OptimizedSchemaRegistry.instance;
    }
    async initializeRedis() {
        await performance_profiler_1.profiler.measure('registry.initializeRedis', async () => {
            const connection = connection_1.RedisConnection.getInstance({
                url: this.options.redisUrl,
            });
            await connection.connect();
            this.client = await connection.getClient();
            if (!this.options.lazyLoad) {
                await this.loadPersistedSchemas();
            }
        });
    }
    async loadPersistedSchemas() {
        if (!this.client)
            return;
        await performance_profiler_1.profiler.measure('registry.loadPersistedSchemas', async () => {
            try {
                const schemas = await this.client.hGetAll(this.registryKey);
                // Load schemas in parallel for better performance
                const loadPromises = Object.entries(schemas).map(async ([name, metadataJson]) => {
                    try {
                        const metadata = JSON.parse(metadataJson);
                        // Convert date strings back to Date objects
                        metadata.createdAt = new Date(metadata.createdAt);
                        metadata.updatedAt = new Date(metadata.updatedAt);
                        metadata.lastAccessedAt = new Date(metadata.lastAccessedAt);
                        if (metadata.performance?.slowestOperation?.timestamp) {
                            metadata.performance.slowestOperation.timestamp = new Date(metadata.performance.slowestOperation.timestamp);
                        }
                        this.registeredSchemas.set(name, metadata);
                        // Pre-cache frequently accessed schemas
                        if (metadata.statistics.totalOperations > 100) {
                            this.hotSchemas.add(name);
                            this.cacheSchema(name, metadata);
                        }
                    }
                    catch (error) {
                        console.error(`Failed to parse schema metadata for ${name}:`, error);
                    }
                });
                await Promise.all(loadPromises);
            }
            catch (error) {
                console.error('Failed to load persisted schemas:', error);
            }
        });
    }
    async register(name, schema, options) {
        return performance_profiler_1.profiler.measure('registry.register', async () => {
            // Check cache first
            const cached = this.getCached(`register:${name}`);
            if (cached && this.schemasEqual(cached.schema, schema)) {
                return cached;
            }
            // Check for pending registration to avoid duplicate work
            const pendingKey = `register:${name}`;
            if (this.pendingOperations.has(pendingKey)) {
                return this.pendingOperations.get(pendingKey);
            }
            const registrationPromise = this.performRegistration(name, schema, options);
            this.pendingOperations.set(pendingKey, registrationPromise);
            try {
                const result = await registrationPromise;
                return result;
            }
            finally {
                this.pendingOperations.delete(pendingKey);
            }
        }, { name, fieldCount: Object.keys(schema).length });
    }
    async performRegistration(name, schema, options) {
        // Check if schema already exists
        let metadata = this.registeredSchemas.get(name);
        if (metadata) {
            // Check if schema has changed
            if (!this.schemasEqual(metadata.schema, schema)) {
                // Update version
                metadata.version++;
                metadata.schema = schema;
                metadata.updatedAt = new Date();
                metadata.attributes = this.analyzeSchema(schema);
                metadata.options = options;
            }
            metadata.lastAccessedAt = new Date();
        }
        else {
            // Create new metadata
            metadata = {
                name,
                version: 1,
                schema,
                options,
                createdAt: new Date(),
                updatedAt: new Date(),
                lastAccessedAt: new Date(),
                attributes: this.analyzeSchema(schema),
                statistics: {
                    objectCount: 0,
                    createCount: 0,
                    readCount: 0,
                    updateCount: 0,
                    deleteCount: 0,
                    totalOperations: 0,
                    averageObjectSize: 0,
                },
                performance: {
                    averageCreateTime: 0,
                    averageReadTime: 0,
                    averageUpdateTime: 0,
                    averageDeleteTime: 0,
                    slowestOperation: {
                        type: 'none',
                        duration: 0,
                        timestamp: new Date(),
                    },
                },
                status: 'active',
            };
        }
        this.registeredSchemas.set(name, metadata);
        // Cache the metadata
        this.cacheSchema(name, metadata);
        // Persist to Redis asynchronously to avoid blocking
        if (this.options.persistMetadata && this.client) {
            this.persistSchemaAsync(name, metadata);
        }
        return metadata;
    }
    analyzeSchema(schema) {
        return performance_profiler_1.profiler.measureSync('registry.analyzeSchema', () => {
            const fields = {};
            let fieldCount = 0;
            let indexedFieldCount = 0;
            for (const [fieldName, fieldConfig] of Object.entries(schema)) {
                fieldCount++;
                const config = typeof fieldConfig === 'string'
                    ? { type: fieldConfig, indexed: false }
                    : fieldConfig;
                if (config.indexed) {
                    indexedFieldCount++;
                }
                fields[fieldName] = {
                    type: config.type,
                    indexed: config.indexed || false,
                    nullCount: 0,
                    uniqueValues: 0,
                };
            }
            return {
                fieldCount,
                indexedFieldCount,
                fields,
            };
        });
    }
    async persistSchemaAsync(name, metadata) {
        if (!this.client)
            return;
        // Use Promise to track persistence
        const persistPromise = (async () => {
            try {
                const metadataJson = this.options.compressionEnabled
                    ? await this.compress(metadata)
                    : JSON.stringify(metadata);
                await this.client.hSet(this.registryKey, name, metadataJson);
            }
            catch (error) {
                console.error(`Failed to persist schema ${name}:`, error);
            }
        })();
        // Track the promise for cleanup
        if (!this.pendingPersistence) {
            this.pendingPersistence = new Set();
        }
        this.pendingPersistence.add(persistPromise);
        persistPromise.finally(() => {
            this.pendingPersistence?.delete(persistPromise);
        });
        // Don't await to keep it non-blocking
        return;
    }
    async getStorage(name) {
        return performance_profiler_1.profiler.measure('registry.getStorage', async () => {
            // Track access frequency for hot schema detection
            this.trackAccess(name);
            // Check cache first
            let storage = this.storageInstances.get(name);
            if (storage) {
                this.updateLastAccessed(name);
                return storage;
            }
            // Check for pending storage creation
            const pendingKey = `storage:${name}`;
            if (this.pendingOperations.has(pendingKey)) {
                return this.pendingOperations.get(pendingKey);
            }
            // Check if schema is registered
            let metadata = this.getCached(`metadata:${name}`) || this.registeredSchemas.get(name);
            if (!metadata && this.client) {
                // Lazy load from Redis if not in memory
                metadata = await this.lazyLoadSchema(name);
            }
            if (!metadata) {
                return null;
            }
            const creationPromise = this.createStorageInstance(name, metadata);
            this.pendingOperations.set(pendingKey, creationPromise);
            try {
                storage = await creationPromise;
                return storage;
            }
            finally {
                this.pendingOperations.delete(pendingKey);
            }
        }, { name });
    }
    async lazyLoadSchema(name) {
        return performance_profiler_1.profiler.measure('registry.lazyLoadSchema', async () => {
            if (!this.client)
                return null;
            try {
                const metadataJson = await this.client.hGet(this.registryKey, name);
                if (!metadataJson)
                    return null;
                const metadata = this.options.compressionEnabled
                    ? await this.decompress(metadataJson)
                    : JSON.parse(metadataJson);
                // Convert date strings back to Date objects
                metadata.createdAt = new Date(metadata.createdAt);
                metadata.updatedAt = new Date(metadata.updatedAt);
                metadata.lastAccessedAt = new Date(metadata.lastAccessedAt);
                if (metadata.performance?.slowestOperation?.timestamp) {
                    metadata.performance.slowestOperation.timestamp = new Date(metadata.performance.slowestOperation.timestamp);
                }
                this.registeredSchemas.set(name, metadata);
                this.cacheSchema(name, metadata);
                return metadata;
            }
            catch (error) {
                console.error(`Failed to lazy load schema ${name}:`, error);
                return null;
            }
        }, { name });
    }
    async registerStorageInstance(name, storage) {
        await performance_profiler_1.profiler.measure('registry.registerStorageInstance', async () => {
            // Wrap storage methods to track statistics
            this.wrapStorageMethods(storage, name);
            // Store the instance
            this.storageInstances.set(name, storage);
            this.updateLastAccessed(name);
            // Pre-warm cache for hot schemas
            if (this.hotSchemas.has(name)) {
                const metadata = this.registeredSchemas.get(name);
                if (metadata) {
                    this.cacheSchema(name, metadata);
                }
            }
        }, { name });
    }
    async createStorageInstance(name, metadata) {
        return performance_profiler_1.profiler.measure('registry.createStorageInstance', async () => {
            const storage = new storage_1.StorageObject(name, metadata.schema, metadata.options);
            await storage.initialize();
            // Wrap storage methods to track statistics
            this.wrapStorageMethods(storage, name);
            this.storageInstances.set(name, storage);
            this.updateLastAccessed(name);
            return storage;
        }, { name });
    }
    wrapStorageMethods(storage, name) {
        const metadata = this.registeredSchemas.get(name);
        if (!metadata)
            return;
        // Wrap create method
        const originalCreate = storage.create.bind(storage);
        storage.create = async (data) => {
            const startTime = Date.now();
            try {
                const result = await originalCreate(data);
                const duration = Date.now() - startTime;
                this.updateStatistics(name, 'create', duration);
                // Update cache if this is a hot schema
                if (this.hotSchemas.has(name)) {
                    this.invalidateCache(`storage:${name}:*`);
                }
                return result;
            }
            catch (error) {
                throw error;
            }
        };
        // Wrap findById method with caching
        const originalFindById = storage.findById.bind(storage);
        storage.findById = async (id) => {
            const cacheKey = `storage:${name}:findById:${id}`;
            const cached = this.getCached(cacheKey);
            if (cached) {
                this.updateStatistics(name, 'read', 0); // Cache hit
                return cached;
            }
            const startTime = Date.now();
            try {
                const result = await originalFindById(id);
                const duration = Date.now() - startTime;
                this.updateStatistics(name, 'read', duration);
                // Cache the result if this is a hot schema
                if (this.hotSchemas.has(name) && result) {
                    this.setCached(cacheKey, result, this.cacheTTL / 2); // Shorter TTL for individual items
                }
                return result;
            }
            catch (error) {
                throw error;
            }
        };
        // Wrap find method with query caching
        const originalFind = storage.find.bind(storage);
        storage.find = async (query, options) => {
            const cacheKey = `storage:${name}:find:${JSON.stringify({ query, options })}`;
            const cached = this.getCached(cacheKey);
            if (cached) {
                this.updateStatistics(name, 'read', 0); // Cache hit
                return cached;
            }
            const startTime = Date.now();
            try {
                const result = await originalFind(query, options);
                const duration = Date.now() - startTime;
                this.updateStatistics(name, 'read', duration);
                // Cache query results for hot schemas
                if (this.hotSchemas.has(name) && result.length < 100) {
                    this.setCached(cacheKey, result, this.cacheTTL / 4); // Even shorter TTL for queries
                }
                return result;
            }
            catch (error) {
                throw error;
            }
        };
        // Wrap update method
        const originalUpdate = storage.update.bind(storage);
        storage.update = async (id, data, options) => {
            const startTime = Date.now();
            try {
                const result = await originalUpdate(id, data, options);
                const duration = Date.now() - startTime;
                this.updateStatistics(name, 'update', duration);
                // Invalidate caches
                this.invalidateCache(`storage:${name}:findById:${id}`);
                this.invalidateCache(`storage:${name}:find:*`);
                return result;
            }
            catch (error) {
                throw error;
            }
        };
        // Wrap delete method
        const originalDelete = storage.delete.bind(storage);
        storage.delete = async (id) => {
            const startTime = Date.now();
            try {
                const result = await originalDelete(id);
                if (result) {
                    const duration = Date.now() - startTime;
                    this.updateStatistics(name, 'delete', duration);
                    // Invalidate caches
                    this.invalidateCache(`storage:${name}:findById:${id}`);
                    this.invalidateCache(`storage:${name}:find:*`);
                }
                return result;
            }
            catch (error) {
                throw error;
            }
        };
    }
    updateStatistics(name, operation, duration) {
        const metadata = this.registeredSchemas.get(name);
        if (!metadata)
            return;
        // Update operation counts
        metadata.statistics[`${operation}Count`]++;
        metadata.statistics.totalOperations++;
        // Update performance metrics with exponential moving average for smoothing
        const perfKey = `average${operation.charAt(0).toUpperCase() + operation.slice(1)}Time`;
        const alpha = 0.1; // Smoothing factor
        const currentAvg = metadata.performance[perfKey];
        metadata.performance[perfKey] = currentAvg * (1 - alpha) + duration * alpha;
        // Track slowest operation
        if (duration > metadata.performance.slowestOperation.duration) {
            metadata.performance.slowestOperation = {
                type: operation,
                duration,
                timestamp: new Date(),
            };
        }
        // Update last accessed time
        metadata.lastAccessedAt = new Date();
    }
    updateLastAccessed(name) {
        const metadata = this.registeredSchemas.get(name);
        if (metadata) {
            metadata.lastAccessedAt = new Date();
        }
    }
    schemasEqual(schema1, schema2) {
        const keys1 = Object.keys(schema1).sort();
        const keys2 = Object.keys(schema2).sort();
        if (keys1.length !== keys2.length) {
            return false;
        }
        for (let i = 0; i < keys1.length; i++) {
            if (keys1[i] !== keys2[i]) {
                return false;
            }
            const field1 = schema1[keys1[i]];
            const field2 = schema2[keys2[i]];
            if (JSON.stringify(field1) !== JSON.stringify(field2)) {
                return false;
            }
        }
        return true;
    }
    // Cache management methods
    getCached(key) {
        if (!this.options.enableCache)
            return null;
        const entry = this.cache.get(key);
        if (!entry)
            return null;
        // Check if cache is expired
        if (Date.now() - entry.timestamp > this.cacheTTL) {
            this.cache.delete(key);
            return null;
        }
        // Update hit count
        entry.hits++;
        return entry.data;
    }
    setCached(key, data, ttl) {
        if (!this.options.enableCache)
            return;
        // Enforce cache size limit
        if (this.cache.size >= this.cacheSize) {
            this.evictLRU();
        }
        this.cache.set(key, {
            data,
            timestamp: Date.now(),
            hits: 0,
        });
    }
    cacheSchema(name, metadata) {
        this.setCached(`metadata:${name}`, metadata);
        this.setCached(`register:${name}`, metadata);
    }
    invalidateCache(pattern) {
        if (pattern.includes('*')) {
            // Wildcard pattern
            const regex = new RegExp(pattern.replace(/\*/g, '.*'));
            for (const key of this.cache.keys()) {
                if (regex.test(key)) {
                    this.cache.delete(key);
                }
            }
        }
        else {
            // Exact key
            this.cache.delete(pattern);
        }
    }
    evictLRU() {
        // Find least recently used cache entry
        let lruKey = null;
        let minHits = Infinity;
        let oldestTime = Date.now();
        for (const [key, entry] of this.cache.entries()) {
            const score = entry.hits * 1000 + (Date.now() - entry.timestamp);
            if (score < minHits) {
                minHits = score;
                lruKey = key;
            }
        }
        if (lruKey) {
            this.cache.delete(lruKey);
        }
    }
    cleanupCache() {
        const now = Date.now();
        for (const [key, entry] of this.cache.entries()) {
            if (now - entry.timestamp > this.cacheTTL) {
                this.cache.delete(key);
            }
        }
    }
    trackAccess(name) {
        const count = (this.accessFrequency.get(name) || 0) + 1;
        this.accessFrequency.set(name, count);
    }
    detectHotSchemas() {
        // Calculate access frequency threshold
        const frequencies = Array.from(this.accessFrequency.values());
        if (frequencies.length === 0)
            return;
        frequencies.sort((a, b) => b - a);
        const threshold = frequencies[Math.min(5, Math.floor(frequencies.length * 0.2))]; // Top 20% or top 5
        // Update hot schemas set
        this.hotSchemas.clear();
        for (const [name, frequency] of this.accessFrequency.entries()) {
            if (frequency >= threshold) {
                this.hotSchemas.add(name);
            }
        }
        // Reset frequencies for next period
        this.accessFrequency.clear();
    }
    async compress(data) {
        // Implement compression if needed
        // For now, just stringify
        return JSON.stringify(data);
    }
    async decompress(data) {
        // Implement decompression if needed
        // For now, just parse
        return JSON.parse(data);
    }
    // Public methods for monitoring and management
    async getSchemaMetadata(name) {
        return performance_profiler_1.profiler.measure('registry.getSchemaMetadata', async () => {
            const cached = this.getCached(`metadata:${name}`);
            if (cached)
                return cached;
            const metadata = this.registeredSchemas.get(name);
            if (metadata) {
                this.cacheSchema(name, metadata);
                return metadata;
            }
            // Try lazy loading if enabled
            if (this.options.lazyLoad && this.client) {
                return this.lazyLoadSchema(name);
            }
            return null;
        }, { name });
    }
    async getAllSchemas() {
        return performance_profiler_1.profiler.measure('registry.getAllSchemas', async () => {
            // Load all schemas if using lazy loading
            if (this.options.lazyLoad && this.client) {
                await this.loadPersistedSchemas();
            }
            return Array.from(this.registeredSchemas.values());
        });
    }
    getPerformanceReport() {
        let hits = 0;
        let totalAccess = 0;
        for (const entry of this.cache.values()) {
            hits += entry.hits;
            totalAccess += entry.hits + 1;
        }
        return {
            cacheStats: {
                size: this.cache.size,
                hits,
                misses: totalAccess - hits,
                hitRate: totalAccess > 0 ? hits / totalAccess : 0,
            },
            hotSchemas: Array.from(this.hotSchemas),
            registryMetrics: performance_profiler_1.profiler.getMetrics().filter(m => m.operation.startsWith('registry.')),
            bottlenecks: performance_profiler_1.profiler.getBottlenecks(),
        };
    }
    async updateObjectCount(name) {
        await performance_profiler_1.profiler.measure('registry.updateObjectCount', async () => {
            const storage = await this.getStorage(name);
            if (!storage)
                return;
            const metadata = this.registeredSchemas.get(name);
            if (!metadata)
                return;
            try {
                const count = await storage.count();
                metadata.statistics.objectCount = count;
                // Update storage size estimate
                if (count > 0) {
                    // Estimate average object size (this is a rough estimate)
                    const sampleSize = Math.min(10, count);
                    const samples = await storage.find({}, { limit: sampleSize });
                    let totalSize = 0;
                    for (const sample of samples) {
                        totalSize += JSON.stringify(sample).length;
                    }
                    metadata.statistics.averageObjectSize = totalSize / samples.length;
                }
                // Persist updated metadata
                if (this.options.persistMetadata && this.client) {
                    await this.persistSchemaAsync(name, metadata);
                }
            }
            catch (error) {
                console.error(`Failed to update object count for ${name}:`, error);
            }
        }, { name });
    }
    async clearAllPersistedSchemas() {
        if (!this.client)
            return;
        await performance_profiler_1.profiler.measure('registry.clearAllPersistedSchemas', async () => {
            try {
                // Delete the entire registry hash
                await this.client.del(this.registryKey);
                // Clear cache
                this.cache.clear();
            }
            catch (error) {
                console.error('Failed to clear persisted schemas:', error);
            }
        });
    }
    async shutdown() {
        await performance_profiler_1.profiler.measure('registry.shutdown', async () => {
            // Clear intervals
            if (this.cleanupInterval) {
                clearInterval(this.cleanupInterval);
                this.cleanupInterval = undefined;
            }
            if (this.hotSchemaInterval) {
                clearInterval(this.hotSchemaInterval);
                this.hotSchemaInterval = undefined;
            }
            // Wait for pending persistence operations
            if (this.pendingPersistence && this.pendingPersistence.size > 0) {
                await Promise.all(Array.from(this.pendingPersistence));
                this.pendingPersistence.clear();
            }
            // Clear all storage instances
            for (const storage of this.storageInstances.values()) {
                try {
                    await storage.disconnect();
                }
                catch (error) {
                    console.error('Error disconnecting storage:', error);
                }
            }
            // Clear instances
            this.storageInstances.clear();
            this.cache.clear();
            this.pendingOperations.clear();
            // Close Redis connection
            if (this.client) {
                try {
                    await this.client.quit();
                }
                catch (error) {
                    console.error('Error closing Redis client:', error);
                }
                this.client = null;
            }
            // Print performance report if profiling is enabled
            if (this.options.enableProfiling) {
                console.log('\n📊 Schema Registry Performance Report:');
                const report = this.getPerformanceReport();
                console.log('Cache Hit Rate:', (report.cacheStats.hitRate * 100).toFixed(2) + '%');
                console.log('Hot Schemas:', report.hotSchemas.join(', '));
                performance_profiler_1.profiler.printReport();
            }
        });
    }
    static async reset() {
        if (OptimizedSchemaRegistry.instance) {
            try {
                await OptimizedSchemaRegistry.instance.shutdown();
            }
            catch (error) {
                console.error('Error during OptimizedSchemaRegistry shutdown:', error);
            }
            OptimizedSchemaRegistry.instance = null;
        }
    }
}
exports.OptimizedSchemaRegistry = OptimizedSchemaRegistry;
// Export singleton getter
function getOptimizedSchemaRegistry(options) {
    return OptimizedSchemaRegistry.getInstance(options);
}
//# sourceMappingURL=schema-registry-optimized.js.map