@ahmedhegazee/nestjs-telescope/dist/telescope/core/services/memory-manager.service.js

Advanced observability and monitoring solution for NestJS applications with ML-powered analytics, enterprise features, and production-ready scaling

"use strict";
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
    return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var MemoryManagerService_1;
Object.defineProperty(exports, "__esModule", { value: true });
exports.MemoryManagerService = void 0;
const common_1 = require("@nestjs/common");
const rxjs_1 = require("rxjs");
const operators_1 = require("rxjs/operators");
class ManagedCollection {
    constructor(config, logger) {
        this.config = config;
        this.logger = logger;
        this.items = [];
        this.totalEvicted = 0;
        this.totalCompressed = 0;
        this.lastEviction = null;
        this.compressionRatio = 1;
    }
    add(item) {
        const timestamp = this.config.itemAgeExtractor ? this.config.itemAgeExtractor(item) : new Date();
        const size = this.config.itemSizeEstimator ? this.config.itemSizeEstimator(item) : this.estimateSize(item);
        const managedItem = {
            data: item,
            timestamp,
            accessCount: 0,
            lastAccessed: new Date(),
            size
        };
        this.items.push(managedItem);
        this.enforceRetentionPolicy();
    }
    addBatch(items) {
        const timestamp = new Date();
        const managedItems = items.map(item => ({
            data: item,
            timestamp: this.config.itemAgeExtractor ? this.config.itemAgeExtractor(item) : timestamp,
            accessCount: 0,
            lastAccessed: timestamp,
            size: this.config.itemSizeEstimator ? this.config.itemSizeEstimator(item) : this.estimateSize(item)
        }));
        this.items.push(...managedItems);
        this.enforceRetentionPolicy();
    }
    getAll() {
        return this.items.map(item => {
            item.accessCount++;
            item.lastAccessed = new Date();
            return item.data;
        });
    }
    getRecent(count) {
        const recent = this.items
            .slice(-count)
            .map(item => {
            item.accessCount++;
            item.lastAccessed = new Date();
            return item.data;
        });
        return recent;
    }
    filter(predicate) {
        return this.items
            .filter(managedItem => predicate(managedItem.data))
            .map(item => {
            item.accessCount++;
            item.lastAccessed = new Date();
            return item.data;
        });
    }
    clear() {
        const count = this.items.length;
        this.items = [];
        this.logger.debug(`Cleared collection ${this.config.id}: ${count} items`);
    }
    size() {
        return this.items.length;
    }
    enforceRetentionPolicy() {
        if (!this.config.policy.enabled)
            return;
        const policy = this.config.policy;
        let itemsToEvict = [];
        if (policy.maxAge > 0) {
            const cutoffTime = Date.now() - policy.maxAge;
            const ageEvictions = this.items.filter(item => item.timestamp.getTime() < cutoffTime);
            itemsToEvict.push(...ageEvictions);
        }
        if (policy.maxSize > 0 && this.items.length > policy.maxSize) {
            const excess = this.items.length - policy.maxSize;
            const sizeEvictions = this.selectItemsForEviction(excess);
            itemsToEvict.push(...sizeEvictions);
        }
        if (policy.compressionThreshold > 0) {
            const thresholdSize = Math.floor(policy.maxSize * policy.compressionThreshold);
            if (this.items.length > thresholdSize) {
                this.performCompression();
            }
        }
        if (itemsToEvict.length > 0) {
            this.evictItems(itemsToEvict);
        }
    }
    selectItemsForEviction(count) {
        const strategy = this.config.policy.evictionStrategy;
        const sorted = [...this.items];
        switch (strategy) {
            case 'fifo':
                return sorted.slice(0, count);
            case 'lifo':
                return sorted.slice(-count);
            case 'lru':
                sorted.sort((a, b) => a.lastAccessed.getTime() - b.lastAccessed.getTime());
                return sorted.slice(0, count);
            case 'lfu':
                sorted.sort((a, b) => a.accessCount - b.accessCount);
                return sorted.slice(0, count);
            case 'ttl':
                sorted.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime());
                return sorted.slice(0, count);
            default:
                return sorted.slice(0, count);
        }
    }
    evictItems(itemsToEvict) {
        if (itemsToEvict.length === 0)
            return;
        const evictedData = itemsToEvict.map(item => item.data);
        if (this.config.onEviction) {
            try {
                this.config.onEviction(evictedData);
            }
            catch (error) {
                this.logger.warn(`Eviction callback failed for collection ${this.config.id}: ${error}`);
            }
        }
        this.items = this.items.filter(item => !itemsToEvict.includes(item));
        this.totalEvicted += itemsToEvict.length;
        this.lastEviction = new Date();
        this.logger.debug(`Evicted ${itemsToEvict.length} items from collection ${this.config.id} using ${this.config.policy.evictionStrategy} strategy`);
    }
    performCompression() {
        if (!this.config.onCompression)
            return;
        try {
            const originalCount = this.items.length;
            const compressedData = this.config.onCompression(this.items.map(item => item.data));
            if (compressedData.length < originalCount) {
                const timestamp = new Date();
                this.items = compressedData.map(data => ({
                    data,
                    timestamp,
                    accessCount: 0,
                    lastAccessed: timestamp,
                    size: this.config.itemSizeEstimator ? this.config.itemSizeEstimator(data) : this.estimateSize(data)
                }));
                this.totalCompressed += (originalCount - compressedData.length);
                this.compressionRatio = compressedData.length / originalCount;
                this.logger.debug(`Compressed collection ${this.config.id}: ${originalCount} → ${compressedData.length} items`);
            }
        }
        catch (error) {
            this.logger.warn(`Compression failed for collection ${this.config.id}: ${error}`);
        }
    }
    estimateSize(item) {
        if (typeof item === 'string') {
            return item.length * 2;
        }
        if (typeof item === 'object' && item !== null) {
            try {
                return JSON.stringify(item).length * 2;
            }
            catch {
                return 1000;
            }
        }
        return 100;
    }
    getStats() {
        const now = new Date();
        const sizes = this.items.map(item => item.size);
        const timestamps = this.items.map(item => item.timestamp);
        return {
            id: this.config.id,
            itemCount: this.items.length,
            estimatedSize: sizes.reduce((sum, size) => sum + size, 0),
            oldestItem: timestamps.length > 0 ? new Date(Math.min(...timestamps.map(t => t.getTime()))) : null,
            newestItem: timestamps.length > 0 ? new Date(Math.max(...timestamps.map(t => t.getTime()))) : null,
            lastEviction: this.lastEviction,
            evictedCount: this.totalEvicted,
            compressionRatio: this.compressionRatio,
            policy: this.config.policy
        };
    }
}
let MemoryManagerService = MemoryManagerService_1 = class MemoryManagerService {
    constructor() {
        this.logger = new common_1.Logger(MemoryManagerService_1.name);
        this.collections = new Map();
        this.destroy$ = new rxjs_1.Subject();
        this.statsSubject = new rxjs_1.BehaviorSubject(this.getInitialStats());
        this.cleanupCount = 0;
        this.lastCleanup = null;
        this.startMemoryMonitoring();
    }
    async onModuleInit() {
        this.logger.log('Memory Manager Service initialized');
    }
    onModuleDestroy() {
        this.destroy$.next();
        this.destroy$.complete();
    }
    createCollection(config) {
        const collection = new ManagedCollection(config, this.logger);
        this.collections.set(config.id, collection);
        this.logger.log(`Created managed collection: ${config.id} (maxSize: ${config.policy.maxSize}, maxAge: ${config.policy.maxAge}ms)`);
        return config.id;
    }
    addToCollection(collectionId, item) {
        const collection = this.collections.get(collectionId);
        if (!collection) {
            throw new Error(`Collection not found: ${collectionId}`);
        }
        collection.add(item);
    }
    addBatchToCollection(collectionId, items) {
        const collection = this.collections.get(collectionId);
        if (!collection) {
            throw new Error(`Collection not found: ${collectionId}`);
        }
        collection.addBatch(items);
    }
    getFromCollection(collectionId) {
        const collection = this.collections.get(collectionId);
        if (!collection) {
            throw new Error(`Collection not found: ${collectionId}`);
        }
        return collection.getAll();
    }
    getRecentFromCollection(collectionId, count) {
        const collection = this.collections.get(collectionId);
        if (!collection) {
            throw new Error(`Collection not found: ${collectionId}`);
        }
        return collection.getRecent(count);
    }
    filterCollection(collectionId, predicate) {
        const collection = this.collections.get(collectionId);
        if (!collection) {
            throw new Error(`Collection not found: ${collectionId}`);
        }
        return collection.filter(predicate);
    }
    clearCollection(collectionId) {
        const collection = this.collections.get(collectionId);
        if (!collection) {
            throw new Error(`Collection not found: ${collectionId}`);
        }
        collection.clear();
    }
    removeCollection(collectionId) {
        return this.collections.delete(collectionId);
    }
    getCollectionStats(collectionId) {
        const collection = this.collections.get(collectionId);
        return collection ? collection.getStats() : null;
    }
    getAllCollectionStats() {
        return Array.from(this.collections.values()).map(collection => collection.getStats());
    }
    getMemoryUsage() {
        const stats = this.getAllCollectionStats();
        const totalItems = stats.reduce((sum, stat) => sum + stat.itemCount, 0);
        const estimatedSize = stats.reduce((sum, stat) => sum + stat.estimatedSize, 0);
        const collectionsOverThreshold = stats.filter(stat => stat.policy.compressionThreshold > 0 &&
            stat.itemCount > Math.floor(stat.policy.maxSize * stat.policy.compressionThreshold)).length;
        return {
            totalCollections: this.collections.size,
            totalItems,
            estimatedSize,
            collectionsOverThreshold,
            lastCleanup: this.lastCleanup,
            cleanupCount: this.cleanupCount,
            evictedItems: stats.reduce((sum, stat) => sum + stat.evictedCount, 0),
            compressedItems: stats.reduce((sum, stat) => sum + Math.floor(stat.itemCount * (1 - stat.compressionRatio)), 0)
        };
    }
    getMemoryUsageStream() {
        return this.statsSubject.asObservable();
    }
    forceCleanup() {
        this.logger.log('Forcing memory cleanup across all collections');
        this.collections.forEach((collection, id) => {
            try {
                collection.enforceRetentionPolicy();
            }
            catch (error) {
                this.logger.error(`Failed to cleanup collection ${id}: ${error}`);
            }
        });
        this.cleanupCount++;
        this.lastCleanup = new Date();
        this.updateStats();
    }
    static createTimelinePolicy(maxItems = 1000, maxHours = 24) {
        return {
            maxSize: maxItems,
            maxAge: maxHours * 60 * 60 * 1000,
            compressionThreshold: 0.8,
            evictionStrategy: 'fifo',
            checkInterval: 300000,
            enabled: true
        };
    }
    static createMetricsPolicy(maxItems = 5000, maxDays = 7) {
        return {
            maxSize: maxItems,
            maxAge: maxDays * 24 * 60 * 60 * 1000,
            compressionThreshold: 0.7,
            evictionStrategy: 'lru',
            checkInterval: 600000,
            enabled: true
        };
    }
    static createAlertPolicy(maxItems = 10000, maxDays = 30) {
        return {
            maxSize: maxItems,
            maxAge: maxDays * 24 * 60 * 60 * 1000,
            compressionThreshold: 0.9,
            evictionStrategy: 'ttl',
            checkInterval: 900000,
            enabled: true
        };
    }
    startMemoryMonitoring() {
        (0, rxjs_1.interval)(30000)
            .pipe((0, operators_1.takeUntil)(this.destroy$))
            .subscribe(() => {
            this.updateStats();
            this.checkMemoryPressure();
        });
        (0, rxjs_1.interval)(300000)
            .pipe((0, operators_1.takeUntil)(this.destroy$))
            .subscribe(() => {
            this.performPeriodicCleanup();
        });
    }
    updateStats() {
        const stats = this.getMemoryUsage();
        this.statsSubject.next(stats);
    }
    checkMemoryPressure() {
        const usage = process.memoryUsage();
        const heapUsedMB = usage.heapUsed / 1024 / 1024;
        const heapTotalMB = usage.heapTotal / 1024 / 1024;
        const heapUsagePercent = (heapUsedMB / heapTotalMB) * 100;
        if (heapUsagePercent > 80) {
            this.logger.warn(`High memory usage detected: ${heapUsagePercent.toFixed(1)}% (${heapUsedMB.toFixed(1)}MB/${heapTotalMB.toFixed(1)}MB)`);
            this.forceCleanup();
        }
    }
    performPeriodicCleanup() {
        this.collections.forEach((collection, id) => {
            try {
                collection.enforceRetentionPolicy();
            }
            catch (error) {
                this.logger.error(`Periodic cleanup failed for collection ${id}: ${error}`);
            }
        });
        this.cleanupCount++;
        this.lastCleanup = new Date();
    }
    getInitialStats() {
        return {
            totalCollections: 0,
            totalItems: 0,
            estimatedSize: 0,
            collectionsOverThreshold: 0,
            lastCleanup: null,
            cleanupCount: 0,
            evictedItems: 0,
            compressedItems: 0
        };
    }
};
exports.MemoryManagerService = MemoryManagerService;
exports.MemoryManagerService = MemoryManagerService = MemoryManagerService_1 = __decorate([
    (0, common_1.Injectable)(),
    __metadata("design:paramtypes", [])
], MemoryManagerService);
//# sourceMappingURL=memory-manager.service.js.map