@ahmedhegazee/nestjs-telescope
Version:
Advanced observability and monitoring solution for NestJS applications with ML-powered analytics, enterprise features, and production-ready scaling
365 lines • 13.5 kB
JavaScript
"use strict";
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var __param = (this && this.__param) || function (paramIndex, decorator) {
return function (target, key) { decorator(target, key, paramIndex); }
};
var EnhancedEntryManagerService_1;
Object.defineProperty(exports, "__esModule", { value: true });
exports.EnhancedEntryManagerService = void 0;
const common_1 = require("@nestjs/common");
const storage_manager_service_1 = require("../../storage/storage-manager.service");
const uuid_1 = require("uuid");
let EnhancedEntryManagerService = EnhancedEntryManagerService_1 = class EnhancedEntryManagerService {
constructor(storageManager, config) {
this.storageManager = storageManager;
this.config = config;
this.logger = new common_1.Logger(EnhancedEntryManagerService_1.name);
this.batchQueues = new Map();
this.retryQueue = [];
this.metrics = new ProcessingMetrics();
this.sequenceCounter = 0;
this.processingIntervals = new Map();
}
async onModuleInit() {
this.startBatchProcessors();
this.startRetryProcessor();
this.startMetricsReporting();
}
async process(entry) {
this.ensureEntryFields(entry);
try {
const queueKey = this.getQueueKey(entry);
if (this.shouldBatch(entry)) {
await this.addToQueue(queueKey, entry);
}
else {
await this.processImmediate(entry);
}
this.metrics.recordEntry(entry);
}
catch (error) {
this.logger.error(`Failed to process entry ${entry.id}:`, error);
this.metrics.recordError();
}
}
async processBatch(entries) {
entries.forEach(entry => this.ensureEntryFields(entry));
try {
const queueGroups = this.groupEntriesByQueue(entries);
for (const [queueKey, queueEntries] of queueGroups) {
if (this.shouldBatch(queueEntries[0])) {
await this.addToQueue(queueKey, ...queueEntries);
}
else {
await Promise.all(queueEntries.map(entry => this.processImmediate(entry)));
}
}
entries.forEach(entry => this.metrics.recordEntry(entry));
}
catch (error) {
this.logger.error('Failed to process batch:', error);
this.metrics.recordError();
}
}
getQueueKey(entry) {
if (entry.tags.includes('critical'))
return 'critical';
if (entry.tags.includes('error') || entry.tags.includes('exception'))
return 'error';
if (entry.tags.includes('performance'))
return 'performance';
if (entry.type.startsWith('devtools'))
return 'devtools';
switch (entry.type) {
case 'request':
return 'request';
case 'query':
return 'query';
case 'job':
return 'job';
case 'cache':
return 'cache';
default:
return 'default';
}
}
shouldBatch(entry) {
if (entry.tags.includes('critical'))
return false;
if (entry.tags.includes('error') || entry.tags.includes('exception'))
return false;
const entrySize = JSON.stringify(entry).length;
if (entrySize > 100000) {
this.logger.warn(`Large entry ${entry.id} (${entrySize} bytes) bypassing batch`);
return false;
}
if (entry.tags.includes('realtime'))
return false;
return this.config.storage.batch.enabled;
}
async addToQueue(queueKey, ...entries) {
if (!this.batchQueues.has(queueKey)) {
this.batchQueues.set(queueKey, []);
}
const queue = this.batchQueues.get(queueKey);
queue.push(...entries);
const queueConfig = this.getQueueConfig(queueKey);
if (queue.length >= queueConfig.batchSize) {
await this.flushQueue(queueKey);
}
}
async flushQueue(queueKey) {
const queue = this.batchQueues.get(queueKey);
if (!queue || queue.length === 0)
return;
const entries = queue.splice(0);
try {
await this.storageManager.storeBatch(entries);
this.metrics.recordBatch(entries.length, true);
this.logger.debug(`Flushed queue '${queueKey}' with ${entries.length} entries`);
}
catch (error) {
this.logger.error(`Failed to flush queue '${queueKey}':`, error);
if (queueKey === 'critical' || queueKey === 'error') {
this.retryQueue.unshift(...entries);
}
else {
this.retryQueue.push(...entries);
}
this.metrics.recordBatch(entries.length, false);
}
}
async processImmediate(entry) {
try {
await this.storageManager.store(entry);
this.metrics.recordImmediate(true);
this.logger.debug(`Processed immediate entry: ${entry.id}`);
}
catch (error) {
this.logger.error(`Failed to process immediate entry ${entry.id}:`, error);
this.retryQueue.push(entry);
this.metrics.recordImmediate(false);
}
}
async processRetryQueue() {
if (this.retryQueue.length === 0)
return;
const maxRetries = 10;
const entries = this.retryQueue.splice(0, maxRetries);
for (const entry of entries) {
try {
await this.storageManager.store(entry);
this.metrics.recordRetry(true);
this.logger.debug(`Retry successful for entry: ${entry.id}`);
}
catch (error) {
this.retryQueue.push(entry);
this.metrics.recordRetry(false);
this.logger.warn(`Retry failed for entry ${entry.id}:`, error);
}
}
}
startBatchProcessors() {
if (!this.config.storage.batch.enabled)
return;
const queueTypes = ['default', 'devtools', 'request', 'query', 'job', 'cache', 'performance'];
for (const queueKey of queueTypes) {
const queueConfig = this.getQueueConfig(queueKey);
const interval = setInterval(async () => {
const queue = this.batchQueues.get(queueKey);
if (queue && queue.length > 0) {
await this.flushQueue(queueKey);
}
}, queueConfig.flushInterval);
this.processingIntervals.set(queueKey, interval);
}
}
startRetryProcessor() {
setInterval(async () => {
await this.processRetryQueue();
}, 30000);
}
startMetricsReporting() {
setInterval(() => {
this.reportMetrics();
}, 300000);
}
reportMetrics() {
const metrics = this.metrics.getMetrics();
this.logger.log(`Processing metrics: ${JSON.stringify(metrics)}`);
this.metrics.reset();
}
ensureEntryFields(entry) {
if (!entry.id) {
entry.id = `tel_${(0, uuid_1.v4)()}`;
}
if (!entry.timestamp) {
entry.timestamp = new Date();
}
if (!entry.sequence) {
entry.sequence = ++this.sequenceCounter;
}
if (!entry.familyHash) {
entry.familyHash = this.generateFamilyHash(entry);
}
if (!entry.tags) {
entry.tags = [];
}
}
generateFamilyHash(entry) {
const hashInput = `${entry.type}:${JSON.stringify(entry.content).substring(0, 100)}`;
let hash = 0;
for (let i = 0; i < hashInput.length; i++) {
const char = hashInput.charCodeAt(i);
hash = ((hash << 5) - hash) + char;
hash = hash & hash;
}
return Math.abs(hash).toString(16);
}
getQueueConfig(queueKey) {
const baseConfig = {
batchSize: this.config.storage.batch.size,
flushInterval: this.config.storage.batch.flushInterval
};
switch (queueKey) {
case 'critical':
return { batchSize: 1, flushInterval: 100 };
case 'error':
return { batchSize: 5, flushInterval: 1000 };
case 'performance':
return { batchSize: baseConfig.batchSize * 2, flushInterval: baseConfig.flushInterval * 2 };
case 'devtools':
return { batchSize: baseConfig.batchSize, flushInterval: baseConfig.flushInterval * 3 };
default:
return baseConfig;
}
}
groupEntriesByQueue(entries) {
const groups = new Map();
for (const entry of entries) {
const queueKey = this.getQueueKey(entry);
if (!groups.has(queueKey)) {
groups.set(queueKey, []);
}
groups.get(queueKey).push(entry);
}
return groups;
}
getMetrics() {
return this.metrics;
}
getQueueStatus() {
const status = {};
for (const [queueKey, queue] of this.batchQueues) {
status[queueKey] = {
name: queueKey,
size: queue.length,
config: this.getQueueConfig(queueKey)
};
}
status['retry'] = {
name: 'retry',
size: this.retryQueue.length,
config: { batchSize: 10, flushInterval: 30000 }
};
return status;
}
async forceFlushAll() {
const flushPromises = Array.from(this.batchQueues.keys()).map(queueKey => this.flushQueue(queueKey));
await Promise.all(flushPromises);
this.logger.log('All queues flushed');
}
async cleanup() {
for (const [queueKey, interval] of this.processingIntervals) {
clearInterval(interval);
}
await this.forceFlushAll();
await this.processRetryQueue();
this.logger.log('Enhanced entry manager cleaned up');
}
};
exports.EnhancedEntryManagerService = EnhancedEntryManagerService;
exports.EnhancedEntryManagerService = EnhancedEntryManagerService = EnhancedEntryManagerService_1 = __decorate([
(0, common_1.Injectable)(),
__param(1, (0, common_1.Inject)('TELESCOPE_CONFIG')),
__metadata("design:paramtypes", [storage_manager_service_1.StorageManagerService, Object])
], EnhancedEntryManagerService);
class ProcessingMetrics {
constructor() {
this.entriesProcessed = 0;
this.batchesProcessed = 0;
this.batchesSuccessful = 0;
this.immediateProcessed = 0;
this.immediateSuccessful = 0;
this.retriesAttempted = 0;
this.retriesSuccessful = 0;
this.errors = 0;
this.startTime = Date.now();
}
recordEntry(entry) {
this.entriesProcessed++;
}
recordBatch(size, success) {
this.batchesProcessed++;
if (success) {
this.batchesSuccessful++;
}
else {
this.errors += size;
}
}
recordImmediate(success) {
this.immediateProcessed++;
if (success) {
this.immediateSuccessful++;
}
else {
this.errors++;
}
}
recordRetry(success) {
this.retriesAttempted++;
if (success) {
this.retriesSuccessful++;
}
}
recordError() {
this.errors++;
}
getMetrics() {
const elapsed = Date.now() - this.startTime;
const throughput = this.entriesProcessed / (elapsed / 1000);
return {
entriesProcessed: this.entriesProcessed,
batchesProcessed: this.batchesProcessed,
batchSuccessRate: this.batchesProcessed > 0 ? this.batchesSuccessful / this.batchesProcessed : 0,
immediateProcessed: this.immediateProcessed,
immediateSuccessRate: this.immediateProcessed > 0 ? this.immediateSuccessful / this.immediateProcessed : 0,
retriesAttempted: this.retriesAttempted,
retrySuccessRate: this.retriesAttempted > 0 ? this.retriesSuccessful / this.retriesAttempted : 0,
errors: this.errors,
throughput: throughput,
elapsedSeconds: elapsed / 1000
};
}
reset() {
this.entriesProcessed = 0;
this.batchesProcessed = 0;
this.batchesSuccessful = 0;
this.immediateProcessed = 0;
this.immediateSuccessful = 0;
this.retriesAttempted = 0;
this.retriesSuccessful = 0;
this.errors = 0;
this.startTime = Date.now();
}
}
//# sourceMappingURL=enhanced-entry-manager.service.js.map