@ahmedhegazee/nestjs-telescope
Version:
Advanced observability and monitoring solution for NestJS applications with ML-powered analytics, enterprise features, and production-ready scaling
266 lines • 11.5 kB
JavaScript
"use strict";
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var __param = (this && this.__param) || function (paramIndex, decorator) {
return function (target, key) { decorator(target, key, paramIndex); }
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.StreamProcessingBridgeService = void 0;
const common_1 = require("@nestjs/common");
const rxjs_1 = require("rxjs");
const operators_1 = require("rxjs/operators");
const rxjs_2 = require("rxjs");
const devtools_bridge_service_1 = require("./devtools-bridge.service");
const telescope_service_1 = require("../../core/services/telescope.service");
const enhanced_entry_manager_service_1 = require("../../core/services/enhanced-entry-manager.service");
const metrics_service_1 = require("../../core/services/metrics.service");
let StreamProcessingBridgeService = class StreamProcessingBridgeService extends devtools_bridge_service_1.DevToolsBridgeService {
constructor(telescopeService, entryManager, metricsService, config) {
super(telescopeService, config);
this.entryManager = entryManager;
this.metricsService = metricsService;
this.entryStream = new rxjs_1.Subject();
this.errorStream = new rxjs_1.Subject();
this.subscriptions = [];
this.isProcessing = false;
this.streamConfig = {
bufferTimeMs: 1000,
maxBufferSize: 100,
maxConcurrentBatches: 3,
retryDelayMs: 1000,
maxRetries: 3,
errorThrottleMs: 5000
};
this.updateStreamConfig(config);
}
async onModuleInit() {
this.setupStreams();
this.isProcessing = true;
this.logger.log('Stream processing bridge initialized');
}
async onModuleDestroy() {
this.isProcessing = false;
this.subscriptions.forEach(sub => sub.unsubscribe());
this.entryStream.complete();
this.errorStream.complete();
this.logger.log('Stream processing bridge destroyed');
}
updateStreamConfig(config) {
if (config.storage?.batch) {
this.streamConfig.bufferTimeMs = config.storage.batch.flushInterval || 1000;
this.streamConfig.maxBufferSize = config.storage.batch.size || 100;
}
if (config.features?.realTimeUpdates === false) {
this.streamConfig.bufferTimeMs = Math.max(this.streamConfig.bufferTimeMs, 5000);
}
}
setupStreams() {
const entrySubscription = this.entryStream
.pipe((0, operators_1.bufferTime)(this.streamConfig.bufferTimeMs, null, this.streamConfig.maxBufferSize), (0, operators_1.filter)(entries => entries.length > 0), (0, operators_1.tap)(entries => this.logger.debug(`Processing batch of ${entries.length} entries`)), (0, operators_1.mergeMap)(entries => this.processBatchSafely(entries), this.streamConfig.maxConcurrentBatches), (0, operators_1.retry)({
count: this.streamConfig.maxRetries,
delay: (error, retryCount) => {
this.logger.warn(`Batch processing failed, retry ${retryCount}:`, error.message);
return (0, rxjs_1.timer)(this.streamConfig.retryDelayMs * Math.pow(2, retryCount - 1));
}
}), (0, operators_1.catchError)(error => {
this.logger.error('Stream processing failed after retries:', error);
this.errorStream.next({
message: `Stream processing failed: ${error.message}`,
stack: error.stack,
timestamp: new Date(),
retryCount: this.streamConfig.maxRetries
});
return (0, rxjs_2.of)(null);
}), (0, operators_1.finalize)(() => this.logger.debug('Entry stream finalized')))
.subscribe({
next: (result) => {
if (result) {
this.handleBatchResults(result);
}
},
error: (error) => {
this.logger.error('Unhandled stream error:', error);
this.errorStream.next({
message: `Unhandled stream error: ${error.message}`,
stack: error.stack,
timestamp: new Date(),
retryCount: 0
});
}
});
const errorSubscription = this.errorStream
.pipe((0, operators_1.throttleTime)(this.streamConfig.errorThrottleMs), (0, operators_1.tap)(error => {
this.logger.error('Stream processing error:', {
message: error.message,
timestamp: error.timestamp,
retryCount: error.retryCount
});
}))
.subscribe();
this.subscriptions.push(entrySubscription, errorSubscription);
}
async processBatchSafely(entries) {
const startTime = Date.now();
const batchId = `batch_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
try {
this.logger.debug(`Starting batch processing: ${batchId}`);
const entriesWithBatch = entries.map(entry => ({
...entry,
batchId
}));
await this.entryManager.processBatch(entriesWithBatch);
const result = {
processed: entries.length,
failed: 0,
duration: Date.now() - startTime,
success: true,
timestamp: new Date()
};
this.logger.debug(`Batch processed successfully: ${batchId} (${result.duration}ms)`);
return result;
}
catch (error) {
const result = {
processed: 0,
failed: entries.length,
duration: Date.now() - startTime,
success: false,
error: error.message,
timestamp: new Date()
};
this.logger.error(`Batch processing failed: ${batchId}`, error);
const individualResults = await this.fallbackToIndividualProcessing(entries);
result.processed = individualResults.processed;
result.failed = individualResults.failed;
if (result.processed > 0) {
result.success = true;
this.logger.warn(`Batch partially recovered: ${result.processed}/${entries.length} entries processed`);
}
return result;
}
}
async fallbackToIndividualProcessing(entries) {
let processed = 0;
let failed = 0;
for (const entry of entries) {
try {
await this.entryManager.process(entry);
processed++;
}
catch (error) {
failed++;
this.logger.debug(`Individual entry processing failed: ${entry.id}`, error.message);
}
}
return { processed, failed };
}
handleBatchResults(result) {
this.metricsService.recordBatchProcessing(result);
if (result.success) {
this.logger.debug(`Batch processed: ${result.processed} entries in ${result.duration}ms`);
}
else {
this.logger.warn(`Batch failed: ${result.failed} entries failed, ${result.processed} recovered`);
}
}
async processDevToolsEntry(entry, type) {
if (!this.isProcessing) {
this.logger.warn('Stream processing is not active, dropping entry');
return;
}
try {
const telescopeEntry = this.transformToTelescopeFormat(entry, type);
this.entryStream.next(telescopeEntry);
}
catch (error) {
this.logger.error('Failed to transform DevTools entry:', error);
this.errorStream.next({
message: `Transformation failed: ${error.message}`,
stack: error.stack,
timestamp: new Date(),
retryCount: 0
});
}
}
transformToTelescopeFormat(entry, type) {
try {
const baseEntry = super.transformToTelescopeFormat(entry, type);
return {
...baseEntry,
tags: [
...baseEntry.tags,
'stream-processed',
`stream-${this.streamConfig.bufferTimeMs}ms`
],
content: {
...baseEntry.content,
streamMetadata: {
processedAt: new Date().toISOString(),
streamConfig: this.streamConfig
}
}
};
}
catch (error) {
this.logger.error('Transformation failed:', error);
throw new Error(`Failed to transform entry: ${error.message}`);
}
}
getStreamMetrics() {
const baseMetrics = this.metricsService.getStreamMetrics();
return {
...baseMetrics,
entriesInQueue: this.entryStream.observers?.length || 0,
isProcessing: this.isProcessing,
subscriptions: this.subscriptions.length
};
}
getStreamConfiguration() {
return { ...this.streamConfig };
}
updateStreamConfiguration(config) {
Object.assign(this.streamConfig, config);
this.logger.log('Stream configuration updated:', config);
}
async flushBuffer() {
this.logger.log('Forcing stream buffer flush');
return Promise.resolve();
}
getHealthStatus() {
const issues = [];
const metrics = this.metricsService.getStreamMetrics();
if (!this.isProcessing) {
issues.push('Stream processing is not active');
}
if (metrics.errorCount > 10) {
issues.push(`High error count: ${metrics.errorCount}`);
}
if (metrics.averageProcessingTime > 5000) {
issues.push(`High average processing time: ${metrics.averageProcessingTime}ms`);
}
if (metrics.throughput < 1) {
issues.push(`Low throughput: ${metrics.throughput} entries/second`);
}
return {
isHealthy: issues.length === 0,
issues,
lastProcessedAt: metrics.lastProcessedAt
};
}
};
exports.StreamProcessingBridgeService = StreamProcessingBridgeService;
exports.StreamProcessingBridgeService = StreamProcessingBridgeService = __decorate([
(0, common_1.Injectable)(),
__param(3, (0, common_1.Inject)('TELESCOPE_CONFIG')),
__metadata("design:paramtypes", [telescope_service_1.TelescopeService,
enhanced_entry_manager_service_1.EnhancedEntryManagerService,
metrics_service_1.MetricsService, Object])
], StreamProcessingBridgeService);
//# sourceMappingURL=stream-processing-bridge.service.js.map