@ahmedhegazee/nestjs-telescope
Version:
Advanced observability and monitoring solution for NestJS applications with ML-powered analytics, enterprise features, and production-ready scaling
492 lines • 20 kB
JavaScript
"use strict";
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var __param = (this && this.__param) || function (paramIndex, decorator) {
return function (target, key) { decorator(target, key, paramIndex); }
};
var DatabaseOptimizerService_1;
Object.defineProperty(exports, "__esModule", { value: true });
exports.DatabaseOptimizerService = void 0;
const common_1 = require("@nestjs/common");
const rxjs_1 = require("rxjs");
const common_2 = require("@nestjs/common");
let DatabaseOptimizerService = DatabaseOptimizerService_1 = class DatabaseOptimizerService {
constructor(telescopeConfig) {
this.telescopeConfig = telescopeConfig;
this.logger = new common_1.Logger(DatabaseOptimizerService_1.name);
this.queryMetrics = new Map();
this.indexSuggestions = new Map();
this.performanceSubject = new rxjs_1.Subject();
this.optimizationSubject = new rxjs_1.Subject();
this.monitoringInterval = null;
this.optimizationInterval = null;
this.config =
this.telescopeConfig.database || this.getDefaultDatabaseConfig();
}
async onModuleInit() {
if (!this.config.enabled) {
this.logger.log('Database optimization disabled');
return;
}
await this.initializeOptimizer();
this.startMonitoring();
this.startOptimization();
this.logger.log('Database optimizer service initialized');
}
getDefaultDatabaseConfig() {
return {
enabled: true,
type: 'postgresql',
connection: {
host: 'localhost',
port: 5432,
database: 'telescope',
username: 'telescope_user',
password: 'password',
ssl: false,
poolSize: 20,
timeout: 30000,
},
optimization: {
autoIndexing: true,
queryOptimization: true,
connectionPooling: true,
queryCaching: true,
slowQueryThreshold: 1000,
maxQueryTime: 30000,
},
monitoring: {
enabled: true,
metricsInterval: 60000,
slowQueryLogging: true,
performanceAlerts: true,
},
};
}
async initializeOptimizer() {
await this.initializeConnectionPool();
await this.initializeQueryCache();
await this.analyzeExistingIndexes();
}
async initializeConnectionPool() {
if (!this.config.optimization.connectionPooling)
return;
this.logger.log('Initializing connection pool');
}
async initializeQueryCache() {
if (!this.config.optimization.queryCaching)
return;
this.logger.log('Initializing query cache');
}
async analyzeExistingIndexes() {
this.logger.log('Analyzing existing database indexes');
}
startMonitoring() {
if (!this.config.monitoring.enabled)
return;
this.monitoringInterval = (0, rxjs_1.interval)(this.config.monitoring.metricsInterval).subscribe(async () => {
const performance = await this.getDatabasePerformance();
this.performanceSubject.next(performance);
if (this.config.monitoring.performanceAlerts) {
await this.checkPerformanceAlerts(performance);
}
});
}
startOptimization() {
this.optimizationInterval = (0, rxjs_1.interval)(300000).subscribe(async () => {
await this.runOptimizationCycle();
});
}
async recordQuery(queryMetrics) {
const queryHash = queryMetrics.queryHash;
if (!this.queryMetrics.has(queryHash)) {
this.queryMetrics.set(queryHash, []);
}
this.queryMetrics.get(queryHash).push(queryMetrics);
const queries = this.queryMetrics.get(queryHash);
if (queries.length > 1000) {
queries.splice(0, queries.length - 1000);
}
if (queryMetrics.slow) {
await this.analyzeSlowQuery(queryMetrics);
}
if (this.config.monitoring.slowQueryLogging && queryMetrics.slow) {
this.logger.warn(`Slow query detected: ${queryMetrics.sql} (${queryMetrics.executionTime}ms)`);
}
}
async analyzeSlowQuery(queryMetrics) {
const suggestions = await this.generateIndexSuggestions(queryMetrics);
for (const suggestion of suggestions) {
this.indexSuggestions.set(suggestion.id, suggestion);
if (suggestion.priority === 'critical' || suggestion.priority === 'high') {
this.logger.warn(`High priority index suggestion: ${suggestion.reason}`);
}
}
}
async generateIndexSuggestions(queryMetrics) {
const suggestions = [];
const queryAnalysis = this.analyzeQueryPattern(queryMetrics.sql);
if (queryAnalysis.needsIndex) {
suggestions.push({
id: `idx_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`,
table: queryAnalysis.table,
columns: queryAnalysis.columns,
type: this.determineIndexType(queryAnalysis.columns, queryAnalysis.usage),
reason: `Slow query on ${queryAnalysis.table} with ${queryAnalysis.columns.join(', ')}`,
estimatedImprovement: this.estimateIndexImprovement(queryMetrics.executionTime),
creationCost: this.estimateIndexCreationCost(queryAnalysis.table, queryAnalysis.columns),
priority: this.determineIndexPriority(queryMetrics.executionTime, queryAnalysis.usage),
status: 'pending',
});
}
return suggestions;
}
analyzeQueryPattern(sql) {
const lowerSql = sql.toLowerCase();
const needsIndex = lowerSql.includes('where') || lowerSql.includes('join') || lowerSql.includes('order by');
const tableMatch = lowerSql.match(/from\s+(\w+)/);
const table = tableMatch ? tableMatch[1] : 'unknown';
const columnMatch = lowerSql.match(/where\s+(\w+)/);
const columns = columnMatch ? [columnMatch[1]] : [];
let usage = 'where';
if (lowerSql.includes('join'))
usage = 'join';
else if (lowerSql.includes('order by'))
usage = 'order';
else if (lowerSql.includes('group by'))
usage = 'group';
return { needsIndex, table, columns, usage };
}
determineIndexType(columns, usage) {
if (columns.length > 1)
return 'btree';
if (usage === 'join')
return 'hash';
return 'btree';
}
estimateIndexImprovement(currentTime) {
if (currentTime > 10000)
return 90;
if (currentTime > 5000)
return 70;
if (currentTime > 1000)
return 50;
return 20;
}
estimateIndexCreationCost(table, columns) {
return columns.length * 10;
}
determineIndexPriority(executionTime, usage) {
if (executionTime > 10000)
return 'critical';
if (executionTime > 5000)
return 'high';
if (executionTime > 1000)
return 'medium';
return 'low';
}
async getDatabasePerformance() {
const connections = await this.getConnectionMetrics();
const queries = await this.getQueryMetrics();
const storage = await this.getStorageMetrics();
const cache = await this.getCacheMetrics();
const locks = await this.getLockMetrics();
return {
connections,
queries,
storage,
cache,
locks,
};
}
async getConnectionMetrics() {
return {
active: 5,
idle: 15,
max: 20,
utilization: 25,
};
}
async getQueryMetrics() {
const allQueries = Array.from(this.queryMetrics.values()).flat();
const recentQueries = allQueries.filter((q) => Date.now() - q.timestamp.getTime() < 60000);
const slowQueries = recentQueries.filter((q) => q.slow);
const executionTimes = recentQueries.map((q) => q.executionTime);
return {
total: recentQueries.length,
slow: slowQueries.length,
averageTime: executionTimes.length > 0
? executionTimes.reduce((sum, time) => sum + time, 0) / executionTimes.length
: 0,
peakTime: executionTimes.length > 0 ? Math.max(...executionTimes) : 0,
throughput: recentQueries.length,
};
}
async getStorageMetrics() {
return {
size: 1024 * 1024 * 1024,
growth: 1024 * 1024 * 100,
fragmentation: 5,
};
}
async getCacheMetrics() {
return {
hitRate: 0.85,
size: 1024 * 1024 * 50,
evictions: 100,
};
}
async getLockMetrics() {
return {
active: 2,
waiting: 0,
deadlocks: 0,
};
}
async checkPerformanceAlerts(performance) {
if (performance.connections.utilization > 80) {
this.logger.warn(`High connection pool utilization: ${performance.connections.utilization}%`);
}
const slowQueryRate = performance.queries.total > 0 ? performance.queries.slow / performance.queries.total : 0;
if (slowQueryRate > 0.1) {
this.logger.warn(`High slow query rate: ${(slowQueryRate * 100).toFixed(1)}%`);
}
if (performance.cache.hitRate < 0.8) {
this.logger.warn(`Low cache hit rate: ${(performance.cache.hitRate * 100).toFixed(1)}%`);
}
if (performance.locks.deadlocks > 0) {
this.logger.error(`Deadlocks detected: ${performance.locks.deadlocks}`);
}
}
async runOptimizationCycle() {
this.logger.log('Starting database optimization cycle');
try {
await this.analyzeQueryPatterns();
await this.processIndexSuggestions();
await this.optimizeSlowQueries();
await this.cleanupOldData();
await this.updateStatistics();
this.logger.log('Database optimization cycle completed');
}
catch (error) {
this.logger.error(`Database optimization cycle failed: ${error.message}`);
}
}
async analyzeQueryPatterns() {
for (const [queryHash, queries] of this.queryMetrics.entries()) {
if (queries.length < 10)
continue;
const avgTime = queries.reduce((sum, q) => sum + q.executionTime, 0) / queries.length;
const slowCount = queries.filter((q) => q.slow).length;
const slowRate = slowCount / queries.length;
if (slowRate > 0.5) {
await this.analyzeSlowQuery(queries[0]);
}
}
}
async processIndexSuggestions() {
const suggestions = Array.from(this.indexSuggestions.values())
.filter((s) => s.status === 'pending')
.sort((a, b) => {
const priorityOrder = { critical: 4, high: 3, medium: 2, low: 1 };
return priorityOrder[b.priority] - priorityOrder[a.priority];
});
for (const suggestion of suggestions.slice(0, 5)) {
await this.createIndex(suggestion);
}
}
async createIndex(suggestion) {
try {
this.logger.log(`Creating index: ${suggestion.table} (${suggestion.columns.join(', ')})`);
const startTime = Date.now();
await new Promise((resolve) => setTimeout(resolve, suggestion.creationCost * 1000));
const duration = Date.now() - startTime;
suggestion.status = 'created';
const result = {
success: true,
operation: `create_index_${suggestion.id}`,
duration,
improvements: {
queryTime: suggestion.estimatedImprovement,
throughput: suggestion.estimatedImprovement * 0.5,
resourceUsage: suggestion.estimatedImprovement * 0.3,
},
recommendations: [
`Index created successfully on ${suggestion.table}`,
`Expected query time improvement: ${suggestion.estimatedImprovement}%`,
],
};
this.optimizationSubject.next(result);
}
catch (error) {
suggestion.status = 'failed';
this.logger.error(`Failed to create index ${suggestion.id}: ${error.message}`);
const result = {
success: false,
operation: `create_index_${suggestion.id}`,
duration: 0,
improvements: { queryTime: 0, throughput: 0, resourceUsage: 0 },
recommendations: [],
errors: [error.message],
};
this.optimizationSubject.next(result);
}
}
async optimizeSlowQueries() {
const slowQueries = Array.from(this.queryMetrics.values())
.flat()
.filter((q) => q.slow && !q.optimized)
.slice(0, 10);
for (const query of slowQueries) {
await this.optimizeQuery(query);
}
}
async optimizeQuery(query) {
try {
this.logger.log(`Optimizing query: ${query.sql.substring(0, 100)}...`);
const optimization = await this.analyzeQueryOptimization(query.sql);
if (optimization.canOptimize) {
const optimizedSql = this.generateOptimizedQuery(query.sql, optimization);
const improvement = await this.testQueryOptimization(query.sql, optimizedSql);
if (improvement > 20) {
this.logger.log(`Query optimization successful: ${improvement.toFixed(1)}% improvement`);
}
}
}
catch (error) {
this.logger.error(`Failed to optimize query: ${error.message}`);
}
}
async analyzeQueryOptimization(sql) {
const suggestions = [];
let canOptimize = false;
const lowerSql = sql.toLowerCase();
if (lowerSql.includes('select *')) {
suggestions.push('Replace SELECT * with specific columns');
canOptimize = true;
}
if (lowerSql.includes('order by') && !lowerSql.includes('limit')) {
suggestions.push('Add LIMIT clause to ORDER BY queries');
canOptimize = true;
}
if (lowerSql.includes('like') && lowerSql.includes('%')) {
suggestions.push('Consider using full-text search instead of LIKE with wildcards');
canOptimize = true;
}
return { canOptimize, suggestions };
}
generateOptimizedQuery(originalSql, optimization) {
let optimizedSql = originalSql;
if (optimization.suggestions.includes('Replace SELECT * with specific columns')) {
optimizedSql = optimizedSql.replace(/select \*/i, 'SELECT id, name, created_at');
}
if (optimization.suggestions.includes('Add LIMIT clause to ORDER BY queries')) {
if (!optimizedSql.toLowerCase().includes('limit')) {
optimizedSql += ' LIMIT 1000';
}
}
return optimizedSql;
}
async testQueryOptimization(originalSql, optimizedSql) {
return 25;
}
async cleanupOldData() {
const cutoffTime = Date.now() - 7 * 24 * 60 * 60 * 1000;
for (const [queryHash, queries] of this.queryMetrics.entries()) {
const recentQueries = queries.filter((q) => q.timestamp.getTime() > cutoffTime);
if (recentQueries.length === 0) {
this.queryMetrics.delete(queryHash);
}
else {
this.queryMetrics.set(queryHash, recentQueries);
}
}
this.logger.log('Cleaned up old query metrics');
}
async updateStatistics() {
this.logger.log('Updating database statistics');
}
getPerformanceUpdates() {
return this.performanceSubject.asObservable();
}
getOptimizationUpdates() {
return this.optimizationSubject.asObservable();
}
getIndexSuggestions() {
return Array.from(this.indexSuggestions.values());
}
async createIndexManually(suggestion) {
const fullSuggestion = {
...suggestion,
id: `manual_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`,
status: 'pending',
};
this.indexSuggestions.set(fullSuggestion.id, fullSuggestion);
await this.createIndex(fullSuggestion);
return {
success: true,
operation: `manual_create_index_${fullSuggestion.id}`,
duration: 0,
improvements: {
queryTime: fullSuggestion.estimatedImprovement,
throughput: fullSuggestion.estimatedImprovement * 0.5,
resourceUsage: fullSuggestion.estimatedImprovement * 0.3,
},
recommendations: [
`Manual index creation initiated for ${fullSuggestion.table}`,
`Expected improvement: ${fullSuggestion.estimatedImprovement}%`,
],
};
}
async getQueryAnalysis(queryHash) {
const queries = this.queryMetrics.get(queryHash);
if (!queries || queries.length === 0)
return null;
const totalExecutions = queries.length;
const averageTime = queries.reduce((sum, q) => sum + q.executionTime, 0) / totalExecutions;
const slowExecutions = queries.filter((q) => q.slow).length;
const slowRate = slowExecutions / totalExecutions;
const recentQueries = queries.slice(-10);
const olderQueries = queries.slice(-20, -10);
const recentAvg = recentQueries.reduce((sum, q) => sum + q.executionTime, 0) / recentQueries.length;
const olderAvg = olderQueries.reduce((sum, q) => sum + q.executionTime, 0) / olderQueries.length;
let trend = 'stable';
if (recentAvg < olderAvg * 0.8)
trend = 'improving';
else if (recentAvg > olderAvg * 1.2)
trend = 'degrading';
return {
metrics: queries,
analysis: {
totalExecutions,
averageTime,
slowExecutions,
slowRate,
trend,
},
};
}
async shutdown() {
if (this.monitoringInterval) {
clearInterval(this.monitoringInterval);
}
if (this.optimizationInterval) {
clearInterval(this.optimizationInterval);
}
this.logger.log('Database optimizer service shutdown');
}
};
exports.DatabaseOptimizerService = DatabaseOptimizerService;
exports.DatabaseOptimizerService = DatabaseOptimizerService = DatabaseOptimizerService_1 = __decorate([
(0, common_1.Injectable)(),
__param(0, (0, common_2.Inject)('TELESCOPE_CONFIG')),
__metadata("design:paramtypes", [Object])
], DatabaseOptimizerService);
//# sourceMappingURL=database-optimizer.service.js.map