semantic-ds-toolkit
Version:
Performance-first semantic layer for modern data stacks - Stable Column Anchors & intelligent inference
532 lines • 20.4 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.WarehouseConnectionPool = exports.ConnectionPool = void 0;
exports.createOptimizedPool = createOptimizedPool;
exports.createGlobalPools = createGlobalPools;
const events_1 = require("events");
const performance_profiler_1 = require("./performance-profiler");
class ConnectionPool extends events_1.EventEmitter {
config;
connectionConfigs;
connections = new Map();
availableConnections = [];
requestQueue = [];
stats = {
totalConnections: 0,
idleConnections: 0,
activeConnections: 0,
queuedRequests: 0,
totalQueries: 0,
averageQueryTime: 0,
errorRate: 0
};
healthCheckTimer;
isShuttingDown = false;
constructor(connectionConfigs, poolConfig = {}) {
super();
this.connectionConfigs = Array.isArray(connectionConfigs) ? connectionConfigs : [connectionConfigs];
this.config = {
minConnections: 2,
maxConnections: 20,
acquireTimeout: 30000,
idleTimeout: 300000, // 5 minutes
maxLifetime: 1800000, // 30 minutes
healthCheckInterval: 60000, // 1 minute
reconnectInterval: 5000,
maxReconnectAttempts: 3,
...poolConfig
};
this.initialize();
}
async initialize() {
try {
// Create minimum connections
for (let i = 0; i < this.config.minConnections; i++) {
await this.createConnection();
}
// Start health check timer
this.startHealthCheck();
this.emit('initialized', { connectionCount: this.connections.size });
}
catch (error) {
this.emit('error', error);
}
}
async createConnection() {
if (this.connections.size >= this.config.maxConnections) {
throw new Error('Maximum connection limit reached');
}
const connectionId = `conn_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
const config = this.selectConnectionConfig();
const connection = {
id: connectionId,
config,
isConnected: false,
isIdle: true,
lastUsed: Date.now(),
createdAt: Date.now(),
queryCount: 0,
errorCount: 0
};
try {
// Simulate connection creation - in real implementation, connect to database
await this.simulateConnection(connection);
connection.isConnected = true;
this.connections.set(connectionId, connection);
this.availableConnections.push(connection);
this.stats.totalConnections++;
this.stats.idleConnections++;
this.emit('connectionCreated', { connectionId });
return connection;
}
catch (error) {
this.emit('connectionError', { connectionId, error });
throw error;
}
}
selectConnectionConfig() {
// Round-robin selection for load balancing
const index = this.connections.size % this.connectionConfigs.length;
return this.connectionConfigs[index];
}
async simulateConnection(connection) {
// Simulate database connection time
await new Promise(resolve => setTimeout(resolve, Math.random() * 100 + 50));
// In real implementation, create actual database connection:
// connection.nativeConnection = await createDatabaseConnection(connection.config);
}
async acquireConnection(timeout) {
const acquireTimeout = timeout || this.config.acquireTimeout;
const profilerKey = 'connection_acquire';
performance_profiler_1.globalProfiler.startOperation(profilerKey);
try {
// Check for available connection
const availableConnection = this.getAvailableConnection();
if (availableConnection) {
this.markConnectionBusy(availableConnection);
performance_profiler_1.globalProfiler.endOperation(profilerKey, 1);
return availableConnection;
}
// Try to create new connection if under limit
if (this.connections.size < this.config.maxConnections) {
const newConnection = await this.createConnection();
this.markConnectionBusy(newConnection);
performance_profiler_1.globalProfiler.endOperation(profilerKey, 1);
return newConnection;
}
// Queue the request
const queuedRequest = await this.queueConnectionRequest(acquireTimeout);
performance_profiler_1.globalProfiler.endOperation(profilerKey, 1);
return queuedRequest;
}
catch (error) {
performance_profiler_1.globalProfiler.endOperation(profilerKey, 0);
throw error;
}
}
getAvailableConnection() {
const available = this.availableConnections.find(conn => conn.isConnected && conn.isIdle && this.isConnectionHealthy(conn));
if (available) {
// Remove from available pool
const index = this.availableConnections.indexOf(available);
this.availableConnections.splice(index, 1);
}
return available || null;
}
isConnectionHealthy(connection) {
const now = Date.now();
const age = now - connection.createdAt;
const idleTime = now - connection.lastUsed;
// Check if connection is too old or idle too long
if (age > this.config.maxLifetime || idleTime > this.config.idleTimeout) {
return false;
}
// Check error rate
if (connection.errorCount > 0 && connection.errorCount / connection.queryCount > 0.1) {
return false;
}
return true;
}
markConnectionBusy(connection) {
connection.isIdle = false;
connection.lastUsed = Date.now();
this.stats.idleConnections--;
this.stats.activeConnections++;
}
async queueConnectionRequest(timeout) {
return new Promise((resolve, reject) => {
const requestId = `req_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
const queuedRequest = {
id: requestId,
resolve,
reject,
timestamp: Date.now(),
priority: 'medium',
timeout
};
this.requestQueue.push(queuedRequest);
this.stats.queuedRequests++;
// Sort queue by priority and timestamp
this.requestQueue.sort((a, b) => {
const priorityWeight = { high: 3, medium: 2, low: 1 };
const aPriority = priorityWeight[a.priority];
const bPriority = priorityWeight[b.priority];
if (aPriority !== bPriority) {
return bPriority - aPriority; // Higher priority first
}
return a.timestamp - b.timestamp; // FIFO for same priority
});
// Set timeout
setTimeout(() => {
const index = this.requestQueue.findIndex(req => req.id === requestId);
if (index >= 0) {
this.requestQueue.splice(index, 1);
this.stats.queuedRequests--;
reject(new Error(`Connection acquisition timeout after ${timeout}ms`));
}
}, timeout);
});
}
releaseConnection(connection) {
if (!connection.isIdle) {
connection.isIdle = true;
connection.lastUsed = Date.now();
this.stats.activeConnections--;
this.stats.idleConnections++;
// Check if connection is still healthy
if (this.isConnectionHealthy(connection)) {
this.availableConnections.push(connection);
// Process queued requests
this.processQueuedRequests();
}
else {
// Remove unhealthy connection
this.removeConnection(connection);
// Ensure minimum connections
this.ensureMinimumConnections();
}
}
}
processQueuedRequests() {
while (this.requestQueue.length > 0 && this.availableConnections.length > 0) {
const request = this.requestQueue.shift();
const connection = this.getAvailableConnection();
if (connection) {
this.markConnectionBusy(connection);
this.stats.queuedRequests--;
request.resolve(connection);
}
else {
// Put request back if no connection available
this.requestQueue.unshift(request);
break;
}
}
}
removeConnection(connection) {
this.connections.delete(connection.id);
// Remove from available pool
const index = this.availableConnections.indexOf(connection);
if (index >= 0) {
this.availableConnections.splice(index, 1);
}
// Close native connection
this.closeNativeConnection(connection);
this.stats.totalConnections--;
if (connection.isIdle) {
this.stats.idleConnections--;
}
else {
this.stats.activeConnections--;
}
this.emit('connectionRemoved', { connectionId: connection.id });
}
closeNativeConnection(connection) {
// In real implementation, close actual database connection
if (connection.nativeConnection) {
// connection.nativeConnection.close();
}
}
async ensureMinimumConnections() {
const currentConnections = this.connections.size;
const needed = this.config.minConnections - currentConnections;
if (needed > 0) {
const createPromises = [];
for (let i = 0; i < needed; i++) {
createPromises.push(this.createConnection().catch(err => {
this.emit('error', err);
}));
}
await Promise.allSettled(createPromises);
}
}
// High-level query execution
async executeQuery(sql, params, options = {}) {
const profilerKey = 'query_execution';
performance_profiler_1.globalProfiler.startOperation(profilerKey, { sql: sql.substring(0, 100) });
let connection = null;
try {
connection = await this.acquireConnection(options.timeout);
const result = await this.runQuery(connection, sql, params, options);
this.updateQueryStats(connection, result.executionTime, false);
performance_profiler_1.globalProfiler.endOperation(profilerKey, 1);
return result;
}
catch (error) {
if (connection) {
this.updateQueryStats(connection, 0, true);
}
performance_profiler_1.globalProfiler.endOperation(profilerKey, 0);
throw error;
}
finally {
if (connection) {
this.releaseConnection(connection);
}
}
}
async runQuery(connection, sql, params, options = {}) {
const startTime = Date.now();
// Simulate query execution - in real implementation, use actual database query
await new Promise(resolve => setTimeout(resolve, Math.random() * 100 + 10));
const executionTime = Date.now() - startTime;
// Mock result - in real implementation, return actual query results
const result = {
rows: this.generateMockRows(sql),
rowCount: Math.floor(Math.random() * 1000) + 1,
executionTime,
columns: this.extractColumns(sql)
};
return result;
}
generateMockRows(sql) {
// Generate mock data based on query type
const rowCount = Math.floor(Math.random() * 100) + 1;
const rows = [];
for (let i = 0; i < rowCount; i++) {
rows.push({
id: i + 1,
name: `item_${i}`,
value: Math.random() * 1000,
created_at: new Date()
});
}
return rows;
}
extractColumns(sql) {
// Mock column extraction - in real implementation, get from query metadata
return ['id', 'name', 'value', 'created_at'];
}
updateQueryStats(connection, executionTime, isError) {
connection.queryCount++;
if (isError) {
connection.errorCount++;
}
this.stats.totalQueries++;
this.stats.averageQueryTime =
(this.stats.averageQueryTime * (this.stats.totalQueries - 1) + executionTime) / this.stats.totalQueries;
if (isError) {
this.stats.errorRate =
(this.stats.errorRate * (this.stats.totalQueries - 1) + 1) / this.stats.totalQueries;
}
else {
this.stats.errorRate =
(this.stats.errorRate * (this.stats.totalQueries - 1)) / this.stats.totalQueries;
}
}
// Batch query execution
async executeBatch(queries) {
const profilerKey = 'batch_query_execution';
performance_profiler_1.globalProfiler.startOperation(profilerKey, { batchSize: queries.length });
try {
// Acquire connections for parallel execution
const connectionPromises = queries.map(() => this.acquireConnection());
const connections = await Promise.all(connectionPromises);
// Execute queries in parallel
const resultPromises = queries.map((query, index) => this.runQuery(connections[index], query.sql, query.params));
const results = await Promise.all(resultPromises);
// Release all connections
connections.forEach(conn => this.releaseConnection(conn));
performance_profiler_1.globalProfiler.endOperation(profilerKey, queries.length);
return results;
}
catch (error) {
performance_profiler_1.globalProfiler.endOperation(profilerKey, 0);
throw error;
}
}
startHealthCheck() {
this.healthCheckTimer = setInterval(() => {
this.performHealthCheck();
}, this.config.healthCheckInterval);
}
async performHealthCheck() {
if (this.isShuttingDown)
return;
const unhealthyConnections = [];
for (const connection of this.connections.values()) {
if (!this.isConnectionHealthy(connection)) {
unhealthyConnections.push(connection);
}
}
// Remove unhealthy connections
for (const connection of unhealthyConnections) {
if (connection.isIdle) {
this.removeConnection(connection);
}
else {
// Mark for removal when released
connection.errorCount = Number.MAX_SAFE_INTEGER;
}
}
// Ensure minimum connections
await this.ensureMinimumConnections();
this.emit('healthCheck', {
totalConnections: this.connections.size,
removedConnections: unhealthyConnections.length
});
}
getStats() {
return { ...this.stats };
}
// Graceful shutdown
async shutdown() {
this.isShuttingDown = true;
// Clear health check timer
if (this.healthCheckTimer) {
clearInterval(this.healthCheckTimer);
}
// Reject queued requests
for (const request of this.requestQueue) {
request.reject(new Error('Connection pool is shutting down'));
}
this.requestQueue = [];
// Close all connections
const closePromises = Array.from(this.connections.values()).map(async (connection) => {
// Wait for active connections to finish (with timeout)
const timeout = 10000; // 10 seconds
const start = Date.now();
while (!connection.isIdle && Date.now() - start < timeout) {
await new Promise(resolve => setTimeout(resolve, 100));
}
this.closeNativeConnection(connection);
});
await Promise.allSettled(closePromises);
this.connections.clear();
this.availableConnections = [];
this.emit('shutdown');
}
}
exports.ConnectionPool = ConnectionPool;
// Warehouse-specific connection pool with optimizations
class WarehouseConnectionPool extends ConnectionPool {
queryCache = new Map();
constructor(connectionConfigs, poolConfig) {
super(connectionConfigs, {
minConnections: 5,
maxConnections: 50, // Higher for data warehouse workloads
acquireTimeout: 60000, // Longer timeout for complex queries
idleTimeout: 600000, // 10 minutes
maxLifetime: 3600000, // 1 hour
...poolConfig
});
}
async executeQuery(sql, params, options = {}) {
// Check cache for SELECT queries
if (options.cacheable && sql.trim().toLowerCase().startsWith('select')) {
const cacheKey = this.getCacheKey(sql, params);
const cached = this.getCachedResult(cacheKey);
if (cached) {
return cached;
}
}
const result = await super.executeQuery(sql, params, options);
// Cache SELECT results
if (options.cacheable && sql.trim().toLowerCase().startsWith('select')) {
const cacheKey = this.getCacheKey(sql, params);
this.cacheResult(cacheKey, result, 300000); // 5 minutes TTL
}
return result;
}
getCacheKey(sql, params) {
return `${sql}|${JSON.stringify(params || [])}`;
}
getCachedResult(cacheKey) {
const cached = this.queryCache.get(cacheKey);
if (!cached)
return null;
if (Date.now() > cached.timestamp + cached.ttl) {
this.queryCache.delete(cacheKey);
return null;
}
return cached.result;
}
cacheResult(cacheKey, result, ttl) {
this.queryCache.set(cacheKey, {
result: { ...result }, // Clone to prevent mutations
timestamp: Date.now(),
ttl
});
// Cleanup old cache entries periodically
if (this.queryCache.size > 1000) {
this.cleanupCache();
}
}
cleanupCache() {
const now = Date.now();
for (const [key, cached] of this.queryCache.entries()) {
if (now > cached.timestamp + cached.ttl) {
this.queryCache.delete(key);
}
}
}
}
exports.WarehouseConnectionPool = WarehouseConnectionPool;
// Factory function for creating optimized pools
function createOptimizedPool(type, connectionConfigs) {
const baseConfig = {
oltp: {
minConnections: 10,
maxConnections: 100,
acquireTimeout: 5000,
idleTimeout: 180000, // 3 minutes
maxLifetime: 900000 // 15 minutes
},
olap: {
minConnections: 3,
maxConnections: 20,
acquireTimeout: 60000,
idleTimeout: 1800000, // 30 minutes
maxLifetime: 7200000 // 2 hours
},
mixed: {
minConnections: 5,
maxConnections: 50,
acquireTimeout: 30000,
idleTimeout: 600000, // 10 minutes
maxLifetime: 1800000 // 30 minutes
}
};
const config = baseConfig[type];
if (type === 'olap') {
return new WarehouseConnectionPool(connectionConfigs, config);
}
else {
return new ConnectionPool(connectionConfigs, config);
}
}
// Global pool instances would be created based on configuration
function createGlobalPools(configs) {
const pools = {};
if (configs.oltp) {
pools.oltpPool = createOptimizedPool('oltp', configs.oltp);
}
if (configs.olap) {
pools.olapPool = createOptimizedPool('olap', configs.olap);
}
if (configs.mixed) {
pools.mixedPool = createOptimizedPool('mixed', configs.mixed);
}
return pools;
}
//# sourceMappingURL=connection-pool.js.map