@kenniy/godeye-data-contracts
Version:
Enterprise-grade base repository architecture for GOD-EYE microservices with zero overhead and maximum code reuse
361 lines (360 loc) • 15.8 kB
JavaScript
;
/**
* BLAZING FAST Enhanced Aggregation Algorithms
*
* Next-generation aggregation engine with advanced optimization:
* - Adaptive query planning based on data size
* - Memory-efficient streaming for large datasets
* - Intelligent index utilization
* - Parallel processing for multi-step aggregations
* - Query result caching with smart invalidation
* - Performance analytics and auto-tuning
*/
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.PerformanceMonitor = exports.SmartCache = exports.AdaptiveQueryPlanner = exports.AggregationStepType = exports.QueryOptimizationLevel = exports.AggregationStrategy = void 0;
const os = __importStar(require("os"));
const crypto = __importStar(require("crypto"));
// ============================================================================
// PERFORMANCE OPTIMIZATION ENUMS
// ============================================================================
var AggregationStrategy;
(function (AggregationStrategy) {
/** Small datasets (<1K records) - Memory-based processing */
AggregationStrategy["MEMORY_OPTIMIZED"] = "memory_optimized";
/** Medium datasets (1K-100K records) - Hybrid approach */
AggregationStrategy["HYBRID"] = "hybrid";
/** Large datasets (>100K records) - Streaming + chunked processing */
AggregationStrategy["STREAMING"] = "streaming";
/** Very large datasets (>1M records) - Distributed processing */
AggregationStrategy["DISTRIBUTED"] = "distributed";
})(AggregationStrategy || (exports.AggregationStrategy = AggregationStrategy = {}));
var QueryOptimizationLevel;
(function (QueryOptimizationLevel) {
/** Basic optimization - Standard indexes */
QueryOptimizationLevel["BASIC"] = "basic";
/** Advanced optimization - Compound indexes + query hints */
QueryOptimizationLevel["ADVANCED"] = "advanced";
/** Enterprise optimization - Partitioning + materialized views */
QueryOptimizationLevel["ENTERPRISE"] = "enterprise";
})(QueryOptimizationLevel || (exports.QueryOptimizationLevel = QueryOptimizationLevel = {}));
var AggregationStepType;
(function (AggregationStepType) {
// Memory-optimized strategy steps
/** Standard filtering operation */
AggregationStepType["FILTER"] = "FILTER";
/** Standard join operation */
AggregationStepType["JOIN"] = "JOIN";
/** Standard aggregation operation */
AggregationStepType["AGGREGATE"] = "AGGREGATE";
/** Standard sorting operation */
AggregationStepType["SORT"] = "SORT";
// Streaming strategy steps
/** Streaming-based filtering */
AggregationStepType["STREAM_FILTER"] = "STREAM_FILTER";
/** Chunked join operation for large datasets */
AggregationStepType["CHUNK_JOIN"] = "CHUNK_JOIN";
/** Streaming aggregation for memory efficiency */
AggregationStepType["STREAMING_AGGREGATE"] = "STREAMING_AGGREGATE";
/** Merge sort for streaming results */
AggregationStepType["MERGE_SORT"] = "MERGE_SORT";
// Distributed strategy steps
/** Data partitioning for distributed processing */
AggregationStepType["PARTITION"] = "PARTITION";
/** Distributed filtering across partitions */
AggregationStepType["DISTRIBUTED_FILTER"] = "DISTRIBUTED_FILTER";
/** Map-reduce style join operation */
AggregationStepType["MAP_REDUCE_JOIN"] = "MAP_REDUCE_JOIN";
/** Distributed aggregation across partitions */
AggregationStepType["DISTRIBUTED_AGGREGATE"] = "DISTRIBUTED_AGGREGATE";
/** Merge results from distributed operations */
AggregationStepType["MERGE_RESULTS"] = "MERGE_RESULTS";
})(AggregationStepType || (exports.AggregationStepType = AggregationStepType = {}));
// ============================================================================
// ADAPTIVE QUERY PLANNER
// ============================================================================
class AdaptiveQueryPlanner {
/**
* Analyze query and dataset to determine optimal strategy
*/
static determineStrategy(config, estimatedRows) {
if (config.strategy) {
return config.strategy;
}
// Auto-detect based on estimated dataset size
if (estimatedRows < 1000) {
return AggregationStrategy.MEMORY_OPTIMIZED;
}
else if (estimatedRows < 100000) {
return AggregationStrategy.HYBRID;
}
else if (estimatedRows < 1000000) {
return AggregationStrategy.STREAMING;
}
else {
return AggregationStrategy.DISTRIBUTED;
}
}
/**
* Generate optimized query plan based on strategy
*/
static createQueryPlan(config, strategy) {
return {
strategy,
steps: this.generateQuerySteps(config, strategy),
indexHints: this.suggestIndexes(config),
memoryLimits: this.calculateMemoryLimits(config),
parallelization: this.determineParallelization(config, strategy),
};
}
static generateQuerySteps(config, strategy) {
const steps = [];
switch (strategy) {
case AggregationStrategy.MEMORY_OPTIMIZED:
steps.push({ type: AggregationStepType.FILTER, priority: 1, parallel: false }, { type: AggregationStepType.JOIN, priority: 2, parallel: false }, { type: AggregationStepType.AGGREGATE, priority: 3, parallel: false }, { type: AggregationStepType.SORT, priority: 4, parallel: false });
break;
case AggregationStrategy.HYBRID:
steps.push({ type: AggregationStepType.FILTER, priority: 1, parallel: false }, { type: AggregationStepType.JOIN, priority: 2, parallel: true }, { type: AggregationStepType.AGGREGATE, priority: 3, parallel: true }, { type: AggregationStepType.SORT, priority: 4, parallel: false });
break;
case AggregationStrategy.STREAMING:
steps.push({ type: AggregationStepType.STREAM_FILTER, priority: 1, parallel: false }, { type: AggregationStepType.CHUNK_JOIN, priority: 2, parallel: true }, { type: AggregationStepType.STREAMING_AGGREGATE, priority: 3, parallel: true }, { type: AggregationStepType.MERGE_SORT, priority: 4, parallel: true });
break;
case AggregationStrategy.DISTRIBUTED:
steps.push({ type: AggregationStepType.PARTITION, priority: 1, parallel: true }, { type: AggregationStepType.DISTRIBUTED_FILTER, priority: 2, parallel: true }, { type: AggregationStepType.MAP_REDUCE_JOIN, priority: 3, parallel: true }, { type: AggregationStepType.DISTRIBUTED_AGGREGATE, priority: 4, parallel: true }, { type: AggregationStepType.MERGE_RESULTS, priority: 5, parallel: false });
break;
}
return steps;
}
static suggestIndexes(config) {
const indexes = [];
// Add indexes for WHERE conditions
if (config.conditions) {
Object.keys(config.conditions).forEach(field => {
indexes.push(`idx_${field}`);
});
}
// Add indexes for JOIN fields
if (config.joins) {
config.joins.forEach(join => {
indexes.push(`idx_${join.localField}`);
indexes.push(`idx_${join.foreignField}`);
});
}
// Add indexes for GROUP BY fields
if (config.groupBy) {
config.groupBy.forEach(field => {
indexes.push(`idx_${field}`);
});
}
// Add compound indexes for common query patterns
if (config.conditions && config.groupBy) {
const compoundFields = [
...Object.keys(config.conditions),
...config.groupBy
].slice(0, 3); // Limit to 3 fields for compound index
indexes.push(`idx_compound_${compoundFields.join('_')}`);
}
return [...new Set(indexes)]; // Remove duplicates
}
static calculateMemoryLimits(config) {
const maxMemoryMB = config.maxMemoryMB || 512;
return {
totalMB: maxMemoryMB,
bufferMB: Math.min(maxMemoryMB * 0.3, 128), // 30% or 128MB max for buffers
cacheMB: Math.min(maxMemoryMB * 0.2, 64), // 20% or 64MB max for cache
workingMB: maxMemoryMB * 0.5, // 50% for working memory
};
}
static determineParallelization(config, strategy) {
if (!config.enableParallel) {
return { enabled: false, maxWorkers: 1 };
}
const maxCpuCores = os.cpus().length;
switch (strategy) {
case AggregationStrategy.MEMORY_OPTIMIZED:
return { enabled: false, maxWorkers: 1 };
case AggregationStrategy.HYBRID:
return {
enabled: true,
maxWorkers: Math.min(2, maxCpuCores),
chunkSize: config.chunkSize || 1000
};
case AggregationStrategy.STREAMING:
return {
enabled: true,
maxWorkers: Math.min(4, maxCpuCores),
chunkSize: config.chunkSize || 5000
};
case AggregationStrategy.DISTRIBUTED:
return {
enabled: true,
maxWorkers: maxCpuCores,
chunkSize: config.chunkSize || 10000
};
default:
return { enabled: false, maxWorkers: 1 };
}
}
}
exports.AdaptiveQueryPlanner = AdaptiveQueryPlanner;
// ============================================================================
// SMART CACHING SYSTEM
// ============================================================================
class SmartCache {
static generateCacheKey(config) {
// Create deterministic cache key from query config
const keyData = {
joins: config.joins || [],
aggregations: config.aggregations || [],
conditions: config.conditions || {},
groupBy: config.groupBy || [],
sort: config.sort || {},
select: config.select || [],
};
return `aggregation_${this.hashObject(keyData)}`;
}
static async get(key) {
const entry = this.cache.get(key);
if (!entry) {
return null;
}
if (Date.now() > entry.expiresAt) {
this.cache.delete(key);
return null;
}
entry.hitCount++;
entry.lastAccessed = Date.now();
return entry.data;
}
static set(key, data, ttlSeconds) {
const ttl = (ttlSeconds || this.DEFAULT_TTL) * 1000;
const entry = {
data,
createdAt: Date.now(),
expiresAt: Date.now() + ttl,
lastAccessed: Date.now(),
hitCount: 0,
size: this.estimateSize(data),
};
this.cache.set(key, entry);
this.evictIfNeeded();
}
static invalidate(pattern) {
for (const key of this.cache.keys()) {
if (key.includes(pattern)) {
this.cache.delete(key);
}
}
}
static hashObject(obj) {
return crypto
.createHash('md5')
.update(JSON.stringify(obj, Object.keys(obj).sort()))
.digest('hex');
}
static estimateSize(data) {
return JSON.stringify(data).length * 2; // Rough estimate in bytes
}
static evictIfNeeded() {
const MAX_CACHE_SIZE = 100 * 1024 * 1024; // 100MB
const MAX_ENTRIES = 1000;
if (this.cache.size <= MAX_ENTRIES) {
const totalSize = Array.from(this.cache.values())
.reduce((sum, entry) => sum + entry.size, 0);
if (totalSize <= MAX_CACHE_SIZE) {
return;
}
}
// Evict least recently used entries
const entries = Array.from(this.cache.entries())
.sort(([, a], [, b]) => a.lastAccessed - b.lastAccessed);
const toEvict = Math.max(1, Math.floor(entries.length * 0.2)); // Evict 20%
for (let i = 0; i < toEvict; i++) {
this.cache.delete(entries[i][0]);
}
}
}
exports.SmartCache = SmartCache;
SmartCache.cache = new Map();
SmartCache.DEFAULT_TTL = 300; // 5 minutes
// ============================================================================
// PERFORMANCE MONITOR
// ============================================================================
class PerformanceMonitor {
static startQuery(queryId) {
const session = {
queryId,
startTime: Date.now(),
startCpu: process.cpuUsage(),
startMemory: process.memoryUsage(),
};
return session;
}
static endQuery(session, result) {
const endTime = Date.now();
const endCpu = process.cpuUsage(session.startCpu);
const endMemory = process.memoryUsage();
const executionTimeMs = endTime - session.startTime;
const cpuUsagePercent = (endCpu.user + endCpu.system) / (executionTimeMs * 1000) * 100;
const memoryUsedMB = (endMemory.heapUsed - session.startMemory.heapUsed) / 1024 / 1024;
return {
executionTimeMs,
memoryUsedMB,
cpuUsagePercent,
cacheHit: false, // Will be set by calling code
strategy: AggregationStrategy.HYBRID, // Will be set by calling code
indexesUsed: [], // Will be populated by database-specific implementation
totalRows: Array.isArray(result) ? result.length : 1,
optimizationLevel: QueryOptimizationLevel.BASIC,
};
}
static analyzePerformance(metrics) {
const suggestions = [];
if (metrics.executionTimeMs > 5000) {
suggestions.push('Query execution time is high. Consider adding indexes or optimizing joins.');
}
if (metrics.memoryUsedMB > 100) {
suggestions.push('Memory usage is high. Consider using streaming aggregation for large datasets.');
}
if (metrics.cpuUsagePercent > 80) {
suggestions.push('CPU usage is high. Consider enabling parallel processing or reducing aggregation complexity.');
}
if (!metrics.cacheHit && metrics.executionTimeMs > 1000) {
suggestions.push('Consider enabling query result caching for frequently accessed data.');
}
return suggestions;
}
}
exports.PerformanceMonitor = PerformanceMonitor;