supa-seed
Version:
A constraint-aware, framework-agnostic database seeding framework with deep PostgreSQL business logic discovery and MakerKit integration support
326 lines • 13.6 kB
JavaScript
"use strict";
/**
* Streaming Batch Processor for Memory-Efficient Data Processing
* FEAT-003: Memory Management & Schema Mapping Fixes
*
* This module provides streaming batch processing capabilities to prevent
* out-of-memory errors when processing large datasets.
*/
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.StreamingBatchProcessor = void 0;
exports.createBatchProcessor = createBatchProcessor;
exports.batchProcessed = batchProcessed;
const logger_1 = require("./utils/logger");
const memory_manager_1 = __importDefault(require("./utils/memory-manager"));
const performance_monitor_1 = require("./utils/performance-monitor");
/**
* Streaming batch processor for memory-efficient data processing
*/
class StreamingBatchProcessor {
constructor(config) {
this.startTime = 0;
this.memoryReadings = [];
this.config = {
defaultBatchSize: 50,
memoryThresholdMB: 800,
minBatchSize: 10,
maxBatchSize: 200,
forceGCBetweenBatches: true,
batchDelayMs: 100,
...config
};
this.stats = {
totalItems: 0,
processedItems: 0,
batchesCompleted: 0,
currentBatchSize: this.config.defaultBatchSize,
averageMemoryUsageMB: 0,
peakMemoryUsageMB: 0,
totalProcessingTimeMs: 0,
averageBatchTimeMs: 0
};
logger_1.Logger.info('🔄 StreamingBatchProcessor initialized:', {
defaultBatchSize: this.config.defaultBatchSize,
memoryThresholdMB: this.config.memoryThresholdMB,
forceGC: this.config.forceGCBetweenBatches
});
}
/**
* Process items in streaming batches with automatic memory management
*/
async *processBatches(items, processor) {
this.initializeProcessing(items.length);
try {
for await (const batchResults of this.streamBatches(items, processor)) {
yield batchResults;
// Memory management between batches
await this.performBatchCleanup();
}
}
finally {
this.finalizeProcessing();
}
}
/**
* Process all items and return aggregated results
*/
async processAll(items, processor) {
const allResults = [];
for await (const batchResults of this.processBatches(items, processor)) {
allResults.push(...batchResults);
}
return {
results: allResults,
stats: this.getStats(),
recommendations: this.getRecommendations()
};
}
/**
* Get current processing statistics
*/
getStats() {
return { ...this.stats };
}
/**
* Get processing recommendations based on current performance
*/
getRecommendations() {
const recommendations = [];
const memoryStats = memory_manager_1.default.getMemoryStats();
if (this.stats.peakMemoryUsageMB > this.config.memoryThresholdMB) {
recommendations.push(`Peak memory usage (${this.stats.peakMemoryUsageMB.toFixed(1)}MB) exceeded threshold - consider reducing batch size`);
}
if (this.stats.averageBatchTimeMs > 30000) { // 30 seconds
recommendations.push('Long batch processing times detected - consider optimizing batch operations');
}
if (memoryStats.percentageUsed > 80) {
recommendations.push('High memory usage detected - enable more aggressive garbage collection');
}
if (this.stats.currentBatchSize < this.config.defaultBatchSize * 0.5) {
recommendations.push('Batch size was reduced due to memory pressure - consider optimizing memory usage per item');
}
return recommendations;
}
/**
* Create batches with dynamic sizing based on memory pressure
*/
*createDynamicBatches(items) {
let startIndex = 0;
while (startIndex < items.length) {
const currentBatchSize = this.calculateOptimalBatchSize();
const endIndex = Math.min(startIndex + currentBatchSize, items.length);
const batch = items
.slice(startIndex, endIndex)
.map((data, localIndex) => ({
data,
index: startIndex + localIndex,
metadata: { batchNumber: this.stats.batchesCompleted + 1 }
}));
this.stats.currentBatchSize = batch.length;
yield batch;
startIndex = endIndex;
}
}
/**
* Stream batches with memory monitoring
*/
async *streamBatches(items, processor) {
for (const batch of this.createDynamicBatches(items)) {
const batchStartTime = Date.now();
const memoryBefore = process.memoryUsage().heapUsed / 1024 / 1024;
logger_1.Logger.debug(`🔄 Processing batch ${this.stats.batchesCompleted + 1}:`, {
size: batch.length,
progress: `${this.stats.processedItems}/${this.stats.totalItems}`,
memoryMB: memoryBefore.toFixed(1)
});
try {
// Process the batch
const results = await processor(batch);
// Update statistics
const batchEndTime = Date.now();
const processingTime = batchEndTime - batchStartTime;
const memoryAfter = process.memoryUsage().heapUsed / 1024 / 1024;
this.updateBatchStats(batch.length, processingTime, memoryAfter);
yield results;
}
catch (error) {
logger_1.Logger.error(`❌ Batch processing failed:`, error);
// Create error results for the failed batch
const errorResults = batch.map(item => ({
success: false,
error: error,
processingTimeMs: Date.now() - batchStartTime,
memoryUsageMB: process.memoryUsage().heapUsed / 1024 / 1024
}));
yield errorResults;
}
}
}
/**
* Calculate optimal batch size based on current memory conditions
*/
calculateOptimalBatchSize() {
const memoryStats = memory_manager_1.default.getMemoryStats();
const currentMemoryMB = memoryStats.usage.heapUsedMB;
// If memory usage is high, reduce batch size
if (currentMemoryMB > this.config.memoryThresholdMB) {
const reductionFactor = Math.min(this.config.memoryThresholdMB / currentMemoryMB, 1.0);
const reducedSize = Math.floor(this.config.defaultBatchSize * reductionFactor);
return Math.max(reducedSize, this.config.minBatchSize);
}
// If memory usage is low, we can use the default or even increase slightly
if (currentMemoryMB < this.config.memoryThresholdMB * 0.6) {
return Math.min(this.config.defaultBatchSize, this.config.maxBatchSize);
}
return this.config.defaultBatchSize;
}
/**
* Perform cleanup between batches
*/
async performBatchCleanup() {
// Record memory usage
const currentMemory = process.memoryUsage().heapUsed / 1024 / 1024;
this.memoryReadings.push(currentMemory);
// Force garbage collection if enabled and memory usage is high
if (this.config.forceGCBetweenBatches) {
const memoryStats = memory_manager_1.default.getMemoryStats();
if (memoryStats.usage.heapUsedMB > this.config.memoryThresholdMB * 0.7) {
logger_1.Logger.debug('🗑️ Forcing garbage collection between batches');
await memory_manager_1.default.forceCleanup('batch_processing');
}
}
// Add delay for memory pressure relief if configured
if (this.config.batchDelayMs > 0) {
await new Promise(resolve => setTimeout(resolve, this.config.batchDelayMs));
}
// Record performance metric
performance_monitor_1.PerformanceMonitor.recordMetric({
name: 'batch_processor_memory_usage',
value: currentMemory,
unit: 'mb',
timestamp: new Date(),
tags: {
batch_number: this.stats.batchesCompleted.toString(),
batch_size: this.stats.currentBatchSize.toString()
}
});
}
/**
* Initialize processing statistics
*/
initializeProcessing(totalItems) {
this.startTime = Date.now();
this.stats.totalItems = totalItems;
this.stats.processedItems = 0;
this.stats.batchesCompleted = 0;
this.memoryReadings = [];
logger_1.Logger.info('🚀 Starting batch processing:', {
totalItems,
estimatedBatches: Math.ceil(totalItems / this.config.defaultBatchSize),
batchSize: this.config.defaultBatchSize
});
}
/**
* Update statistics after each batch
*/
updateBatchStats(batchSize, processingTime, memoryUsage) {
this.stats.processedItems += batchSize;
this.stats.batchesCompleted++;
this.stats.peakMemoryUsageMB = Math.max(this.stats.peakMemoryUsageMB, memoryUsage);
// Calculate averages
this.stats.averageMemoryUsageMB = this.memoryReadings.length > 0
? this.memoryReadings.reduce((a, b) => a + b, 0) / this.memoryReadings.length
: memoryUsage;
this.stats.totalProcessingTimeMs += processingTime;
this.stats.averageBatchTimeMs = this.stats.totalProcessingTimeMs / this.stats.batchesCompleted;
// Log progress
const progressPercent = (this.stats.processedItems / this.stats.totalItems * 100).toFixed(1);
logger_1.Logger.info(`📊 Batch ${this.stats.batchesCompleted} completed:`, {
progress: `${progressPercent}%`,
processed: `${this.stats.processedItems}/${this.stats.totalItems}`,
batchTime: `${processingTime}ms`,
memoryMB: memoryUsage.toFixed(1)
});
}
/**
* Finalize processing and log summary
*/
finalizeProcessing() {
const totalTime = Date.now() - this.startTime;
const avgItemTime = totalTime / this.stats.totalItems;
logger_1.Logger.info('✅ Batch processing completed:', {
totalItems: this.stats.totalItems,
totalBatches: this.stats.batchesCompleted,
totalTimeMs: totalTime,
avgItemTimeMs: avgItemTime.toFixed(2),
peakMemoryMB: this.stats.peakMemoryUsageMB.toFixed(1),
avgMemoryMB: this.stats.averageMemoryUsageMB.toFixed(1)
});
// Record final performance metrics
performance_monitor_1.PerformanceMonitor.recordMetric({
name: 'batch_processor_completed',
value: this.stats.totalItems,
unit: 'count',
timestamp: new Date(),
tags: {
total_batches: this.stats.batchesCompleted.toString(),
peak_memory_mb: this.stats.peakMemoryUsageMB.toFixed(1),
total_time_ms: totalTime.toString()
}
});
}
}
exports.StreamingBatchProcessor = StreamingBatchProcessor;
/**
* Utility function to create a batch processor with common configuration
*/
function createBatchProcessor(config) {
return new StreamingBatchProcessor(config);
}
/**
* Memory-aware batch processing decorator
*/
function batchProcessed(batchSize = 50, memoryThresholdMB = 800) {
return function (target, propertyName, descriptor) {
const method = descriptor.value;
descriptor.value = async function (items, ...args) {
const processor = createBatchProcessor({
defaultBatchSize: batchSize,
memoryThresholdMB: memoryThresholdMB
});
// Create a batch processing function
const batchFunc = async (batch) => {
const results = [];
for (const item of batch) {
const itemStartTime = Date.now();
const memoryBefore = process.memoryUsage().heapUsed / 1024 / 1024;
try {
const result = await method.call(this, item.data, ...args);
results.push({
success: true,
result,
processingTimeMs: Date.now() - itemStartTime,
memoryUsageMB: process.memoryUsage().heapUsed / 1024 / 1024
});
}
catch (error) {
results.push({
success: false,
error: error,
processingTimeMs: Date.now() - itemStartTime,
memoryUsageMB: process.memoryUsage().heapUsed / 1024 / 1024
});
}
}
return results;
};
const { results } = await processor.processAll(items, batchFunc);
return results.filter(r => r.success).map(r => r.result);
};
};
}
exports.default = StreamingBatchProcessor;
//# sourceMappingURL=batch-processor.js.map