supa-seed/dist/core/batch-processor.d.ts

A constraint-aware, framework-agnostic database seeding framework with deep PostgreSQL business logic discovery and MakerKit integration support

/**
 * Streaming Batch Processor for Memory-Efficient Data Processing
 * FEAT-003: Memory Management & Schema Mapping Fixes
 *
 * This module provides streaming batch processing capabilities to prevent
 * out-of-memory errors when processing large datasets.
 */
export interface BatchProcessorConfig {
    /** Default batch size for processing (default: 50) */
    defaultBatchSize: number;
    /** Memory threshold in MB to trigger smaller batches (default: 800) */
    memoryThresholdMB: number;
    /** Minimum batch size when memory pressure is high (default: 10) */
    minBatchSize: number;
    /** Maximum batch size allowed (default: 200) */
    maxBatchSize: number;
    /** Whether to force garbage collection between batches (default: true) */
    forceGCBetweenBatches: boolean;
    /** Delay in ms between batches for memory pressure relief (default: 100) */
    batchDelayMs: number;
}
export interface BatchProcessingStats {
    totalItems: number;
    processedItems: number;
    batchesCompleted: number;
    currentBatchSize: number;
    averageMemoryUsageMB: number;
    peakMemoryUsageMB: number;
    totalProcessingTimeMs: number;
    averageBatchTimeMs: number;
}
export interface BatchItem<T> {
    data: T;
    index: number;
    metadata?: Record<string, any>;
}
export interface BatchResult<R> {
    success: boolean;
    result?: R;
    error?: Error;
    processingTimeMs: number;
    memoryUsageMB: number;
}
/**
 * Streaming batch processor for memory-efficient data processing
 */
export declare class StreamingBatchProcessor<T, R> {
    private config;
    private stats;
    private startTime;
    private memoryReadings;
    constructor(config?: Partial<BatchProcessorConfig>);
    /**
     * Process items in streaming batches with automatic memory management
     */
    processBatches(items: T[], processor: (batch: BatchItem<T>[]) => Promise<BatchResult<R>[]>): AsyncGenerator<BatchResult<R>[], void, unknown>;
    /**
     * Process all items and return aggregated results
     */
    processAll(items: T[], processor: (batch: BatchItem<T>[]) => Promise<BatchResult<R>[]>): Promise<{
        results: BatchResult<R>[];
        stats: BatchProcessingStats;
        recommendations: string[];
    }>;
    /**
     * Get current processing statistics
     */
    getStats(): BatchProcessingStats;
    /**
     * Get processing recommendations based on current performance
     */
    getRecommendations(): string[];
    /**
     * Create batches with dynamic sizing based on memory pressure
     */
    private createDynamicBatches;
    /**
     * Stream batches with memory monitoring
     */
    private streamBatches;
    /**
     * Calculate optimal batch size based on current memory conditions
     */
    private calculateOptimalBatchSize;
    /**
     * Perform cleanup between batches
     */
    private performBatchCleanup;
    /**
     * Initialize processing statistics
     */
    private initializeProcessing;
    /**
     * Update statistics after each batch
     */
    private updateBatchStats;
    /**
     * Finalize processing and log summary
     */
    private finalizeProcessing;
}
/**
 * Utility function to create a batch processor with common configuration
 */
export declare function createBatchProcessor<T, R>(config?: Partial<BatchProcessorConfig>): StreamingBatchProcessor<T, R>;
/**
 * Memory-aware batch processing decorator
 */
export declare function batchProcessed(batchSize?: number, memoryThresholdMB?: number): (target: any, propertyName: string, descriptor: PropertyDescriptor) => void;
export default StreamingBatchProcessor;
//# sourceMappingURL=batch-processor.d.ts.map