semantic-ds-toolkit/dist/src/optimization/batch-processor.d.ts

Performance-first semantic layer for modern data stacks - Stable Column Anchors & intelligent inference

import { ColumnData, StableColumnAnchor, ColumnFingerprint } from '../types/anchor.types';
interface BatchProcessingOptions {
    batchSize: number;
    maxConcurrency: number;
    retryAttempts: number;
    priorityWeight: number;
    memoryLimit?: number;
    useWorkerThreads?: boolean;
    streamingMode?: boolean;
}
interface ProcessingContext {
    batchId: string;
    batchIndex: number;
    totalBatches: number;
    startTime: number;
    memoryUsage: () => NodeJS.MemoryUsage;
}
type BatchProcessor<T, R> = (batch: T[], context: ProcessingContext) => Promise<R[]>;
export declare class HighPerformanceBatchProcessor {
    private activeJobs;
    private jobQueue;
    private workerPool;
    private maxWorkers;
    private processingStats;
    constructor(maxWorkers?: number);
    private initializeWorkerPool;
    processColumns(columns: ColumnData[], processor: BatchProcessor<ColumnData, any>, options?: Partial<BatchProcessingOptions>): Promise<any[]>;
    generateFingerprints(columns: ColumnData[], fingerprintFn: (column: ColumnData) => ColumnFingerprint): Promise<ColumnFingerprint[]>;
    private parallelFingerprinting;
    private processChunkOptimized;
    private optimizeColumnForProcessing;
    createAnchors(dataset: string, columns: ColumnData[], anchorFn: (dataset: string, column: ColumnData) => StableColumnAnchor): Promise<StableColumnAnchor[]>;
    private processBatch;
    private processStandardBatch;
    private processStreamingBatch;
    private executeBatchWithRetry;
    private createBatches;
    private createDataStream;
    private updateProcessingStats;
    benchmark(testData: ColumnData[], processor: BatchProcessor<ColumnData, any>, iterations?: number): Promise<{
        averageThroughput: number;
        peakThroughput: number;
        averageLatency: number;
        memoryEfficiency: number;
    }>;
    getProcessingStats(): {
        totalJobs: number;
        completedJobs: number;
        failedJobs: number;
        totalRowsProcessed: number;
        averageThroughput: number;
        peakThroughput: number;
    };
    cleanup(): void;
}
export declare const globalBatchProcessor: HighPerformanceBatchProcessor;
export {};
//# sourceMappingURL=batch-processor.d.ts.map