datapilot-cli/dist/performance/adaptive-streamer.d.ts

Enterprise-grade streaming multi-format data analysis with comprehensive statistical insights and intelligent relationship detection - supports CSV, JSON, Excel, TSV, Parquet - memory-efficient, cross-platform

/**
 * Adaptive Streaming Engine
 * Intelligent chunk sizing and streaming optimization for large files
 */
import { EventEmitter } from 'events';
export interface AdaptiveStreamingOptions {
    initialChunkSize?: number;
    minChunkSize?: number;
    maxChunkSize?: number;
    adaptationInterval?: number;
    performanceTargetMBps?: number;
    memoryPressureThreshold?: number;
    enableProgressiveLoading?: boolean;
    maxConcurrentChunks?: number;
}
export interface StreamingMetrics {
    bytesProcessed: number;
    chunksProcessed: number;
    averageChunkSize: number;
    processingRate: number;
    memoryEfficiency: number;
    adaptationCount: number;
    totalTime: number;
}
interface StreamingSession {
    id: string;
    filePath: string;
    fileSize: number;
    totalChunks: number;
    processedChunks: number;
    metrics: StreamingMetrics;
    startTime: number;
    adaptiveChunkSize: number;
}
/**
 * Adaptive streaming engine with intelligent chunk sizing
 */
export declare class AdaptiveStreamer extends EventEmitter {
    private options;
    private memoryOptimizer;
    private activeSessions;
    private performanceHistory;
    private adaptationHistory;
    constructor(options?: AdaptiveStreamingOptions);
    /**
     * Create a new streaming session
     */
    createSession(filePath: string): Promise<string>;
    /**
     * Process file with adaptive streaming
     */
    streamFile(sessionId: string, processor: (chunk: Buffer, metadata: any) => Promise<any>): AsyncGenerator<any, void, unknown>;
    /**
     * Calculate optimal chunk size based on current conditions
     */
    private calculateOptimalChunkSize;
    /**
     * Update session metrics
     */
    private updateSessionMetrics;
    /**
     * Adapt chunk size based on performance
     */
    private adaptChunkSize;
    /**
     * Handle memory pressure by reducing chunk sizes
     */
    private handleMemoryPressure;
    /**
     * Handle critical memory by aggressively reducing chunk sizes
     */
    private handleCriticalMemory;
    /**
     * Finalize session and cleanup
     */
    private finalizeSession;
    /**
     * Get session statistics
     */
    getSessionStats(sessionId: string): StreamingSession | null;
    /**
     * Get overall streaming statistics
     */
    getOverallStats(): {
        activeSessions: number;
        totalBytesProcessed: number;
        averageProcessingRate: number;
        performanceHistory: number[];
        adaptationHistory: number[];
        memoryStats: any;
        options: Required<AdaptiveStreamingOptions>;
    };
    /**
     * Format bytes for human-readable output
     */
    private formatBytes;
    /**
     * Cleanup all sessions and resources
     */
    cleanup(): void;
}
/**
 * Get or create global adaptive streamer
 */
export declare function getGlobalAdaptiveStreamer(options?: AdaptiveStreamingOptions): AdaptiveStreamer;
/**
 * Shutdown global adaptive streamer
 */
export declare function shutdownGlobalAdaptiveStreamer(): void;
export {};
//# sourceMappingURL=adaptive-streamer.d.ts.map