datapilot-cli/dist/cli/progress-orchestrator.js

Enterprise-grade streaming multi-format data analysis with comprehensive statistical insights and intelligent relationship detection - supports CSV, JSON, Excel, TSV, Parquet - memory-efficient, cross-platform

"use strict";
/**
 * Enhanced Progress Orchestration System
 * Manages complex progress reporting across multiple analyzers with aggregation and callbacks
 */
Object.defineProperty(exports, "__esModule", { value: true });
exports.ProgressOrchestrator = void 0;
exports.createProgressOrchestrator = createProgressOrchestrator;
const logger_1 = require("../utils/logger");
/**
 * Enhanced progress orchestrator with hierarchical progress tracking
 */
class ProgressOrchestrator {
    callbacks;
    phases = new Map();
    currentPhase;
    currentSection;
    sectionProgress = 0;
    phaseProgress = 0;
    overallProgress = 0;
    startTime;
    phaseStartTime = 0;
    sectionStartTime = 0;
    completedSections = 0;
    totalSections = 0;
    resourceMetrics;
    context;
    constructor(callbacks = {}, options = {}, context = {}) {
        this.callbacks = callbacks;
        this.context = context;
        this.startTime = Date.now();
        this.resourceMetrics = {
            memoryUsage: 0,
            processingTime: 0,
            rowsProcessed: 0,
            sectionsCompleted: 0,
        };
        this.setupDefaultPhases();
        this.setupProgressLogging(options);
    }
    /**
     * Setup default phase configurations for DataPilot sections
     */
    setupDefaultPhases() {
        const defaultPhases = [
            {
                name: 'initialization',
                weight: 5,
                estimatedDuration: 2000,
                subPhases: [
                    { name: 'validation', weight: 40 },
                    { name: 'setup', weight: 60 },
                ],
            },
            {
                name: 'section1',
                weight: 15,
                estimatedDuration: 10000,
                subPhases: [
                    { name: 'metadata_collection', weight: 30 },
                    { name: 'structural_analysis', weight: 40 },
                    { name: 'environment_profiling', weight: 30 },
                ],
            },
            {
                name: 'section2',
                weight: 20,
                estimatedDuration: 15000,
                subPhases: [
                    { name: 'completeness_analysis', weight: 25 },
                    { name: 'uniqueness_analysis', weight: 25 },
                    { name: 'validity_analysis', weight: 25 },
                    { name: 'quality_scoring', weight: 25 },
                ],
            },
            {
                name: 'section3',
                weight: 25,
                estimatedDuration: 20000,
                subPhases: [
                    { name: 'univariate_analysis', weight: 40 },
                    { name: 'bivariate_analysis', weight: 35 },
                    { name: 'correlation_analysis', weight: 25 },
                ],
            },
            {
                name: 'section4',
                weight: 15,
                estimatedDuration: 8000,
                subPhases: [
                    { name: 'chart_selection', weight: 50 },
                    { name: 'accessibility_optimization', weight: 30 },
                    { name: 'recommendation_generation', weight: 20 },
                ],
            },
            {
                name: 'section5',
                weight: 10,
                estimatedDuration: 12000,
                subPhases: [
                    { name: 'schema_analysis', weight: 40 },
                    { name: 'transformation_analysis', weight: 35 },
                    { name: 'ml_readiness', weight: 25 },
                ],
            },
            {
                name: 'section6',
                weight: 10,
                estimatedDuration: 10000,
                subPhases: [
                    { name: 'task_identification', weight: 30 },
                    { name: 'algorithm_selection', weight: 40 },
                    { name: 'workflow_generation', weight: 30 },
                ],
            },
        ];
        defaultPhases.forEach((phase) => this.phases.set(phase.name, phase));
    }
    /**
     * Setup progress logging based on CLI options
     */
    setupProgressLogging(options) {
        if (options.verbose) {
            this.callbacks.onProgress = (state) => {
                logger_1.logger.debug(`Progress: ${state.phase} - ${state.progress}% - ${state.message}`, {
                    ...this.context,
                    progress: state.progress,
                    phase: state.phase,
                });
            };
        }
        if (!options.quiet && options.showProgress !== false) {
            // Add default console progress if no custom callback
            if (!this.callbacks.onProgress) {
                this.callbacks.onProgress = (state) => {
                    const timeStr = this.formatDuration(state.timeElapsed);
                    const etaStr = state.estimatedTimeRemaining
                        ? ` (ETA: ${this.formatDuration(state.estimatedTimeRemaining)})`
                        : '';
                    console.log(`  ${state.progress}% - ${state.message} [${timeStr}]${etaStr}`);
                };
            }
        }
    }
    /**
     * Start monitoring for resource usage
     */
    startMonitoring(context) {
        this.context = { ...this.context, ...context };
        this.startTime = Date.now();
        logger_1.logger.debug('Started progress monitoring', this.context);
    }
    /**
     * Set the total number of sections for progress calculation
     */
    setTotalSections(total) {
        this.totalSections = total;
        logger_1.logger.debug(`Set total sections to ${total}`, this.context);
    }
    /**
     * Start a new phase with optional subphase tracking
     */
    startPhase(phaseName, message) {
        this.currentPhase = phaseName;
        this.phaseStartTime = Date.now();
        this.phaseProgress = 0;
        const phase = this.phases.get(phaseName);
        if (phase) {
            logger_1.logger.debug(`Starting phase ${phaseName} (weight: ${phase.weight}%, estimated: ${phase.estimatedDuration}ms)`, { ...this.context, phase: phaseName });
        }
        this.callbacks.onPhaseStart?.(phaseName, message);
        this.updateOverallProgress();
    }
    /**
     * Start a new section within the current phase
     */
    startSection(sectionName, message) {
        this.currentSection = sectionName;
        this.sectionStartTime = Date.now();
        this.sectionProgress = 0;
        logger_1.logger.debug(`Starting section ${sectionName}`, {
            ...this.context,
            phase: this.currentPhase,
            section: sectionName,
        });
        this.callbacks.onPhaseStart?.(sectionName, message);
        this.updateOverallProgress();
    }
    /**
     * Update progress within the current phase/section
     */
    updateProgress(state) {
        if (state.progress !== undefined) {
            this.sectionProgress = Math.max(0, Math.min(100, state.progress));
        }
        // Update resource metrics if provided
        if (state.timeElapsed) {
            this.resourceMetrics.processingTime = state.timeElapsed;
        }
        this.updateOverallProgress();
        const fullState = {
            phase: state.phase || this.currentPhase || 'unknown',
            progress: this.sectionProgress,
            message: state.message || 'Processing...',
            timeElapsed: state.timeElapsed || Date.now() - this.startTime,
            estimatedTimeRemaining: state.estimatedTimeRemaining || this.calculateETA(),
        };
        this.callbacks.onProgress?.(fullState);
    }
    /**
     * Update resource metrics
     */
    updateMetrics(metrics) {
        this.resourceMetrics = { ...this.resourceMetrics, ...metrics };
        if (metrics.rowsProcessed !== undefined) {
            logger_1.logger.debug(`Processed ${metrics.rowsProcessed} rows`, {
                ...this.context,
                rowsProcessed: metrics.rowsProcessed,
            });
        }
    }
    /**
     * Complete the current phase
     */
    completePhase(message) {
        const duration = Date.now() - this.phaseStartTime;
        if (this.currentPhase) {
            logger_1.logger.info(`Completed phase ${this.currentPhase} in ${duration}ms`, {
                ...this.context,
                phase: this.currentPhase,
                duration,
            });
        }
        this.callbacks.onPhaseComplete?.(message, duration);
        this.updateOverallProgress();
    }
    /**
     * Complete the current section
     */
    completeSection(message) {
        const duration = Date.now() - this.sectionStartTime;
        this.completedSections++;
        this.resourceMetrics.sectionsCompleted = this.completedSections;
        if (this.currentSection) {
            logger_1.logger.info(`Completed section ${this.currentSection} in ${duration}ms`, {
                ...this.context,
                section: this.currentSection,
                duration,
                completedSections: this.completedSections,
            });
        }
        this.callbacks.onPhaseComplete?.(message, duration);
        this.updateOverallProgress();
    }
    /**
     * Report an error in the current phase
     */
    reportError(message, error) {
        logger_1.logger.error(`Error in ${this.currentPhase || 'unknown phase'}: ${message}`, this.context, error);
        this.callbacks.onError?.(message);
    }
    /**
     * Report a warning
     */
    reportWarning(message) {
        logger_1.logger.warn(`Warning in ${this.currentPhase || 'unknown phase'}: ${message}`, this.context);
        this.callbacks.onWarning?.(message);
    }
    /**
     * Calculate overall progress based on phase weights and section completion
     */
    updateOverallProgress() {
        let totalProgress = 0;
        // Add progress from completed phases
        for (const [phaseName, config] of this.phases) {
            if (phaseName === this.currentPhase) {
                // Current phase contributes partial progress
                totalProgress += (config.weight * this.sectionProgress) / 100;
                break;
            }
            else if (this.isPhaseCompleted(phaseName)) {
                // Completed phases contribute full weight
                totalProgress += config.weight;
            }
        }
        this.overallProgress = Math.max(0, Math.min(100, totalProgress));
    }
    /**
     * Check if a phase has been completed
     */
    isPhaseCompleted(phaseName) {
        // This is a simplified check - in practice, we'd track completed phases
        const phaseOrder = [
            'initialization',
            'section1',
            'section2',
            'section3',
            'section4',
            'section5',
            'section6',
        ];
        const currentIndex = phaseOrder.indexOf(this.currentPhase || '');
        const checkIndex = phaseOrder.indexOf(phaseName);
        return checkIndex >= 0 && checkIndex < currentIndex;
    }
    /**
     * Calculate estimated time remaining
     */
    calculateETA() {
        if (this.overallProgress <= 0)
            return undefined;
        const timeElapsed = Date.now() - this.startTime;
        const estimatedTotal = (timeElapsed / this.overallProgress) * 100;
        const remaining = estimatedTotal - timeElapsed;
        return Math.max(0, remaining);
    }
    /**
     * Get current overall progress state
     */
    getOverallState() {
        return {
            currentPhase: this.currentPhase || 'unknown',
            currentSection: this.currentSection || 'unknown',
            overallProgress: this.overallProgress,
            sectionProgress: this.sectionProgress,
            phaseProgress: this.phaseProgress,
            timeElapsed: Date.now() - this.startTime,
            estimatedTimeRemaining: this.calculateETA(),
            sectionsCompleted: this.completedSections,
            totalSections: this.totalSections,
            memoryUsage: this.resourceMetrics.memoryUsage,
            rowsProcessed: this.resourceMetrics.rowsProcessed,
        };
    }
    /**
     * Check if resource thresholds are exceeded
     */
    checkThresholds() {
        const warnings = [];
        let exceeded = false;
        // Check memory usage (if available)
        if (this.resourceMetrics.memoryUsage > 0) {
            const memoryMB = this.resourceMetrics.memoryUsage / (1024 * 1024);
            if (memoryMB > 1024) {
                // 1GB threshold
                warnings.push(`High memory usage: ${memoryMB.toFixed(1)}MB`);
                exceeded = true;
            }
        }
        // Check processing time
        const timeElapsed = Date.now() - this.startTime;
        if (timeElapsed > 300000) {
            // 5 minutes
            warnings.push(`Long processing time: ${this.formatDuration(timeElapsed)}`);
        }
        return { exceeded, warnings };
    }
    /**
     * Stop monitoring and return final metrics
     */
    stopMonitoring() {
        const finalMetrics = {
            ...this.resourceMetrics,
            processingTime: Date.now() - this.startTime,
            sectionsCompleted: this.completedSections,
        };
        logger_1.logger.debug('Stopped progress monitoring', {
            ...this.context,
            finalMetrics,
        });
        return finalMetrics;
    }
    /**
     * Format duration in human-readable format
     */
    formatDuration(milliseconds) {
        if (milliseconds < 1000) {
            return `${Math.round(milliseconds)}ms`;
        }
        else if (milliseconds < 60000) {
            return `${(milliseconds / 1000).toFixed(1)}s`;
        }
        else {
            const minutes = Math.floor(milliseconds / 60000);
            const seconds = Math.round((milliseconds % 60000) / 1000);
            return `${minutes}m ${seconds}s`;
        }
    }
    /**
     * Create a scoped progress callback for a specific section
     */
    createSectionCallback(sectionName, sectionWeight = 100) {
        return (state) => {
            // Scale the progress based on section weight
            const scaledProgress = (state.progress * sectionWeight) / 100;
            this.updateProgress({
                ...state,
                progress: scaledProgress,
                phase: sectionName,
            });
        };
    }
    /**
     * Create a progress callback that aggregates multiple sub-operations
     */
    createAggregatedCallback(subOperations) {
        const operationProgress = new Map();
        return (operationName, progress) => {
            operationProgress.set(operationName, progress);
            // Calculate weighted average
            let totalWeight = 0;
            let weightedSum = 0;
            for (const op of subOperations) {
                const opProgress = operationProgress.get(op.name) || 0;
                totalWeight += op.weight;
                weightedSum += opProgress * op.weight;
            }
            const aggregatedProgress = totalWeight > 0 ? weightedSum / totalWeight : 0;
            this.updateProgress({
                progress: aggregatedProgress,
                message: `${operationName}: ${progress}%`,
                timeElapsed: Date.now() - this.startTime,
            });
        };
    }
}
exports.ProgressOrchestrator = ProgressOrchestrator;
/**
 * Factory function to create a progress orchestrator with sensible defaults
 */
function createProgressOrchestrator(options, context = {}) {
    const callbacks = {};
    // Setup callbacks based on options
    if (!options.quiet) {
        callbacks.onPhaseStart = (phase, message) => {
            if (options.verbose) {
                console.log(`\n🔄 ${phase}: ${message}`);
            }
        };
        callbacks.onPhaseComplete = (message, timeElapsed) => {
            const duration = timeElapsed < 1000 ? `${Math.round(timeElapsed)}ms` : `${(timeElapsed / 1000).toFixed(1)}s`;
            console.log(`✅ ${message} [${duration}]`);
        };
        callbacks.onError = (message) => {
            console.error(`❌ ${message}`);
        };
        callbacks.onWarning = (message) => {
            console.warn(`⚠️  ${message}`);
        };
    }
    return new ProgressOrchestrator(callbacks, options, context);
}
//# sourceMappingURL=progress-orchestrator.js.map