ai-debug-local-mcp/dist/flutter-quantum-debugger.js

🎯 ENHANCED AI GUIDANCE v4.1.2: Dramatically improved tool descriptions help AI users choose the right tools instead of 'close enough' options. Ultra-fast keyboard automation (10x speed), universal recording, multi-ecosystem debugging support, and compreh

/**
 * Flutter Quantum Debugger
 * The ultimate Flutter web debugging solution that actually works
 */
import { FlutterSemanticAnalyzer } from './flutter-semantic-analyzer.js';
import { FlutterStructureMapper } from './flutter-structure-mapper.js';
import { FlutterInteractionPredictor } from './flutter-interaction-predictor.js';
import { FlutterCanvasInspector } from './flutter-canvas-inspector.js';
import { BrowserPermissionHandler } from './browser-permission-handler.js';
import { FlutterComprehensiveDetector } from './flutter-comprehensive-detector.js';
import { FlutterAccessibilityEnabler } from './flutter-accessibility-enabler.js';
import { FlutterClickHandler } from './flutter-click-handler.js';
export class FlutterQuantumDebugger {
    sessions = new Map();
    /**
     * Initialize a quantum debug session
     */
    async initialize(page, sessionId) {
        // console.log('🚀 Initializing Flutter Quantum Debugger...');
        // Set up permission handlers
        await this.setupPermissionHandlers(page);
        // Wait for Flutter to load
        await this.waitForFlutter(page);
        // CRITICAL: Enable Flutter accessibility for proper semantic tree
        // console.log('🔓 Enabling Flutter accessibility for semantic tree...');
        const accessibilityEnabled = await FlutterAccessibilityEnabler.enableAccessibility(page);
        if (accessibilityEnabled) {
            // console.log('✅ Accessibility enabled successfully!');
            // Wait for accessibility tree to populate
            await FlutterAccessibilityEnabler.waitForAccessibilityTree(page);
        }
        else {
            // console.log('⚠️ Could not enable accessibility - semantic detection may be limited');
        }
        // Try to enable Flutter debug features
        // console.log('🔧 Enabling Flutter debug features...');
        await FlutterCanvasInspector.enableDebugPaint(page);
        // Try to enable semantics if not already enabled
        const semanticsEnabled = await FlutterCanvasInspector.enableSemantics(page);
        if (semanticsEnabled) {
            // console.log('✅ Semantics enabled successfully');
            // Wait for semantics to update
            await page.waitForTimeout(1000);
        }
        // Get debug info
        const debugInfo = await FlutterCanvasInspector.getDebugInfo(page);
        // console.log('📊 Flutter debug info:', debugInfo);
        // Inject debug overlay for better visibility
        await FlutterCanvasInspector.injectDebugOverlay(page);
        // Hook into rendering pipeline for better introspection
        await FlutterCanvasInspector.hookRenderingPipeline(page);
        // Analyze semantic structure
        const semanticNodes = await FlutterSemanticAnalyzer.analyzeSemanticTree(page);
        // console.log(`📊 Found ${semanticNodes.length} semantic nodes`);
        // Use comprehensive detector to find ALL elements
        // console.log('🔍 Running comprehensive element detection...');
        const detectedElements = await FlutterComprehensiveDetector.detectAllElements(page);
        // console.log(`✅ Detected ${detectedElements.length} total elements`);
        // Create debug overlay to visualize detected elements
        await FlutterComprehensiveDetector.createDebugOverlay(page, detectedElements);
        // console.log('🎨 Debug overlay created - all elements marked');
        // Map to UI structure
        const structure = FlutterStructureMapper.mapToUIStructure(semanticNodes, page.url());
        // Log structure summary
        const summary = FlutterStructureMapper.generateSummary(structure);
        // console.log('\n' + summary);
        // Also log detected elements summary
        // console.log('\n📋 Detected Elements by Type:');
        const typeCount = {};
        detectedElements.forEach(el => {
            typeCount[el.type] = (typeCount[el.type] || 0) + 1;
        });
        Object.entries(typeCount).forEach(([type, count]) => {
            // console.log(`  - ${type}: ${count} elements`);
        });
        // console.log('\n🎯 Clickable Elements:');
        detectedElements
            .filter(el => el.clickable && el.label)
            .slice(0, 10)
            .forEach((el, i) => {
            // console.log(`  ${i + 1}. "${el.label}" at (${Math.round(el.bounds.x)}, ${Math.round(el.bounds.y)}) - ${el.type} (${Math.round(el.confidence * 100)}% confidence)`);
        });
        const session = {
            id: sessionId,
            page,
            structure,
            semanticNodes,
            detectedElements,
            interactionHistory: []
        };
        this.sessions.set(sessionId, session);
        return session;
    }
    /**
     * Natural language interaction interface
     */
    async interact(sessionId, command) {
        const session = this.sessions.get(sessionId);
        if (!session) {
            return {
                success: false,
                error: 'Session not found'
            };
        }
        // console.log(`\n🎯 Processing command: "${command}"`);
        // Parse the command to extract intent
        const intent = this.parseCommand(command);
        switch (intent.action) {
            case 'click':
                return await this.handleClick(session, intent.target);
            case 'type':
                return await this.handleType(session, intent.target, intent.value);
            case 'select':
                return await this.handleSelect(session, intent.target, intent.value);
            case 'toggle':
                return await this.handleToggle(session, intent.target);
            case 'scroll':
                return await this.handleScroll(session, intent.target, intent.direction);
            case 'set':
                return await this.handleSet(session, intent.target, intent.value);
            case 'find':
                return await this.handleFind(session, intent.target);
            case 'inspect':
                return await this.handleInspect(session);
            case 'focus':
                return await this.handleFocus(session, intent.target);
            case 'clear':
                return await this.handleClear(session, intent.target);
            default:
                // Try to match the action to available element actions
                return await this.handleGenericAction(session, intent.action, intent.target, intent.value);
        }
    }
    /**
     * Handle click interactions
     */
    async handleClick(session, target) {
        // First try to find element using comprehensive detector
        const element = FlutterComprehensiveDetector.findElementByText(session.detectedElements, target);
        if (element) {
            // console.log(`🎯 Found element "${element.label}" using ${element.type} detection`);
            // console.log(`   Location: (${Math.round(element.bounds.x + element.bounds.width/2)}, ${Math.round(element.bounds.y + element.bounds.height/2)})`);
            // console.log(`   Confidence: ${Math.round(element.confidence * 100)}%`);
            try {
                // Capture before state for verification
                const beforeState = {
                    url: session.page.url(),
                    title: await session.page.title()
                };
                // Use our enhanced Flutter click handler
                const clickSuccess = await FlutterClickHandler.clickElement(session.page, element, { verbose: true });
                if (!clickSuccess) {
                    throw new Error('Click handler returned false');
                }
                // Verify the click had an effect
                const hasEffect = await FlutterClickHandler.verifyClickSuccess(session.page, beforeState, { timeout: 2000 });
                if (hasEffect) {
                    // console.log('✅ Click caused a change in the page!');
                }
                else {
                    // console.log('⚠️ Click executed but no immediate changes detected');
                }
                // Wait for any animations/transitions
                await session.page.waitForTimeout(1000);
                // Re-detect all elements
                session.detectedElements = await FlutterComprehensiveDetector.detectAllElements(session.page);
                // Update overlay
                await FlutterComprehensiveDetector.createDebugOverlay(session.page, session.detectedElements);
                // Record interaction
                session.interactionHistory.push({
                    timestamp: new Date(),
                    command: `click ${target}`,
                    method: `comprehensive-${element.type}`,
                    success: true,
                    result: { element }
                });
                // console.log('✅ Click successful!');
                // console.log(`📊 New element count: ${session.detectedElements.length}`);
                return {
                    success: true,
                    method: `comprehensive-${element.type}`
                };
            }
            catch (error) {
                console.error('Click failed:', error);
            }
        }
        // Fallback to original strategy-based approach
        // console.log('⚠️ Element not found by comprehensive detector, trying strategies...');
        // Generate interaction strategies
        const strategies = await FlutterInteractionPredictor.generateStrategies(session.page, target, session.structure);
        // console.log(`🔍 Generated ${strategies.length} interaction strategies`);
        // Execute strategies
        const result = await FlutterInteractionPredictor.executeStrategies(strategies);
        // Record interaction
        session.interactionHistory.push({
            timestamp: new Date(),
            command: `click ${target}`,
            method: result.method || 'none',
            success: result.success,
            result
        });
        if (result.success) {
            // Re-analyze structure after successful interaction
            await session.page.waitForTimeout(1000);
            session.detectedElements = await FlutterComprehensiveDetector.detectAllElements(session.page);
            await FlutterComprehensiveDetector.createDebugOverlay(session.page, session.detectedElements);
            const newSemanticNodes = await FlutterSemanticAnalyzer.analyzeSemanticTree(session.page);
            const newStructure = FlutterStructureMapper.mapToUIStructure(newSemanticNodes, session.page.url());
            session.structure = newStructure;
            session.semanticNodes = newSemanticNodes;
            // console.log('✅ Interaction successful!');
            // console.log('📊 New page structure:');
            // console.log(FlutterStructureMapper.generateSummary(newStructure));
            return {
                success: true,
                method: result.method,
                newStructure
            };
        }
        return {
            success: false,
            error: result.error
        };
    }
    /**
     * Handle type interactions
     */
    async handleType(session, target, value) {
        if (!value) {
            return {
                success: false,
                error: 'No value provided for type command'
            };
        }
        // First click on the target field
        const clickResult = await this.handleClick(session, target);
        if (!clickResult.success) {
            return clickResult;
        }
        // Then type the value
        await session.page.keyboard.type(value);
        return {
            success: true,
            method: 'keyboard-type'
        };
    }
    /**
     * Handle find interactions
     */
    async handleFind(session, target) {
        const matches = FlutterSemanticAnalyzer.findNodesByText(session.semanticNodes, target);
        if (matches.length > 0) {
            // console.log(`✅ Found ${matches.length} elements matching "${target}":`);
            matches.forEach(match => {
                // console.log(`  - ${match.label} at (${match.bounds.x}, ${match.bounds.y})`);
            });
            return {
                success: true,
                method: 'semantic-search'
            };
        }
        return {
            success: false,
            error: `No elements found matching "${target}"`
        };
    }
    /**
     * Handle inspect command
     */
    async handleInspect(session) {
        const summary = FlutterStructureMapper.generateSummary(session.structure);
        // console.log('\n' + summary);
        return {
            success: true,
            method: 'structure-inspection'
        };
    }
    /**
     * Handle select action (for dropdowns, radio buttons, etc)
     */
    async handleSelect(session, target, value) {
        // First find the element
        const element = session.structure.interactables.find(el => el.label.toLowerCase().includes(target.toLowerCase()) &&
            el.actions.includes('select'));
        if (!element) {
            return { success: false, error: `No selectable element found: ${target}` };
        }
        // Click to select
        return await this.handleClick(session, target);
    }
    /**
     * Handle toggle action (checkboxes, switches)
     */
    async handleToggle(session, target) {
        // Find toggleable element
        const element = session.semanticNodes.find(node => node.label.toLowerCase().includes(target.toLowerCase()) &&
            (node.isToggleable || node.elementType === 'checkbox' || node.elementType === 'toggle'));
        if (!element) {
            return { success: false, error: `No toggleable element found: ${target}` };
        }
        // Click to toggle
        return await this.handleClick(session, target);
    }
    /**
     * Handle scroll action
     */
    async handleScroll(session, target, direction) {
        const scrollAmount = 300; // pixels
        try {
            if (target && target !== 'page') {
                // Find specific scrollable element
                const element = session.semanticNodes.find(node => node.label.toLowerCase().includes(target.toLowerCase()) &&
                    node.isScrollable);
                if (element) {
                    await session.page.evaluate(({ x, y, amount, dir }) => {
                        const el = document.elementFromPoint(x, y);
                        if (el) {
                            if (dir === 'up')
                                el.scrollTop -= amount;
                            else if (dir === 'down')
                                el.scrollTop += amount;
                            else if (dir === 'left')
                                el.scrollLeft -= amount;
                            else if (dir === 'right')
                                el.scrollLeft += amount;
                        }
                    }, {
                        x: element.bounds.x + element.bounds.width / 2,
                        y: element.bounds.y + element.bounds.height / 2,
                        amount: scrollAmount,
                        dir: direction || 'down'
                    });
                }
            }
            else {
                // Scroll the page
                if (direction === 'up') {
                    await session.page.mouse.wheel(0, -scrollAmount);
                }
                else {
                    await session.page.mouse.wheel(0, scrollAmount);
                }
            }
            return { success: true, method: 'scroll' };
        }
        catch (e) {
            return { success: false, error: `Scroll failed: ${e}` };
        }
    }
    /**
     * Handle set action (for inputs with specific values)
     */
    async handleSet(session, target, value) {
        if (!value) {
            return { success: false, error: 'No value provided for set action' };
        }
        // Find the input element
        const element = session.semanticNodes.find(node => node.label.toLowerCase().includes(target.toLowerCase()) &&
            (node.isTextField || node.elementType.includes('input')));
        if (!element) {
            return { success: false, error: `No input element found: ${target}` };
        }
        // Click on the element first
        await session.page.mouse.click(element.bounds.x + element.bounds.width / 2, element.bounds.y + element.bounds.height / 2);
        // Clear existing value
        await session.page.keyboard.press('Control+A');
        await session.page.keyboard.press('Backspace');
        // Type new value
        await session.page.keyboard.type(value);
        return { success: true, method: 'set-value' };
    }
    /**
     * Handle focus action
     */
    async handleFocus(session, target) {
        const element = session.semanticNodes.find(node => node.label.toLowerCase().includes(target.toLowerCase()) &&
            node.actions.includes('focus'));
        if (!element) {
            return { success: false, error: `No focusable element found: ${target}` };
        }
        // Click to focus
        await session.page.mouse.click(element.bounds.x + element.bounds.width / 2, element.bounds.y + element.bounds.height / 2);
        return { success: true, method: 'focus' };
    }
    /**
     * Handle clear action
     */
    async handleClear(session, target) {
        // Find and focus the element
        const focusResult = await this.handleFocus(session, target);
        if (!focusResult.success)
            return focusResult;
        // Clear the value
        await session.page.keyboard.press('Control+A');
        await session.page.keyboard.press('Backspace');
        return { success: true, method: 'clear' };
    }
    /**
     * Handle generic action based on element capabilities
     */
    async handleGenericAction(session, action, target, value) {
        // Find element that supports this action
        const element = session.semanticNodes.find(node => node.label.toLowerCase().includes(target.toLowerCase()) &&
            node.actions.includes(action));
        if (!element) {
            return {
                success: false,
                error: `No element found that supports action "${action}" for target "${target}"`
            };
        }
        // Try to perform the action based on its type
        switch (action) {
            case 'increment':
            case 'increase':
                await session.page.keyboard.press('ArrowUp');
                break;
            case 'decrement':
            case 'decrease':
                await session.page.keyboard.press('ArrowDown');
                break;
            case 'expand':
                await session.page.keyboard.press('Enter');
                break;
            case 'collapse':
                await session.page.keyboard.press('Escape');
                break;
            default:
                // Fall back to click for unknown actions
                return await this.handleClick(session, target);
        }
        return { success: true, method: action };
    }
    /**
     * Parse natural language commands
     */
    parseCommand(command) {
        const lower = command.toLowerCase();
        // Click variations
        if (lower.startsWith('click') || lower.startsWith('tap') || lower.startsWith('press')) {
            const match = command.match(/(?:click|tap|press)\s+(?:on\s+)?(.+)/i);
            return {
                action: 'click',
                target: match?.[1]?.trim() || command
            };
        }
        // Type/enter text variations
        if (lower.includes('type') || lower.includes('enter') || lower.includes('write')) {
            const match = command.match(/(?:type|enter|write)\s+["']?(.+?)["']?\s+(?:in|into)\s+(.+)/i);
            if (match) {
                return {
                    action: 'type',
                    target: match[2].trim(),
                    value: match[1].trim()
                };
            }
        }
        // Select variations
        if (lower.includes('select') || lower.includes('choose') || lower.includes('pick')) {
            const match = command.match(/(?:select|choose|pick)\s+["']?(.+?)["']?\s+(?:from|in)\s+(.+)/i);
            if (match) {
                return {
                    action: 'select',
                    target: match[2].trim(),
                    value: match[1].trim()
                };
            }
            // Simple select
            const simpleMatch = command.match(/(?:select|choose|pick)\s+(.+)/i);
            return {
                action: 'select',
                target: simpleMatch?.[1]?.trim() || ''
            };
        }
        // Toggle/check variations
        if (lower.includes('toggle') || lower.includes('check') || lower.includes('uncheck')) {
            const match = command.match(/(?:toggle|check|uncheck)\s+(.+)/i);
            return {
                action: 'toggle',
                target: match?.[1]?.trim() || ''
            };
        }
        // Scroll variations
        if (lower.includes('scroll')) {
            const match = command.match(/scroll\s+(up|down|left|right)?\s*(?:on|in)?\s*(.+)?/i);
            return {
                action: 'scroll',
                target: match?.[2]?.trim() || 'page',
                direction: match?.[1]?.toLowerCase()
            };
        }
        // Set value variations
        if (lower.includes('set') || lower.includes('change')) {
            const match = command.match(/(?:set|change)\s+(.+?)\s+(?:to|value)\s+["']?(.+?)["']?/i);
            if (match) {
                return {
                    action: 'set',
                    target: match[1].trim(),
                    value: match[2].trim()
                };
            }
        }
        // Clear variations
        if (lower.includes('clear') || lower.includes('empty')) {
            const match = command.match(/(?:clear|empty)\s+(.+)/i);
            return {
                action: 'clear',
                target: match?.[1]?.trim() || ''
            };
        }
        // Focus variations
        if (lower.includes('focus')) {
            const match = command.match(/focus\s+(?:on\s+)?(.+)/i);
            return {
                action: 'focus',
                target: match?.[1]?.trim() || ''
            };
        }
        // Find variations
        if (lower.startsWith('find') || lower.startsWith('search') || lower.startsWith('locate')) {
            const match = command.match(/(?:find|search|locate)\s+(.+)/i);
            return {
                action: 'find',
                target: match?.[1]?.trim() || ''
            };
        }
        // Inspect variations
        if (lower.includes('inspect') || lower.includes('show') || lower.includes('what')) {
            return {
                action: 'inspect',
                target: ''
            };
        }
        // Submit form
        if (lower.includes('submit')) {
            return {
                action: 'click',
                target: 'submit'
            };
        }
        // Default to click if no action specified
        return {
            action: 'click',
            target: command
        };
    }
    /**
     * Wait for Flutter to be ready
     */
    async waitForFlutter(page) {
        await page.waitForLoadState('domcontentloaded');
        // Wait for Flutter-specific elements
        await page.waitForFunction(() => {
            return document.querySelector('flutter-view') !== null ||
                document.querySelector('flt-glass-pane') !== null ||
                document.querySelectorAll('canvas').length > 0;
        }, { timeout: 10000 });
        // Wait for Flutter to finish rendering by checking for:
        // 1. Canvas elements are actually visible (have dimensions)
        // 2. Semantic nodes are in valid positions (not at -1, -1)
        await page.waitForFunction(() => {
            const canvases = document.querySelectorAll('canvas');
            const hasVisibleCanvas = Array.from(canvases).some(canvas => {
                const rect = canvas.getBoundingClientRect();
                return rect.width > 0 && rect.height > 0;
            });
            // Check if semantic nodes have valid positions
            const semanticNodes = document.querySelectorAll('[role], [aria-label], flt-semantics');
            const hasValidSemantics = Array.from(semanticNodes).some(node => {
                const rect = node.getBoundingClientRect();
                return rect.x >= 0 && rect.y >= 0 && rect.width > 0 && rect.height > 0;
            });
            return hasVisibleCanvas || hasValidSemantics;
        }, { timeout: 10000 });
        // Additional wait for Flutter's post-frame callbacks to complete
        await page.waitForTimeout(2000);
    }
    /**
     * Get session by ID
     */
    getSession(sessionId) {
        return this.sessions.get(sessionId);
    }
    /**
     * Get interaction history
     */
    getHistory(sessionId) {
        const session = this.sessions.get(sessionId);
        return session?.interactionHistory || [];
    }
    /**
     * Set up handlers for browser permissions
     */
    async setupPermissionHandlers(page) {
        // Use the shared browser permission handler
        await BrowserPermissionHandler.configurePermissions(page, {
            permissions: ['geolocation', 'notifications', 'camera', 'microphone'],
            geolocation: { latitude: 40.7128, longitude: -74.0060 },
            autoAcceptDialogs: true
        });
    }
}
//# sourceMappingURL=flutter-quantum-debugger.js.map