ai-debug-local-mcp
Version:
🎯 ENHANCED AI GUIDANCE v4.1.2: Dramatically improved tool descriptions help AI users choose the right tools instead of 'close enough' options. Ultra-fast keyboard automation (10x speed), universal recording, multi-ecosystem debugging support, and compreh
535 lines • 28.7 kB
JavaScript
/**
* Flutter Interaction Predictor
* REAL implementation that predicts and executes interactions on Flutter web apps
*/
import { FlutterUIPatterns } from './flutter-ui-patterns.js';
export class FlutterInteractionPredictor {
/**
* Generate real interaction strategies for a target element
*/
static async generateStrategies(page, target, structure) {
const strategies = [];
// Strategy 1: Direct semantic node click with enhanced logic
const semanticMatches = structure.interactables.filter(el => el.label.toLowerCase().includes(target.toLowerCase()));
// console.log(`Found ${semanticMatches.length} semantic matches for "${target}"`);
semanticMatches.forEach(m => {
// console.log(` - "${m.label}" at (${m.bounds.x}, ${m.bounds.y})`);
});
for (const match of semanticMatches) {
strategies.push({
method: `semantic-click-${match.id}`,
confidence: 0.95,
execute: async () => {
try {
const centerX = match.bounds.x + match.bounds.width / 2;
const centerY = match.bounds.y + match.bounds.height / 2;
// console.log(`Clicking semantic element at (${centerX}, ${centerY})`);
// Store initial state for better change detection
const initialSemanticCount = await page.evaluate(() => document.querySelectorAll('[role], [aria-label], flt-semantics').length);
// Click the element
await page.mouse.click(centerX, centerY);
// Also dispatch Flutter-compatible events
await page.evaluate(({ x, y }) => {
const target = document.elementFromPoint(x, y) ||
document.querySelector('flt-glass-pane') ||
document.querySelector('canvas');
if (target) {
const events = [
new PointerEvent('pointerdown', {
bubbles: true,
cancelable: true,
clientX: x,
clientY: y,
pointerId: 1,
pointerType: 'mouse',
button: 0,
buttons: 1
}),
new PointerEvent('pointerup', {
bubbles: true,
cancelable: true,
clientX: x,
clientY: y,
pointerId: 1,
pointerType: 'mouse',
button: 0,
buttons: 0
})
];
events.forEach(event => target.dispatchEvent(event));
}
}, { x: centerX, y: centerY });
await page.waitForTimeout(1000);
// Check for changes with multiple methods
const newSemanticCount = await page.evaluate(() => document.querySelectorAll('[role], [aria-label], flt-semantics').length);
// Check multiple indicators of change
const initialHighZ = await page.evaluate(() => Array.from(document.querySelectorAll('*')).filter(el => {
const zIndex = window.getComputedStyle(el).zIndex;
return zIndex !== 'auto' && parseInt(zIndex) > 1000;
}).length);
const changed = await this.detectPageChange(page) ||
(newSemanticCount !== initialSemanticCount);
// Re-check high z-index elements
const newHighZ = await page.evaluate(() => Array.from(document.querySelectorAll('*')).filter(el => {
const zIndex = window.getComputedStyle(el).zIndex;
return zIndex !== 'auto' && parseInt(zIndex) > 1000;
}).length);
const highZChanged = initialHighZ !== newHighZ;
if (changed || highZChanged) {
// console.log('Semantic click successful - page changed');
if (highZChanged) {
// console.log(`High z-index elements: ${initialHighZ} → ${newHighZ}`);
}
}
return changed || highZChanged;
}
catch (e) {
console.error('Semantic click failed:', e);
return false;
}
}
});
}
// Strategy 2: Visual text search with coordinate mapping
strategies.push({
method: 'visual-text-search',
confidence: 0.8,
execute: async () => {
try {
// Search for text visually in the page
const textLocation = await page.evaluate((searchText) => {
// Create a text search algorithm that works for any Flutter app
const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT, {
acceptNode: (node) => {
const text = node.nodeValue?.trim() || '';
if (text && text.toLowerCase().includes(searchText.toLowerCase())) {
return NodeFilter.FILTER_ACCEPT;
}
return NodeFilter.FILTER_REJECT;
}
});
let node;
while (node = walker.nextNode()) {
const range = document.createRange();
range.selectNodeContents(node);
const rect = range.getBoundingClientRect();
if (rect.width > 0 && rect.height > 0) {
// Found text, now find the clickable parent
let clickableParent = node.parentElement;
while (clickableParent) {
const parentRect = clickableParent.getBoundingClientRect();
const styles = window.getComputedStyle(clickableParent);
// Check if this looks clickable
if (parentRect.width > 40 && parentRect.height > 20 &&
(styles.cursor === 'pointer' ||
clickableParent.role === 'button' ||
clickableParent.tagName === 'BUTTON')) {
return {
x: parentRect.left + parentRect.width / 2,
y: parentRect.top + parentRect.height / 2,
found: true
};
}
clickableParent = clickableParent.parentElement;
}
// If no clickable parent, return text location
return {
x: rect.left + rect.width / 2,
y: rect.top + rect.height / 2,
found: true
};
}
}
return { found: false };
}, target);
if (textLocation.found && textLocation.x !== undefined && textLocation.y !== undefined) {
const x = textLocation.x;
const y = textLocation.y;
// Move mouse to position first
await page.mouse.move(x, y);
await page.waitForTimeout(50);
// Dispatch Flutter-compatible pointer events
await page.evaluate(({ x, y }) => {
const canvas = document.querySelector('canvas');
if (!canvas)
return false;
// Create a sequence of events that Flutter expects
const events = [
new PointerEvent('pointerdown', {
bubbles: true,
cancelable: true,
clientX: x,
clientY: y,
pointerId: 1,
pointerType: 'mouse',
isPrimary: true,
button: 0,
buttons: 1
}),
new MouseEvent('mousedown', {
bubbles: true,
cancelable: true,
clientX: x,
clientY: y,
button: 0,
buttons: 1
}),
new PointerEvent('pointerup', {
bubbles: true,
cancelable: true,
clientX: x,
clientY: y,
pointerId: 1,
pointerType: 'mouse',
isPrimary: true,
button: 0,
buttons: 0
}),
new MouseEvent('mouseup', {
bubbles: true,
cancelable: true,
clientX: x,
clientY: y,
button: 0,
buttons: 0
}),
new PointerEvent('click', {
bubbles: true,
cancelable: true,
clientX: x,
clientY: y,
pointerId: 1,
pointerType: 'mouse',
isPrimary: true
})
];
// Dispatch to both canvas and glass pane
const glassPane = document.querySelector('flt-glass-pane') || canvas;
events.forEach(event => {
glassPane.dispatchEvent(event);
canvas.dispatchEvent(event);
});
return true;
}, { x, y });
// Also try Playwright's click as backup
await page.mouse.click(x, y);
await page.waitForTimeout(1000);
const changed = await this.detectPageChange(page);
return changed;
}
return false;
}
catch (e) {
console.error('Flutter canvas click failed:', e);
return false;
}
}
});
// Strategy 3: Accessibility tree navigation
strategies.push({
method: 'accessibility-navigation',
confidence: 0.7,
execute: async () => {
try {
// Try to find and click using accessibility attributes
const clicked = await page.evaluate((searchText) => {
// Look for elements with matching aria-labels
const elements = document.querySelectorAll('[aria-label], [role="button"]');
for (const el of Array.from(elements)) {
const label = el.getAttribute('aria-label') || el.textContent || '';
if (label.toLowerCase().includes(searchText.toLowerCase())) {
el.click();
// Also dispatch events manually
const rect = el.getBoundingClientRect();
const clickEvent = new MouseEvent('click', {
bubbles: true,
cancelable: true,
clientX: rect.left + rect.width / 2,
clientY: rect.top + rect.height / 2
});
el.dispatchEvent(clickEvent);
return true;
}
}
return false;
}, target);
if (clicked) {
await page.waitForTimeout(500);
const changed = await this.detectPageChange(page);
return changed;
}
return false;
}
catch (e) {
console.error('Accessibility navigation failed:', e);
return false;
}
}
});
// Strategy 4: Pattern-based location with visual markers
strategies.push({
method: 'pattern-location',
confidence: 0.6,
execute: async () => {
try {
// Look for visual patterns that indicate buttons
const buttonLocation = await page.evaluate(() => {
// Find elements that look like buttons based on styling
const candidates = document.querySelectorAll('div, span, button');
for (const el of Array.from(candidates)) {
const styles = window.getComputedStyle(el);
const text = el.textContent || '';
// Check if it looks like a button
if (text.toLowerCase().includes('submit report') ||
(styles.backgroundColor !== 'rgba(0, 0, 0, 0)' &&
styles.cursor === 'pointer')) {
const rect = el.getBoundingClientRect();
if (rect.width > 50 && rect.height > 20) {
return {
x: rect.left + rect.width / 2,
y: rect.top + rect.height / 2
};
}
}
}
return null;
});
if (buttonLocation) {
await page.mouse.click(buttonLocation.x, buttonLocation.y);
await page.waitForTimeout(500);
const changed = await this.detectPageChange(page);
return changed;
}
return false;
}
catch (e) {
console.error('Pattern location failed:', e);
return false;
}
}
});
// Strategy 5: Advanced UI Pattern Library
strategies.push({
method: 'ui-pattern-library',
confidence: 0.85,
execute: async () => {
try {
// console.log(`Using UI Pattern Library to find: "${target}"`);
// Find matching patterns
const matches = await FlutterUIPatterns.findElement(page, target);
if (matches.length > 0) {
const bestMatch = matches[0];
// console.log(`Found pattern match: ${bestMatch.pattern.name} at (${bestMatch.location.x}, ${bestMatch.location.y})`);
// Humanize coordinates slightly
const coords = FlutterUIPatterns.humanizeCoordinates(bestMatch.location.x, bestMatch.location.y);
// Move mouse to position first
await page.mouse.move(coords.x, coords.y);
await page.waitForTimeout(100);
// Try multiple click methods for better compatibility
// Method 1: Direct page click
await page.mouse.click(coords.x, coords.y);
// Method 2: Dispatch Flutter-compatible events
await page.evaluate(({ x, y }) => {
const canvas = document.querySelector('canvas');
const glassPane = document.querySelector('flt-glass-pane');
const target = glassPane || canvas;
if (!target)
return false;
// Create proper event sequence for Flutter
const events = [
new PointerEvent('pointerdown', {
bubbles: true,
cancelable: true,
clientX: x,
clientY: y,
pointerId: 1,
pointerType: 'mouse',
isPrimary: true,
button: 0,
buttons: 1
}),
new PointerEvent('pointerup', {
bubbles: true,
cancelable: true,
clientX: x,
clientY: y,
pointerId: 1,
pointerType: 'mouse',
isPrimary: true,
button: 0,
buttons: 0
}),
new MouseEvent('click', {
bubbles: true,
cancelable: true,
clientX: x,
clientY: y,
button: 0
})
];
events.forEach(event => {
target.dispatchEvent(event);
});
return true;
}, coords);
await page.waitForTimeout(1000);
const changed = await this.detectPageChange(page);
// console.log(`Pattern-based click ${changed ? 'succeeded' : 'failed'}`);
return changed;
}
// console.log('No pattern matches found');
return false;
}
catch (e) {
console.error('UI pattern library strategy failed:', e);
return false;
}
}
});
// Strategy 6: Smart Region-Based Clicking
// When semantic nodes are in sidebar/panel areas, try clicking in the actual content area
strategies.push({
method: 'region-based-click',
confidence: 0.75,
execute: async () => {
try {
// console.log('Using region-based strategy for panel/sidebar elements');
// If we found semantic matches in typical sidebar locations (x < 200)
const sidebarMatches = semanticMatches.filter(m => m.bounds.x < 200);
if (sidebarMatches.length > 0) {
// The actual clickable area might be offset from the sidebar
const sidebarMatch = sidebarMatches[0];
const regions = [
{ x: 50, y: sidebarMatch.bounds.y, name: 'left-panel' },
{ x: 150, y: sidebarMatch.bounds.y, name: 'sidebar-offset' },
{ x: sidebarMatch.bounds.x, y: sidebarMatch.bounds.y - 50, name: 'above' },
{ x: sidebarMatch.bounds.x, y: sidebarMatch.bounds.y + 50, name: 'below' }
];
for (const region of regions) {
// console.log(`Trying region ${region.name} at (${region.x}, ${region.y})`);
const initialState = await page.evaluate(() => ({
semanticCount: document.querySelectorAll('[role], [aria-label], flt-semantics').length,
bodyLength: document.body.textContent?.length || 0
}));
await page.mouse.click(region.x, region.y);
await page.waitForTimeout(800);
const newState = await page.evaluate(() => ({
semanticCount: document.querySelectorAll('[role], [aria-label], flt-semantics').length,
bodyLength: document.body.textContent?.length || 0
}));
if (newState.semanticCount !== initialState.semanticCount ||
Math.abs(newState.bodyLength - initialState.bodyLength) > 50) {
// console.log(`Region click successful at ${region.name}`);
return true;
}
}
}
return false;
}
catch (e) {
console.error('Region-based strategy failed:', e);
return false;
}
}
});
return strategies.sort((a, b) => b.confidence - a.confidence);
}
/**
* Detect if the page changed after an interaction
*/
static async detectPageChange(page) {
try {
// Store initial state
const initialState = await page.evaluate(() => {
return {
url: window.location.href,
bodyText: document.body.textContent?.substring(0, 1000) || '',
canvasCount: document.querySelectorAll('canvas').length,
semanticCount: document.querySelectorAll('[role], [aria-label], flt-semantics').length,
dialogCount: document.querySelectorAll('[role="dialog"], [aria-modal="true"]').length,
// Flutter-specific indicators
glassPaneCount: document.querySelectorAll('flt-glass-pane').length,
flutterViewCount: document.querySelectorAll('flutter-view').length,
// Check for overlay/modal indicators
highZIndexElements: Array.from(document.querySelectorAll('*')).filter(el => {
const zIndex = window.getComputedStyle(el).zIndex;
return zIndex !== 'auto' && parseInt(zIndex) > 1000;
}).length
};
});
// Wait for any transitions/animations
await page.waitForTimeout(1000);
// Check state again
const newState = await page.evaluate(() => {
return {
url: window.location.href,
bodyText: document.body.textContent?.substring(0, 1000) || '',
canvasCount: document.querySelectorAll('canvas').length,
semanticCount: document.querySelectorAll('[role], [aria-label], flt-semantics').length,
dialogCount: document.querySelectorAll('[role="dialog"], [aria-modal="true"]').length,
// Flutter-specific indicators
glassPaneCount: document.querySelectorAll('flt-glass-pane').length,
flutterViewCount: document.querySelectorAll('flutter-view').length,
// Check for overlay/modal indicators
highZIndexElements: Array.from(document.querySelectorAll('*')).filter(el => {
const zIndex = window.getComputedStyle(el).zIndex;
return zIndex !== 'auto' && parseInt(zIndex) > 1000;
}).length
};
});
// Check if any meaningful change occurred
const changed = initialState.url !== newState.url ||
initialState.bodyText !== newState.bodyText ||
initialState.semanticCount !== newState.semanticCount ||
initialState.dialogCount !== newState.dialogCount ||
initialState.highZIndexElements !== newState.highZIndexElements ||
Math.abs(initialState.canvasCount - newState.canvasCount) > 2; // Allow small canvas count changes
if (changed) {
// console.log('🔄 Page change detected:');
if (initialState.url !== newState.url)
console.log(' - URL changed');
if (initialState.bodyText !== newState.bodyText)
console.log(' - Body text changed');
if (initialState.semanticCount !== newState.semanticCount) {
// console.log(` - Semantic elements changed: ${initialState.semanticCount} → ${newState.semanticCount}`);
}
if (initialState.dialogCount !== newState.dialogCount) {
// console.log(` - Dialog count changed: ${initialState.dialogCount} → ${newState.dialogCount}`);
}
if (initialState.highZIndexElements !== newState.highZIndexElements) {
// console.log(` - High z-index elements changed: ${initialState.highZIndexElements} → ${newState.highZIndexElements}`);
}
}
return changed;
}
catch (e) {
console.error('Change detection failed:', e);
return false;
}
}
/**
* Execute strategies until one succeeds
*/
static async executeStrategies(strategies) {
for (const strategy of strategies) {
// console.log(`Trying strategy: ${strategy.method} (confidence: ${strategy.confidence})`);
try {
const success = await strategy.execute();
if (success) {
return {
success: true,
method: strategy.method
};
}
}
catch (e) {
console.error(`Strategy ${strategy.method} error:`, e);
}
}
return {
success: false,
error: 'All strategies failed'
};
}
}
//# sourceMappingURL=flutter-interaction-predictor.js.map