ai-debug-local-mcp
Version:
🎯 ENHANCED AI GUIDANCE v4.1.2: Dramatically improved tool descriptions help AI users choose the right tools instead of 'close enough' options. Ultra-fast keyboard automation (10x speed), universal recording, multi-ecosystem debugging support, and compreh
394 lines • 18.6 kB
JavaScript
import { FlutterSemanticAnalyzer } from '../flutter-semantic-analyzer.js';
import { FlutterSemanticFilter } from '../flutter-semantic-filter.js';
import { FlutterClickHandler } from '../flutter-click-handler.js';
export class UserActionSimulator {
flutterEnhancedEngine;
page;
constructor(flutterEnhancedEngine) {
this.flutterEnhancedEngine = flutterEnhancedEngine;
}
/**
* Set the current page for action simulation
*/
setPage(page) {
this.page = page;
}
/**
* Update Flutter enhanced engine reference
*/
setFlutterEnhancedEngine(engine) {
this.flutterEnhancedEngine = engine;
}
/**
* Basic user action simulation (original simulateUserAction method)
*/
async simulateUserAction(action, selector, value) {
if (!this.page) {
throw new Error('No page attached');
}
switch (action) {
case 'click':
if (selector) {
await this.page.click(selector);
}
break;
case 'type':
if (selector && value) {
await this.page.fill(selector, value);
}
break;
case 'hover':
if (selector) {
await this.page.hover(selector);
}
break;
case 'screenshot':
await this.page.screenshot({
path: value || 'debug-screenshot.png',
fullPage: true
});
break;
case 'reload':
await this.page.reload();
break;
default:
throw new Error(`Unknown action: ${action}`);
}
}
/**
* Flutter-specific action simulation (original simulateFlutterAction method)
*/
async simulateFlutterAction(page, action, selector, value) {
const startTime = Date.now();
try {
// First, analyze the semantic tree for better understanding
const semanticNodes = await FlutterSemanticAnalyzer.analyzeSemanticTree(page);
const semanticMap = FlutterSemanticAnalyzer.generateSemanticMap(semanticNodes);
console.log('Flutter Semantic Map:\n', semanticMap);
// Use FlutterSemanticFilter for better node selection
const filteredNodes = FlutterSemanticFilter.filterNodes(semanticNodes, selector || '', {
requireClickable: action === 'click',
excludeFullPage: true,
preferExactMatch: true
});
// Debug output
if (filteredNodes.length > 0) {
console.log(`\n🎯 Found ${filteredNodes.length} filtered nodes for "${selector}"`);
FlutterSemanticFilter.debugFilter(semanticNodes, selector || '');
}
if (filteredNodes.length > 0) {
const node = filteredNodes[0];
console.log(`\n✅ Selected node: ${node.label} at (${node.bounds.x}, ${node.bounds.y})`);
console.log(` Size: ${node.bounds.width}x${node.bounds.height}`);
console.log(` Clickable: ${node.isClickable}`);
// Try clicking the semantic node using FlutterClickHandler
if (action === 'click' && node.isClickable) {
const clickSuccess = await FlutterClickHandler.performClick(page, {
x: node.bounds.x + node.bounds.width / 2,
y: node.bounds.y + node.bounds.height / 2,
verbose: true
});
if (clickSuccess) {
return {
success: true,
description: `Clicked Flutter semantic node "${node.label}" via enhanced handler`,
duration: Date.now() - startTime,
semanticInfo: node
};
}
}
}
// Flutter web renders on canvas, so we need multiple strategies
// Strategy 1: Try multiple search approaches
const result = await page.evaluate((searchText) => {
// Strategy 1: Check semantic nodes (accessibility tree)
const checkSemanticNodes = () => {
const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_ELEMENT, null);
const elements = [];
let node;
while (node = walker.nextNode()) {
if (node instanceof Element && node.getAttribute('aria-label')?.includes(searchText)) {
elements.push({
element: node,
text: node.getAttribute('aria-label'),
x: node.getBoundingClientRect().x,
y: node.getBoundingClientRect().y,
width: node.getBoundingClientRect().width,
height: node.getBoundingClientRect().height,
method: 'aria-label'
});
}
}
return elements;
};
// Strategy 2: Check Flutter's semantic tree directly
const checkFlutterSemantics = () => {
const flutterSemantics = window.flutterCanvasKit?.semantics;
if (flutterSemantics) {
try {
const semanticNodes = Object.values(flutterSemantics._semanticsNodes || {});
return semanticNodes.filter((node) => node.label?.includes(searchText) ||
node.value?.includes(searchText) ||
node.hint?.includes(searchText)).map((node) => ({
element: null,
text: node.label || node.value || node.hint,
x: node.rect?.left || 0,
y: node.rect?.top || 0,
width: node.rect?.width || 0,
height: node.rect?.height || 0,
method: 'flutter-semantics',
flutterNode: node
}));
}
catch (e) {
return [];
}
}
return [];
};
// Strategy 3: Look for text in DOM that might be rendered by Flutter
const checkDOMText = () => {
const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT);
const textNodes = [];
let node;
while (node = walker.nextNode()) {
if (node.textContent?.includes(searchText) && node.parentElement) {
const parent = node.parentElement;
textNodes.push({
element: parent,
text: node.textContent,
x: parent.getBoundingClientRect().x,
y: parent.getBoundingClientRect().y,
width: parent.getBoundingClientRect().width,
height: parent.getBoundingClientRect().height,
method: 'dom-text'
});
}
}
return textNodes;
};
// Try all strategies and combine results
const semanticResults = checkSemanticNodes();
const flutterResults = checkFlutterSemantics();
const domResults = checkDOMText();
return [...semanticResults, ...flutterResults, ...domResults];
}, selector || '');
console.log(`Found ${result.length} potential Flutter elements for: "${selector}"`);
if (result.length > 0) {
// Sort by preference: semantic nodes > flutter semantics > DOM text
const sortedResults = result.sort((a, b) => {
const methodPriority = { 'aria-label': 3, 'flutter-semantics': 2, 'dom-text': 1 };
return methodPriority[b.method] - methodPriority[a.method];
});
const target = sortedResults[0];
console.log(`Selected target: ${target.text} (${target.method}) at (${target.x}, ${target.y})`);
if (action === 'click') {
// Calculate center point for clicking
const centerX = target.x + target.width / 2;
const centerY = target.y + target.height / 2;
try {
await page.mouse.click(centerX, centerY);
return {
success: true,
description: `Clicked Flutter element "${target.text}" at (${centerX}, ${centerY}) using ${target.method}`,
duration: Date.now() - startTime
};
}
catch (clickError) {
// Try multiple click strategies
const strategies = [
() => page.mouse.click(centerX, centerY, { delay: 100 }),
() => page.mouse.click(centerX, centerY, { button: 'left', clickCount: 1 }),
() => page.evaluate(({ x, y }) => {
const element = document.elementFromPoint(x, y);
if (element) {
element.dispatchEvent(new MouseEvent('click', { bubbles: true }));
}
}, { x: centerX, y: centerY })
];
for (const strategy of strategies) {
try {
await strategy();
return {
success: true,
description: `Clicked Flutter element "${target.text}" at (${centerX}, ${centerY}) using fallback strategy`,
duration: Date.now() - startTime
};
}
catch (e) {
// Continue to next strategy
}
}
throw clickError;
}
}
if (action === 'type' && value) {
// For typing, we need to focus the element first
const centerX = target.x + target.width / 2;
const centerY = target.y + target.height / 2;
await page.mouse.click(centerX, centerY);
await page.waitForTimeout(100); // Wait for focus
await page.keyboard.type(value);
return {
success: true,
description: `Typed "${value}" into Flutter element "${target.text}"`,
duration: Date.now() - startTime
};
}
}
// If no specific elements found, try coordinate-based approach
if (action === 'click') {
const flutterCanvas = await page.$('flt-scene-host canvas, flutter-view canvas');
if (flutterCanvas) {
const bounds = await flutterCanvas.boundingBox();
if (bounds) {
// Click center of canvas as last resort
const centerX = bounds.x + bounds.width / 2;
const centerY = bounds.y + bounds.height / 2;
await page.mouse.click(centerX, centerY);
return {
success: true,
description: `Clicked Flutter canvas center at (${centerX}, ${centerY}) as fallback`,
duration: Date.now() - startTime
};
}
}
}
// If all strategies fail
return {
success: false,
error: `Could not find or interact with Flutter element: ${selector}`,
description: `Failed to ${action} Flutter element "${selector}"`,
duration: Date.now() - startTime
};
}
catch (error) {
return {
success: false,
error: error instanceof Error ? error.message : String(error),
description: `Error during Flutter ${action}: ${selector}`,
duration: Date.now() - startTime
};
}
}
/**
* Comprehensive action simulation (original simulateAction method)
*/
async simulateAction(page, action, selector, value, coordinates) {
const startTime = Date.now();
let success = true;
let error = null;
let description = '';
try {
// Check if this is a Flutter web app
const isFlutter = await page.evaluate(() => {
return !!window.__FLUTTER_DEBUG__ ||
document.querySelector('flt-scene-host') !== null ||
document.querySelector('flutter-view') !== null;
});
// If Flutter and coordinates provided, use FlutterClickHandler
if (isFlutter && action === 'click' && coordinates) {
try {
const clickSuccess = await FlutterClickHandler.performClick(page, {
x: coordinates.x,
y: coordinates.y,
verbose: true
});
if (clickSuccess) {
return {
success: true,
error: null,
description: `Direct clicked Flutter element at (${coordinates.x}, ${coordinates.y})`,
duration: Date.now() - startTime
};
}
}
catch (e) {
// Fall back to regular strategies
error = e instanceof Error ? e.message : String(e);
}
}
// If Flutter, try Flutter-specific interaction first
if (isFlutter && this.flutterEnhancedEngine) {
try {
const flutterResult = await this.simulateFlutterAction(page, action, selector, value);
if (flutterResult.success) {
return flutterResult;
}
}
catch (e) {
// Fall back to regular strategies
}
}
// Try multiple selector strategies
const strategies = [
selector, // Direct selector
`[data-testid="${selector}"]`, // Test ID
`#${selector}`, // ID
`.${selector}`, // Class
`[name="${selector}"]`, // Name attribute
`[placeholder*="${selector}"]`, // Placeholder contains
`[title*="${selector}"]`, // Title contains
`[alt*="${selector}"]`, // Alt text contains
`[aria-label*="${selector}"]`, // ARIA label
`label:has-text("${selector}") + input`, // Label adjacent
`label:has-text("${selector}") input` // Label containing
];
let worked = false;
let lastError = null;
for (const strategy of strategies) {
try {
switch (action) {
case 'click':
await page.click(strategy, { timeout: 2000 });
description = `Clicked element ${selector}`;
worked = true;
break;
case 'type':
await page.fill(strategy, value || '', { timeout: 2000 });
description = `Typed "${value}" into ${selector}`;
worked = true;
break;
case 'submit':
await page.press(strategy, 'Enter', { timeout: 2000 });
description = `Submitted form at ${selector}`;
worked = true;
break;
case 'scroll':
await page.hover(strategy, { timeout: 2000 });
await page.mouse.wheel(0, parseInt(value || '100'));
description = `Scrolled ${value || 100}px at ${selector}`;
worked = true;
break;
case 'hover':
await page.hover(strategy, { timeout: 2000 });
description = `Hovered over ${selector}`;
worked = true;
break;
}
if (worked)
break;
}
catch (e) {
lastError = e;
// Continue to next strategy
}
}
if (!worked) {
throw lastError || new Error(`Could not ${action} element: ${selector}`);
}
}
catch (e) {
success = false;
error = e instanceof Error ? e.message : String(e);
description = `Failed to ${action} ${selector}`;
}
return {
success,
error,
description,
duration: Date.now() - startTime
};
}
}
//# sourceMappingURL=user-action-simulator.js.map