ai-debug-local-mcp
Version:
🎯 ENHANCED AI GUIDANCE v4.1.2: Dramatically improved tool descriptions help AI users choose the right tools instead of 'close enough' options. Ultra-fast keyboard automation (10x speed), universal recording, multi-ecosystem debugging support, and compreh
270 lines • 12 kB
JavaScript
/**
* Flutter Comprehensive Element Detector
* Dynamically finds ALL clickable elements using multiple strategies
*/
export class FlutterComprehensiveDetector {
/**
* Find ALL clickable elements using multiple detection strategies
*/
static async detectAllElements(page) {
const elements = await page.evaluate(() => {
const detectedElements = [];
const processedBounds = new Set();
// Helper to check if bounds already processed
const boundsKey = (x, y, w, h) => `${Math.round(x)},${Math.round(y)},${Math.round(w)},${Math.round(h)}`;
// Helper to check if element is visible
const isVisible = (rect) => rect.width > 0 && rect.height > 0 &&
rect.x >= 0 && rect.y >= 0 &&
rect.x < window.innerWidth && rect.y < window.innerHeight;
// Strategy 1: Semantic elements (Flutter accessibility)
const semanticElements = document.querySelectorAll('[role], [aria-label], flt-semantics, flt-semantics-container');
semanticElements.forEach(el => {
const rect = el.getBoundingClientRect();
if (!isVisible(rect))
return;
const key = boundsKey(rect.x, rect.y, rect.width, rect.height);
if (processedBounds.has(key))
return;
processedBounds.add(key);
const label = el.getAttribute('aria-label') ||
el.getAttribute('role') ||
el.textContent?.trim() || '';
if (label) {
detectedElements.push({
type: 'semantic',
label,
bounds: { x: rect.x, y: rect.y, width: rect.width, height: rect.height },
confidence: 0.9,
clickable: true
});
}
});
// Strategy 2: Find all visible text that might be clickable
const textWalker = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT, {
acceptNode: (node) => {
const text = node.nodeValue?.trim() || '';
return text.length > 1 ? NodeFilter.FILTER_ACCEPT : NodeFilter.FILTER_REJECT;
}
});
let textNode;
while (textNode = textWalker.nextNode()) {
const range = document.createRange();
range.selectNodeContents(textNode);
const rect = range.getBoundingClientRect();
if (!isVisible(rect))
continue;
// Find parent element
let parent = textNode.parentElement;
let clickableParent = null;
let isClickable = false;
while (parent && parent !== document.body) {
const style = window.getComputedStyle(parent);
const parentRect = parent.getBoundingClientRect();
// Check if parent has click indicators
if (style.cursor === 'pointer' ||
parent.onclick !== null ||
parent.hasAttribute('onclick') ||
parent.role === 'button' ||
parent.tagName === 'BUTTON' ||
parent.tagName === 'A') {
clickableParent = parent;
isClickable = true;
rect.x = parentRect.x;
rect.y = parentRect.y;
rect.width = parentRect.width;
rect.height = parentRect.height;
break;
}
parent = parent.parentElement;
}
const key = boundsKey(rect.x, rect.y, rect.width, rect.height);
if (processedBounds.has(key))
continue;
processedBounds.add(key);
const text = textNode.nodeValue?.trim() || '';
if (text) {
detectedElements.push({
type: 'text',
label: text,
bounds: { x: rect.x, y: rect.y, width: rect.width, height: rect.height },
confidence: isClickable ? 0.8 : 0.3,
clickable: isClickable
});
}
}
// Strategy 3: Elements with cursor:pointer
const allElements = document.querySelectorAll('*');
allElements.forEach(el => {
const style = window.getComputedStyle(el);
if (style.cursor === 'pointer') {
const rect = el.getBoundingClientRect();
if (!isVisible(rect))
return;
const key = boundsKey(rect.x, rect.y, rect.width, rect.height);
if (processedBounds.has(key))
return;
processedBounds.add(key);
const label = el.getAttribute('aria-label') ||
el.textContent?.trim() ||
el.tagName.toLowerCase();
detectedElements.push({
type: 'cursor',
label,
bounds: { x: rect.x, y: rect.y, width: rect.width, height: rect.height },
confidence: 0.7,
clickable: true
});
}
});
// Strategy 4: Canvas click detection for Flutter
const canvases = document.querySelectorAll('canvas');
canvases.forEach(canvas => {
const rect = canvas.getBoundingClientRect();
if (!isVisible(rect))
return;
// For Flutter canvas, we need to use heuristics
// Common button positions in Flutter apps
const buttonPositions = [
// Bottom center (submit buttons)
{ x: rect.width / 2, y: rect.height * 0.85, label: 'Bottom Action' },
// Top corners (navigation)
{ x: 24, y: 24, label: 'Top Left Nav' },
{ x: rect.width - 24, y: 24, label: 'Top Right Action' },
// FAB position
{ x: rect.width - 56, y: rect.height - 56, label: 'FAB' }
];
buttonPositions.forEach(pos => {
const absX = rect.x + pos.x;
const absY = rect.y + pos.y;
const key = boundsKey(absX - 25, absY - 25, 50, 50);
if (!processedBounds.has(key)) {
processedBounds.add(key);
detectedElements.push({
type: 'canvas',
label: pos.label,
bounds: { x: absX - 25, y: absY - 25, width: 50, height: 50 },
confidence: 0.4,
clickable: true
});
}
});
});
// Sort by confidence
return detectedElements.sort((a, b) => b.confidence - a.confidence);
});
return elements;
}
/**
* Create visual overlay showing all detected elements
*/
static async createDebugOverlay(page, elements) {
await page.evaluate((elements) => {
// Remove existing overlay
const existing = document.getElementById('flutter-detector-overlay');
if (existing)
existing.remove();
// Create canvas overlay
const overlay = document.createElement('canvas');
overlay.id = 'flutter-detector-overlay';
overlay.style.cssText = `
position: fixed;
top: 0;
left: 0;
width: 100%;
height: 100%;
pointer-events: none;
z-index: 999999;
`;
document.body.appendChild(overlay);
const ctx = overlay.getContext('2d');
if (!ctx)
return;
overlay.width = window.innerWidth;
overlay.height = window.innerHeight;
// Color scheme based on type
const colors = {
semantic: 'red',
text: 'blue',
visual: 'green',
cursor: 'orange',
canvas: 'purple'
};
// Draw each element
elements.forEach((el, index) => {
const color = colors[el.type] || 'gray';
// Draw bounding box
ctx.strokeStyle = color;
ctx.lineWidth = 2;
ctx.strokeRect(el.bounds.x, el.bounds.y, el.bounds.width, el.bounds.height);
// Draw confidence indicator
ctx.fillStyle = color;
ctx.globalAlpha = el.confidence * 0.3;
ctx.fillRect(el.bounds.x, el.bounds.y, el.bounds.width, el.bounds.height);
ctx.globalAlpha = 1;
// Draw label with background
if (el.label) {
ctx.font = '12px Arial';
const metrics = ctx.measureText(el.label);
const labelHeight = 16;
// Background for label
ctx.fillStyle = 'white';
ctx.fillRect(el.bounds.x, el.bounds.y - labelHeight - 2, metrics.width + 4, labelHeight);
// Label text
ctx.fillStyle = color;
ctx.fillText(el.label, el.bounds.x + 2, el.bounds.y - 4);
}
// Draw index number
ctx.fillStyle = 'white';
ctx.beginPath();
ctx.arc(el.bounds.x + el.bounds.width - 10, el.bounds.y + 10, 10, 0, 2 * Math.PI);
ctx.fill();
ctx.fillStyle = 'black';
ctx.font = 'bold 10px Arial';
ctx.textAlign = 'center';
ctx.fillText(index.toString(), el.bounds.x + el.bounds.width - 10, el.bounds.y + 14);
ctx.textAlign = 'left';
});
// Add legend
ctx.fillStyle = 'rgba(255, 255, 255, 0.9)';
ctx.fillRect(10, 10, 150, 120);
ctx.fillStyle = 'black';
ctx.font = 'bold 14px Arial';
ctx.fillText('Element Types:', 15, 30);
ctx.font = '12px Arial';
let y = 50;
Object.entries(colors).forEach(([type, color]) => {
ctx.fillStyle = color;
ctx.fillRect(15, y - 10, 10, 10);
ctx.fillStyle = 'black';
ctx.fillText(type, 30, y);
y += 20;
});
}, elements);
}
/**
* Find element by text content
*/
static findElementByText(elements, searchText) {
const search = searchText.toLowerCase();
// First try exact match
let found = elements.find(el => el.label.toLowerCase() === search);
if (found)
return found;
// Then try contains
found = elements.find(el => el.label.toLowerCase().includes(search));
if (found)
return found;
// Then try if search is contained in label
found = elements.find(el => search.includes(el.label.toLowerCase()));
if (found)
return found;
// Finally try word matching
const searchWords = search.split(/\s+/);
found = elements.find(el => {
const labelWords = el.label.toLowerCase().split(/\s+/);
return searchWords.some(word => labelWords.includes(word));
});
return found || null;
}
}
//# sourceMappingURL=flutter-comprehensive-detector.js.map