ai-debug-local-mcp
Version:
🎯 ENHANCED AI GUIDANCE v4.1.2: Dramatically improved tool descriptions help AI users choose the right tools instead of 'close enough' options. Ultra-fast keyboard automation (10x speed), universal recording, multi-ecosystem debugging support, and compreh
490 lines • 22.4 kB
JavaScript
/**
* Flutter Semantic Analyzer
* Extracts meaningful structure from Flutter's accessibility tree
*/
export class FlutterSemanticAnalyzer {
/**
* Analyze Flutter's semantic tree to understand UI structure
*/
static async analyzeSemanticTree(page) {
return await page.evaluate(() => {
const nodes = [];
// Flutter uses semantic nodes for accessibility
// Expand our search to include more Flutter-specific elements
const semanticElements = document.querySelectorAll('[role], [aria-label], flt-semantics, flt-semantics-container, [class*="flt-"], [data-semantics-label]');
// Also look for clickable elements that might not have semantic markup
const clickableElements = document.querySelectorAll('[style*="cursor: pointer"], [onclick], button, a, input, textarea, select');
// Combine both sets
const allElements = new Set([...Array.from(semanticElements), ...Array.from(clickableElements)]);
function analyzeNode(element) {
const role = element.getAttribute('role') || '';
const label = element.getAttribute('aria-label') || element.textContent || '';
const bounds = element.getBoundingClientRect();
// Skip invisible elements
if (bounds.width === 0 || bounds.height === 0) {
return null;
}
// Determine element type comprehensively
const isButton = role === 'button' ||
element.tagName === 'BUTTON' ||
label.toLowerCase().includes('button') ||
label.toLowerCase().includes('submit');
const isTextField = role === 'textbox' ||
role === 'combobox' ||
element.tagName === 'INPUT' ||
element.tagName === 'TEXTAREA';
// Detect specific input types
const inputElement = element;
const inputType = inputElement.type || element.getAttribute('type') || '';
const isDateInput = inputType === 'date' ||
inputType === 'datetime-local' ||
inputType === 'time' ||
inputType === 'month' ||
inputType === 'week';
const isNumberInput = inputType === 'number' ||
inputType === 'range' ||
role === 'spinbutton' ||
role === 'slider';
const isFileInput = inputType === 'file';
const isColorInput = inputType === 'color';
const isEmailInput = inputType === 'email';
const isPasswordInput = inputType === 'password';
const isTelInput = inputType === 'tel';
const isUrlInput = inputType === 'url';
const isSearchInput = inputType === 'search' || role === 'searchbox';
const isRadioInput = inputType === 'radio' || role === 'radio';
const isCheckboxInput = inputType === 'checkbox' || role === 'checkbox';
const isClickable = isButton ||
element.hasAttribute('onclick') ||
role === 'link' ||
element.tagName === 'A' ||
window.getComputedStyle(element).cursor === 'pointer';
const isScrollable = role === 'scrollbar' ||
element.hasAttribute('aria-scrollable') ||
label.toLowerCase().includes('scroll');
const isToggleable = role === 'switch' ||
role === 'checkbox' ||
element.hasAttribute('aria-checked') ||
element.tagName === 'INPUT' && element.type === 'checkbox';
const isSelectable = role === 'option' ||
role === 'listitem' ||
element.hasAttribute('aria-selected');
const isDraggable = element.hasAttribute('draggable') ||
element.hasAttribute('aria-grabbed');
const isImage = role === 'img' ||
element.tagName === 'IMG' ||
label.toLowerCase().includes('image');
const isLink = role === 'link' ||
element.tagName === 'A';
// Determine element type with comprehensive input type detection
let elementType = 'unknown';
if (isButton)
elementType = 'button';
else if (isDateInput)
elementType = 'date-input';
else if (isNumberInput)
elementType = 'number-input';
else if (isFileInput)
elementType = 'file-input';
else if (isColorInput)
elementType = 'color-input';
else if (isEmailInput)
elementType = 'email-input';
else if (isPasswordInput)
elementType = 'password-input';
else if (isTelInput)
elementType = 'tel-input';
else if (isUrlInput)
elementType = 'url-input';
else if (isSearchInput)
elementType = 'search-input';
else if (isRadioInput)
elementType = 'radio';
else if (isCheckboxInput)
elementType = 'checkbox';
else if (isTextField)
elementType = 'textfield';
else if (isLink)
elementType = 'link';
else if (isImage)
elementType = 'image';
else if (isToggleable)
elementType = 'toggle';
else if (isSelectable)
elementType = 'selectable';
else if (role === 'slider')
elementType = 'slider';
else if (role === 'progressbar')
elementType = 'progressbar';
else if (role === 'tab')
elementType = 'tab';
else if (role === 'menu')
elementType = 'menu';
else if (role === 'menuitem')
elementType = 'menuitem';
else if (role === 'dialog')
elementType = 'dialog';
else if (role === 'heading')
elementType = 'heading';
else if (role === 'list')
elementType = 'list';
else if (role === 'listitem')
elementType = 'listitem';
else if (role === 'grid')
elementType = 'grid';
else if (role === 'table')
elementType = 'table';
else if (role === 'row')
elementType = 'row';
else if (role === 'cell')
elementType = 'cell';
else if (role === 'navigation')
elementType = 'navigation';
else if (role === 'complementary')
elementType = 'sidebar';
else if (role === 'main')
elementType = 'main-content';
else if (role === 'form')
elementType = 'form';
else if (role === 'search')
elementType = 'search-form';
else if (role === 'alert')
elementType = 'alert';
else if (role === 'status')
elementType = 'status';
else if (role === 'tooltip')
elementType = 'tooltip';
// Get available actions based on element type
const actions = [];
// Click action for clickable elements
if (isClickable || isRadioInput || isCheckboxInput)
actions.push('click');
// Text input actions
if (isTextField || isEmailInput || isPasswordInput || isTelInput ||
isUrlInput || isSearchInput) {
actions.push('type', 'clear', 'focus', 'blur', 'select-all');
}
// Date/time input actions
if (isDateInput) {
actions.push('set-date', 'clear', 'focus', 'open-picker');
}
// Number input actions
if (isNumberInput) {
actions.push('set-value', 'increment', 'decrement', 'clear', 'focus');
}
// File input actions
if (isFileInput) {
actions.push('choose-file', 'clear');
}
// Color input actions
if (isColorInput) {
actions.push('set-color', 'open-picker');
}
// Toggle actions
if (isToggleable || isCheckboxInput) {
actions.push('toggle', 'check', 'uncheck');
}
// Radio button actions
if (isRadioInput) {
actions.push('select');
}
// Expandable elements
if (element.getAttribute('aria-expanded') !== null) {
actions.push('expand', 'collapse');
}
// Scrollable elements
if (isScrollable) {
actions.push('scroll', 'scroll-up', 'scroll-down', 'scroll-left', 'scroll-right');
}
// Selectable elements
if (isSelectable) {
actions.push('select', 'deselect');
}
// Draggable elements
if (isDraggable) {
actions.push('drag', 'drop');
}
// Slider/range actions
if (role === 'slider' || inputType === 'range') {
actions.push('slide', 'set-value', 'increase', 'decrease');
}
// Menu actions
if (role === 'menu' || role === 'menuitem') {
actions.push('open', 'close', 'select');
}
// Dialog actions
if (role === 'dialog') {
actions.push('close', 'accept', 'cancel');
}
// Tab actions
if (role === 'tab') {
actions.push('activate', 'focus');
}
// List actions
if (role === 'listitem' || role === 'option') {
actions.push('select', 'activate');
}
// Link actions
if (isLink) {
actions.push('navigate', 'open-in-new-tab');
}
// Image actions
if (isImage) {
actions.push('view', 'download');
}
// Popup actions
if (element.hasAttribute('aria-haspopup')) {
actions.push('open-popup', 'close-popup');
}
// Focus actions for any focusable element
if (element.hasAttribute('tabindex') || element.tagName === 'A' ||
element.tagName === 'BUTTON' || element.tagName === 'INPUT' ||
element.tagName === 'TEXTAREA' || element.tagName === 'SELECT') {
if (!actions.includes('focus'))
actions.push('focus');
if (!actions.includes('blur'))
actions.push('blur');
}
// Get element state
const value = element.getAttribute('aria-valuenow') ||
element.getAttribute('value') ||
element.value;
const checked = element.getAttribute('aria-checked') === 'true' ||
element.checked;
const selected = element.getAttribute('aria-selected') === 'true';
const expanded = element.getAttribute('aria-expanded') === 'true';
// Generate unique ID
const id = `semantic-${role}-${label.replace(/\s+/g, '-').substring(0, 20)}`;
return {
role,
label,
bounds: {
x: bounds.x,
y: bounds.y,
width: bounds.width,
height: bounds.height,
top: bounds.top,
right: bounds.right,
bottom: bounds.bottom,
left: bounds.left
},
actions,
children: [],
id,
isButton,
isTextField,
isClickable,
isScrollable,
isToggleable,
isSelectable,
isDraggable,
isImage,
isLink,
elementType,
value,
checked,
selected,
expanded
};
}
// Build semantic tree
for (const element of Array.from(allElements)) {
const node = analyzeNode(element);
if (node) {
nodes.push(node);
}
}
// Also check for Flutter's custom semantic containers
const flutterSemantics = document.querySelectorAll('flt-semantics-container');
for (const container of Array.from(flutterSemantics)) {
// Flutter might use custom attributes
const customLabel = container.getAttribute('aria-label') ||
container.getAttribute('data-semantics-label') ||
container.textContent || '';
if (customLabel) {
const bounds = container.getBoundingClientRect();
if (bounds.width > 0 && bounds.height > 0) {
nodes.push({
role: 'custom',
label: customLabel,
bounds: bounds,
actions: ['click'],
children: [],
id: `flutter-${customLabel.replace(/\s+/g, '-').substring(0, 20)}`,
isButton: customLabel.toLowerCase().includes('button'),
isTextField: false,
isClickable: true,
isScrollable: false,
isToggleable: false,
isSelectable: false,
isDraggable: false,
isImage: false,
isLink: false,
elementType: 'custom'
});
}
}
}
// Look for visible text in span elements that might be buttons
const allSpans = document.querySelectorAll('span');
for (const span of Array.from(allSpans)) {
const text = span.innerText || span.textContent || '';
if (text && text.trim() &&
(text.toLowerCase().includes('submit') ||
text.toLowerCase().includes('report') ||
text.toLowerCase().includes('button'))) {
const bounds = span.getBoundingClientRect();
if (bounds.width > 0 && bounds.height > 0 && bounds.x >= 0 && bounds.y >= 0) {
// Check if parent has pointer cursor
let parentEl = span.parentElement;
let isClickable = false;
while (parentEl && parentEl !== document.body) {
const style = window.getComputedStyle(parentEl);
if (style.cursor === 'pointer') {
isClickable = true;
break;
}
parentEl = parentEl.parentElement;
}
if (isClickable) {
nodes.push({
role: 'button',
label: text.trim(),
bounds: bounds,
actions: ['click'],
children: [],
id: `flutter-span-${text.replace(/\s+/g, '-').substring(0, 20)}`,
isButton: true,
isTextField: false,
isClickable: true,
isScrollable: false,
isToggleable: false,
isSelectable: false,
isDraggable: false,
isImage: false,
isLink: false,
elementType: 'button'
});
}
}
}
}
// Look for all text elements that might be interactive
const allTextElements = document.querySelectorAll('div, span, p, h1, h2, h3, h4, h5, h6, li');
for (const element of Array.from(allTextElements)) {
const text = element.innerText || element.textContent || '';
if (!text || !text.trim())
continue;
const bounds = element.getBoundingClientRect();
if (bounds.width === 0 || bounds.height === 0)
continue;
// Check if element or any parent has pointer cursor
let checkEl = element;
let hasPointerCursor = false;
while (checkEl && checkEl !== document.body) {
const style = window.getComputedStyle(checkEl);
if (style.cursor === 'pointer') {
hasPointerCursor = true;
break;
}
checkEl = checkEl.parentElement;
}
if (hasPointerCursor) {
// Check if we already have this element
const existingNode = nodes.find(n => Math.abs(n.bounds.x - bounds.x) < 5 &&
Math.abs(n.bounds.y - bounds.y) < 5 &&
n.label === text.trim());
if (!existingNode) {
nodes.push({
role: 'button',
label: text.trim(),
bounds: bounds,
actions: ['click'],
children: [],
id: `flutter-clickable-${text.replace(/\s+/g, '-').substring(0, 20)}`,
isButton: text.toLowerCase().includes('button') || text.toLowerCase().includes('submit'),
isTextField: false,
isClickable: true,
isScrollable: false,
isToggleable: false,
isSelectable: false,
isDraggable: false,
isImage: false,
isLink: false,
elementType: hasPointerCursor ? 'clickable' : 'text'
});
}
}
}
return nodes;
});
}
/**
* Find semantic nodes by text content
*/
static findNodesByText(nodes, searchText) {
const results = [];
const search = searchText.toLowerCase();
for (const node of nodes) {
if (node.label.toLowerCase().includes(search)) {
results.push(node);
}
// Recursively search children
if (node.children.length > 0) {
results.push(...this.findNodesByText(node.children, searchText));
}
}
return results;
}
/**
* Generate a text representation of the semantic tree
*/
static generateSemanticMap(nodes) {
let map = '=== Flutter Semantic Tree ===\n\n';
function nodeToString(node, indent = 0) {
const prefix = ' '.repeat(indent);
const typeIcon = node.isButton ? '🔘' : node.isTextField ? '📝' : '📦';
const position = `(${Math.round(node.bounds.x)}, ${Math.round(node.bounds.y)})`;
let str = `${prefix}${typeIcon} ${node.label || node.role} ${position}`;
if (node.actions.length > 0) {
str += ` [${node.actions.join(', ')}]`;
}
str += '\n';
for (const child of node.children) {
str += nodeToString(child, indent + 1);
}
return str;
}
for (const node of nodes) {
map += nodeToString(node);
}
return map;
}
/**
* Try to infer UI structure from semantic nodes
*/
static inferUIStructure(nodes) {
const structure = {
buttons: [],
textFields: [],
navigation: [],
other: []
};
for (const node of nodes) {
if (node.isButton) {
structure.buttons.push(node);
}
else if (node.isTextField) {
structure.textFields.push(node);
}
else if (node.role === 'navigation' || node.role === 'link') {
structure.navigation.push(node);
}
else {
structure.other.push(node);
}
}
return structure;
}
}
//# sourceMappingURL=flutter-semantic-analyzer.js.map