ai-debug-local-mcp
Version:
🎯 ENHANCED AI GUIDANCE v4.1.2: Dramatically improved tool descriptions help AI users choose the right tools instead of 'close enough' options. Ultra-fast keyboard automation (10x speed), universal recording, multi-ecosystem debugging support, and compreh
304 lines • 13.9 kB
JavaScript
/**
* Flutter UI Pattern Library
* Coordinate-based patterns for common Flutter UI elements
* Used when semantic tree or DOM text search fails (canvas-rendered elements)
*/
export class FlutterUIPatterns {
static patterns = [
{
name: 'submit-button',
type: 'button',
keywords: ['submit', 'send', 'save', 'confirm', 'done', 'next', 'continue'],
confidence: 0.8,
findStrategy: async (page, target) => {
return await page.evaluate((searchText) => {
// Common submit button patterns in Flutter apps
const patterns = [
// Pattern 1: Bottom of screen (common for submit buttons)
{ bottom: true, heightRange: [40, 80], widthRange: [100, 400] },
// Pattern 2: Right side of form
{ rightAlign: true, heightRange: [35, 60], widthRange: [80, 200] },
// Pattern 3: Center of screen (modal/dialog submit)
{ center: true, heightRange: [40, 70], widthRange: [120, 300] }
];
const viewport = {
width: window.innerWidth,
height: window.innerHeight
};
// Try each pattern
for (const pattern of patterns) {
if (pattern.bottom) {
// Look in bottom 20% of screen
const y = viewport.height * 0.8 + (viewport.height * 0.1); // Middle of bottom area
const x = viewport.width / 2; // Center horizontally
// Check if this area has high contrast (likely a button)
const canvas = document.querySelector('canvas');
if (canvas) {
const ctx = canvas.getContext('2d');
if (ctx) {
// Sample pixels to detect button-like areas
const imageData = ctx.getImageData(x - 50, y - 20, 100, 40);
// Inline contrast detection
const data = imageData.data;
let contrastPoints = 0;
// Sample every 10th pixel for performance
for (let i = 0; i < data.length; i += 40) { // 4 channels * 10
const r = data[i];
const g = data[i + 1];
const b = data[i + 2];
// Check if this pixel is significantly different from gray
const avg = (r + g + b) / 3;
const deviation = Math.abs(r - avg) + Math.abs(g - avg) + Math.abs(b - avg);
if (deviation > 30) { // High contrast
contrastPoints++;
}
}
// If more than 20% of sampled pixels show high contrast
const hasHighContrast = contrastPoints > (data.length / 40) * 0.2;
if (hasHighContrast) {
return { x, y };
}
}
}
}
if (pattern.rightAlign) {
// Look in right 30% of screen, middle vertical area
const x = viewport.width * 0.85;
const y = viewport.height * 0.5;
return { x, y };
}
if (pattern.center) {
// Look in center of screen
const x = viewport.width * 0.5;
const y = viewport.height * 0.5;
return { x, y };
}
}
// Fallback: Try bottom-right corner (common for FABs)
if (searchText.toLowerCase().includes('add') || searchText.toLowerCase().includes('new')) {
return {
x: viewport.width - 56 - 16, // FAB size + margin
y: viewport.height - 56 - 16
};
}
return null;
}, target);
}
},
{
name: 'navigation-button',
type: 'navigation',
keywords: ['back', 'menu', 'home', 'settings', 'profile'],
confidence: 0.9,
findStrategy: async (page, target) => {
return await page.evaluate((searchText) => {
const lowerSearch = searchText.toLowerCase();
// Navigation patterns
if (lowerSearch.includes('back')) {
// Back button is typically top-left
return { x: 24, y: 24 };
}
if (lowerSearch.includes('menu') || lowerSearch.includes('drawer')) {
// Menu/hamburger is typically top-left or top-right
return { x: 24, y: 24 };
}
if (lowerSearch.includes('settings') || lowerSearch.includes('profile')) {
// Often top-right
return { x: window.innerWidth - 24, y: 24 };
}
// Bottom navigation items
const bottomNavItems = ['home', 'search', 'cart', 'profile', 'account'];
if (bottomNavItems.some(item => lowerSearch.includes(item))) {
const itemIndex = bottomNavItems.findIndex(item => lowerSearch.includes(item));
const itemCount = 4; // Assume 4 items in bottom nav
const itemWidth = window.innerWidth / itemCount;
const x = (itemIndex * itemWidth) + (itemWidth / 2);
const y = window.innerHeight - 28; // Middle of bottom nav
return { x, y };
}
return null;
}, target);
}
},
{
name: 'input-field',
type: 'input',
keywords: ['name', 'email', 'phone', 'address', 'password', 'search', 'field', 'input'],
confidence: 0.7,
findStrategy: async (page, target) => {
return await page.evaluate((searchText) => {
// Input fields are typically in the middle 60% of the screen
const viewport = {
width: window.innerWidth,
height: window.innerHeight
};
// Common input field positions based on form layout
const positions = [
{ x: viewport.width * 0.5, y: viewport.height * 0.3 }, // First field
{ x: viewport.width * 0.5, y: viewport.height * 0.4 }, // Second field
{ x: viewport.width * 0.5, y: viewport.height * 0.5 }, // Third field
{ x: viewport.width * 0.5, y: viewport.height * 0.6 }, // Fourth field
];
// Try to match based on common field order
const lowerSearch = searchText.toLowerCase();
if (lowerSearch.includes('name') || lowerSearch.includes('first')) {
return positions[0];
}
if (lowerSearch.includes('email') || lowerSearch.includes('second')) {
return positions[1];
}
if (lowerSearch.includes('phone') || lowerSearch.includes('password')) {
return positions[2];
}
// Default to first position
return positions[0];
}, target);
}
},
{
name: 'dropdown-select',
type: 'dropdown',
keywords: ['select', 'choose', 'dropdown', 'picker', 'state', 'country', 'category'],
confidence: 0.7,
findStrategy: async (page, target) => {
return await page.evaluate((searchText) => {
// Dropdowns are often right-aligned with a down arrow
const viewport = {
width: window.innerWidth,
height: window.innerHeight
};
// Look for dropdown in middle area of screen
return {
x: viewport.width * 0.7, // Right side of typical dropdown
y: viewport.height * 0.4
};
}, target);
}
},
{
name: 'checkbox-toggle',
type: 'checkbox',
keywords: ['check', 'agree', 'accept', 'remember', 'subscribe'],
confidence: 0.7,
findStrategy: async (page, target) => {
return await page.evaluate((searchText) => {
const viewport = {
width: window.innerWidth,
height: window.innerHeight
};
// Checkboxes are typically left-aligned
const lowerSearch = searchText.toLowerCase();
if (lowerSearch.includes('agree') || lowerSearch.includes('accept')) {
// Terms/conditions checkboxes are often near bottom
return {
x: viewport.width * 0.1, // Left side
y: viewport.height * 0.7
};
}
// General checkbox position
return {
x: viewport.width * 0.1,
y: viewport.height * 0.5
};
}, target);
}
},
{
name: 'floating-action-button',
type: 'button',
keywords: ['add', 'create', 'new', 'fab', '+'],
confidence: 0.9,
findStrategy: async (page, target) => {
return await page.evaluate(() => {
// FAB is almost always bottom-right
return {
x: window.innerWidth - 56 - 16, // Standard FAB size + margin
y: window.innerHeight - 56 - 16
};
});
}
}
];
/**
* Find UI element using pattern matching
*/
static async findElement(page, target) {
const matches = [];
const lowerTarget = target.toLowerCase();
for (const pattern of this.patterns) {
// Check if any keywords match
const keywordMatch = pattern.keywords.some(keyword => lowerTarget.includes(keyword) || keyword.includes(lowerTarget));
if (keywordMatch) {
const location = await pattern.findStrategy(page, target);
if (location) {
matches.push({
pattern,
location,
confidence: pattern.confidence
});
}
}
}
// Sort by confidence
return matches.sort((a, b) => b.confidence - a.confidence);
}
/**
* Add custom pattern for specific app
*/
static addPattern(pattern) {
this.patterns.push(pattern);
}
/**
* Detect high contrast areas (likely buttons)
*/
static detectHighContrast(imageData) {
const data = imageData.data;
let contrastPoints = 0;
// Sample every 10th pixel for performance
for (let i = 0; i < data.length; i += 40) { // 4 channels * 10
const r = data[i];
const g = data[i + 1];
const b = data[i + 2];
// Check if this pixel is significantly different from gray
const avg = (r + g + b) / 3;
const deviation = Math.abs(r - avg) + Math.abs(g - avg) + Math.abs(b - avg);
if (deviation > 30) { // High contrast
contrastPoints++;
}
}
// If more than 20% of sampled pixels show high contrast
return contrastPoints > (data.length / 40) * 0.2;
}
/**
* Get all available patterns
*/
static getPatterns() {
return [...this.patterns];
}
/**
* Clear all patterns (useful for testing)
*/
static clearPatterns() {
this.patterns = [];
}
/**
* Generate click coordinates with slight randomization
* (to appear more human-like)
*/
static humanizeCoordinates(x, y) {
// Add small random offset (±3 pixels)
const offsetX = (Math.random() - 0.5) * 6;
const offsetY = (Math.random() - 0.5) * 6;
return {
x: Math.round(x + offsetX),
y: Math.round(y + offsetY)
};
}
}
// Make detectHighContrast available in browser context
if (typeof window !== 'undefined') {
window.FlutterUIPatterns = {
detectHighContrast: FlutterUIPatterns.detectHighContrast
};
}
//# sourceMappingURL=flutter-ui-patterns.js.map