browser-x-mcp
Version:
AI-Powered Browser Automation with Advanced Form Testing - A Model Context Provider (MCP) server that enables intelligent browser automation with form testing, element extraction, and comprehensive logging
495 lines (427 loc) • 16.9 kB
JavaScript
/**
* Virtual Canvas Extractor for Browser[X]MCP
* Extracts visible elements and their coordinates from a web page
* as structured data instead of screenshots
*
* @author Browser[X]MCP Team
* @version 1.0.0
*/
/**
* VirtualCanvasExtractor - Core class for extracting virtual canvas data
* Replaces heavy screenshots with lightweight structured element data
*/
class VirtualCanvasExtractor {
/**
* Initialize the extractor
*/
constructor() {
this.viewport = this.getViewport();
this.elements = [];
this.interactiveZones = [];
}
/**
* Get current viewport dimensions
*/
getViewport() {
return {
x: window.pageXOffset || document.documentElement.scrollLeft,
y: window.pageYOffset || document.documentElement.scrollTop,
width: window.innerWidth,
height: window.innerHeight
};
}
/**
* Check if element is visible in viewport
*/
isElementVisible(element) {
const rect = element.getBoundingClientRect();
const viewport = this.viewport;
return (
rect.width > 0 &&
rect.height > 0 &&
rect.bottom > 0 &&
rect.right > 0 &&
rect.top < viewport.height &&
rect.left < viewport.width
);
}
/**
* Check if page is scrollable and get scroll info
*/
getScrollInfo() {
const body = document.body;
const html = document.documentElement;
const documentHeight = Math.max(
body.scrollHeight, body.offsetHeight,
html.clientHeight, html.scrollHeight, html.offsetHeight
);
const documentWidth = Math.max(
body.scrollWidth, body.offsetWidth,
html.clientWidth, html.scrollWidth, html.offsetWidth
);
const viewportHeight = window.innerHeight;
const viewportWidth = window.innerWidth;
const currentScrollX = window.pageXOffset || html.scrollLeft;
const currentScrollY = window.pageYOffset || html.scrollTop;
const maxScrollX = documentWidth - viewportWidth;
const maxScrollY = documentHeight - viewportHeight;
return {
scrollable: {
vertical: documentHeight > viewportHeight,
horizontal: documentWidth > viewportWidth
},
current: {
x: currentScrollX,
y: currentScrollY
},
max: {
x: Math.max(0, maxScrollX),
y: Math.max(0, maxScrollY)
},
progress: {
x: maxScrollX > 0 ? currentScrollX / maxScrollX : 0,
y: maxScrollY > 0 ? currentScrollY / maxScrollY : 0
},
document_size: {
width: documentWidth,
height: documentHeight
},
viewport_size: {
width: viewportWidth,
height: viewportHeight
}
};
}
/**
* Determine element type based on tag and attributes
*/
getElementType(element) {
const tag = element.tagName.toLowerCase();
const type = element.type?.toLowerCase();
const role = element.getAttribute('role')?.toLowerCase();
if (tag === 'button' || type === 'button' || role === 'button') return 'button';
if (tag === 'input') return `input_${type || 'text'}`;
if (tag === 'a') return 'link';
if (tag === 'select') return 'select';
if (tag === 'textarea') return 'textarea';
if (element.onclick || element.getAttribute('onclick')) return 'clickable';
if (role) return role;
return tag;
}
/**
* Get element content (improved version)
*/
getElementContent(element) {
// Get text content and clean it
let text = element.textContent?.trim() || '';
// Remove excessive whitespace and normalize
text = text.replace(/\s+/g, ' ');
// If no text content, try alternative sources
if (!text) {
text = element.getAttribute('aria-label') ||
element.getAttribute('title') ||
element.getAttribute('alt') ||
element.getAttribute('placeholder') ||
element.getAttribute('value') || '';
}
// For images, try to get meaningful text from surrounding context
if (!text && element.tagName === 'IMG') {
const parent = element.parentElement;
if (parent) {
text = parent.getAttribute('aria-label') || parent.getAttribute('title') || '';
}
}
// For other elements, limit text length
text = text.trim();
return text.length > 100 ? text.substring(0, 100) + '...' : text;
}
/**
* Check if element is interactive (improved version)
*/
isInteractive(element) {
const interactiveTags = ['a', 'button', 'input', 'select', 'textarea', 'details', 'summary'];
const interactiveRoles = ['button', 'link', 'tab', 'menuitem', 'option', 'checkbox', 'radio'];
const interactiveTypes = ['button', 'submit', 'reset', 'checkbox', 'radio', 'file'];
// Check tag name
if (interactiveTags.includes(element.tagName.toLowerCase())) {
return true;
}
// Check ARIA role
const role = element.getAttribute('role');
if (role && interactiveRoles.includes(role.toLowerCase())) {
return true;
}
// Check input type
if (element.tagName.toLowerCase() === 'input') {
const type = element.getAttribute('type');
if (!type || interactiveTypes.includes(type.toLowerCase())) {
return true;
}
}
// Check for click handlers
if (element.onclick || element.getAttribute('onclick')) {
return true;
}
// Check for tabindex
if (element.hasAttribute('tabindex') && element.getAttribute('tabindex') !== '-1') {
return true;
}
// Check for cursor pointer style
const style = getComputedStyle(element);
if (style.cursor === 'pointer') {
return true;
}
return false;
}
/**
* Determine primary action for element (improved version)
*/
getElementAction(element) {
const tagName = element.tagName.toLowerCase();
const type = element.getAttribute('type')?.toLowerCase();
const role = element.getAttribute('role')?.toLowerCase();
// Input elements
if (tagName === 'input') {
if (['text', 'email', 'password', 'search', 'tel', 'url'].includes(type) || !type) {
return 'input';
}
if (['button', 'submit', 'reset'].includes(type)) {
return 'click';
}
if (['checkbox', 'radio'].includes(type)) {
return 'click';
}
}
// Other form elements
if (['textarea', 'select'].includes(tagName)) {
return 'input';
}
// Navigation elements
if (tagName === 'a' || role === 'link') {
return 'navigate';
}
// Buttons and clickable elements
if (tagName === 'button' || role === 'button') {
return 'click';
}
// Tab elements
if (role === 'tab') {
return 'interact';
}
// Search functionality
const content = this.getElementContent(element).toLowerCase();
if (content.includes('search') || type === 'search') {
return 'search';
}
// Default for interactive elements
if (this.isInteractive(element)) {
return 'click';
}
return 'interact';
}
/**
* Check if element is a primary action (improved version)
*/
isPrimaryAction(element) {
const text = this.getElementContent(element).toLowerCase();
const className = (element.className || '').toString().toLowerCase();
const id = (element.id || '').toLowerCase();
// Primary action keywords
const primaryKeywords = [
'sign in', 'login', 'search', 'submit', 'save', 'continue', 'next',
'buy', 'purchase', 'order', 'checkout', 'pay', 'download', 'register',
'subscribe', 'follow', 'join', 'start', 'begin', 'create', 'add'
];
// Check text content
if (primaryKeywords.some(keyword => text.includes(keyword))) {
return true;
}
// Check CSS classes for primary button patterns
const primaryClasses = ['primary', 'main', 'cta', 'action', 'submit', 'btn-primary'];
if (primaryClasses.some(cls => className.includes(cls))) {
return true;
}
// Check for search inputs
if (element.tagName.toLowerCase() === 'input' &&
(element.getAttribute('type') === 'search' ||
text.includes('search') ||
element.getAttribute('placeholder')?.toLowerCase().includes('search'))) {
return true;
}
// Check for submit type
if (element.getAttribute('type') === 'submit') {
return true;
}
// Check for prominent positioning and styling
const rect = element.getBoundingClientRect();
// Large buttons are likely primary
if (element.tagName.toLowerCase() === 'button' &&
rect.width > 100 && rect.height > 30) {
return true;
}
return false;
}
/**
* Extract all visible elements from the page
*/
extractVisibleElements() {
const elements = [];
const allElements = document.querySelectorAll('*');
// Get current scroll position
const scrollX = window.pageXOffset || document.documentElement.scrollLeft;
const scrollY = window.pageYOffset || document.documentElement.scrollTop;
for (const element of allElements) {
if (!this.isElementVisible(element)) continue;
const rect = element.getBoundingClientRect();
const isInteractive = this.isInteractive(element);
// Skip non-interactive elements that are too small
if (!isInteractive && (rect.width < 10 || rect.height < 10)) continue;
const elementData = {
id: element.id || `elem_${elements.length}`,
rect: [
Math.round(rect.left + scrollX), // Absolute X coordinate
Math.round(rect.top + scrollY), // Absolute Y coordinate
Math.round(rect.width),
Math.round(rect.height)
],
layer: parseInt(window.getComputedStyle(element).zIndex) || 0,
type: this.getElementType(element),
content: this.getElementContent(element),
interactive: isInteractive,
action: this.getElementAction(element),
primary: this.isPrimaryAction(element)
};
elements.push(elementData);
}
return elements.sort((a, b) => b.layer - a.layer); // Sort by z-index
}
/**
* Detect page type and context
*/
detectPageContext() {
const title = document.title.toLowerCase();
const url = window.location.href.toLowerCase();
const forms = document.querySelectorAll('form');
let pageType = 'general';
let mainAction = 'browse';
let flow = [];
// Detect page types
if (title.includes('search') || url.includes('search') || url.includes('google.com')) {
pageType = 'search_page';
mainAction = 'search';
flow = ['enter_query', 'submit_search', 'browse_results'];
} else if (title.includes('news') || url.includes('news')) {
pageType = 'news_page';
mainAction = 'read_news';
flow = ['browse_articles', 'read_content', 'navigate'];
} else if (title.includes('login') || title.includes('sign in') || url.includes('login')) {
pageType = 'login_page';
mainAction = 'authentication';
flow = ['enter_username', 'enter_password', 'submit_login'];
}
return {
page_type: pageType,
main_action: mainAction,
flow: flow,
forms_count: forms.length,
has_navigation: !!document.querySelector('nav'),
has_search: !!document.querySelector('input[type="search"], input[name="q"], input[placeholder*="search"]')
};
}
/**
* Extract complete virtual canvas data (alias for compatibility)
*/
extractData() {
return this.extract();
}
/**
* Debug coordinate calculation - shows viewport vs absolute coordinates
*/
debugCoordinates(elementId) {
const element = document.getElementById(elementId) ||
document.querySelector(`[data-id="${elementId}"]`);
if (!element) {
return { error: `Element ${elementId} not found` };
}
const rect = element.getBoundingClientRect();
const scrollX = window.pageXOffset || document.documentElement.scrollLeft;
const scrollY = window.pageYOffset || document.documentElement.scrollTop;
return {
element_id: elementId,
viewport_coordinates: {
left: Math.round(rect.left),
top: Math.round(rect.top),
right: Math.round(rect.right),
bottom: Math.round(rect.bottom),
width: Math.round(rect.width),
height: Math.round(rect.height)
},
absolute_coordinates: {
left: Math.round(rect.left + scrollX),
top: Math.round(rect.top + scrollY),
right: Math.round(rect.right + scrollX),
bottom: Math.round(rect.bottom + scrollY),
width: Math.round(rect.width),
height: Math.round(rect.height)
},
scroll_position: {
x: scrollX,
y: scrollY
},
center_point: {
viewport: {
x: Math.round(rect.left + rect.width / 2),
y: Math.round(rect.top + rect.height / 2)
},
absolute: {
x: Math.round(rect.left + rect.width / 2 + scrollX),
y: Math.round(rect.top + rect.height / 2 + scrollY)
}
}
};
}
/**
* Extract complete virtual canvas data
*/
extract() {
this.viewport = this.getViewport();
this.elements = this.extractVisibleElements();
return {
canvas: {
width: window.innerWidth,
height: window.innerHeight,
visible_area: this.viewport,
scroll: this.getScrollInfo(),
timestamp: Date.now()
},
visible_elements: this.elements,
context: this.detectPageContext(),
stats: {
total_elements: this.elements.length,
interactive_elements: this.elements.filter(el => el.interactive).length,
primary_actions: this.elements.filter(el => el.primary).length
}
};
}
/**
* Get data size comparison with screenshot
*/
getDataSizeComparison() {
const canvasData = this.extract();
const jsonSize = new Blob([JSON.stringify(canvasData)]).size;
// Estimate screenshot size (typical 1920x1080 PNG compressed)
const screenshotSize = 300 * 1024; // ~300KB average
return {
canvas_data_size: jsonSize,
estimated_screenshot_size: screenshotSize,
size_reduction: Math.round((screenshotSize / jsonSize) * 100) / 100,
efficiency_gain: `${Math.round(((screenshotSize - jsonSize) / screenshotSize) * 100)}%`
};
}
}
// Export for both Node.js and browser environments
if (typeof module !== 'undefined' && module.exports) {
module.exports = VirtualCanvasExtractor;
} else {
window.VirtualCanvasExtractor = VirtualCanvasExtractor;
}