browser-x-mcp
Version:
AI-Powered Browser Automation with Advanced Form Testing - A Model Context Provider (MCP) server that enables intelligent browser automation with form testing, element extraction, and comprehensive logging
1,194 lines (1,062 loc) ā¢ 77.3 kB
JavaScript
#!/usr/bin/env node
/**
* Browser[X]MCP Server
* Virtual Canvas MCP Server for fast browser automation
*
* @author Browser[X]MCP Team
* @version 1.0.0
*/
import { Server } from '@modelcontextprotocol/sdk/server/index.js';
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
import { CallToolRequestSchema, ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js';
import { chromium } from 'playwright';
import { readFileSync } from 'fs';
import { fileURLToPath } from 'url';
import { dirname, join } from 'path';
import { generateSmartAction } from './atomic-navigation.js';
// Performance metrics tracking
const performanceMetrics = {
actionSuccess: 0,
actionFailure: 0,
fallbackUsed: 0,
averageResponseTime: 0,
totalActions: 0
};
function trackActionPerformance(action, success, responseTime, usedFallback) {
performanceMetrics.totalActions++;
if (success) {
performanceMetrics.actionSuccess++;
} else {
performanceMetrics.actionFailure++;
}
if (usedFallback) {
performanceMetrics.fallbackUsed++;
}
// Update average response time
performanceMetrics.averageResponseTime =
(performanceMetrics.averageResponseTime * (performanceMetrics.totalActions - 1) + responseTime) /
performanceMetrics.totalActions;
}
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
/**
* @typedef {Object} BrowserInstance
* @property {import('playwright').Browser} browser - Playwright browser instance
* @property {import('playwright').Page} page - Current page
* @property {boolean} connected - Connection status
*/
/**
* Browser[X]MCP Server Class
* Provides virtual canvas data extraction tools via MCP protocol
*/
class BrowserXMCPServer {
constructor() {
this.server = new Server(
{
name: 'browser-x-mcp',
version: '1.0.0',
},
{
capabilities: {
tools: {},
}
}
);
this.browserInstance = null;
this.setupToolHandlers();
this.setupErrorHandlers();
}
/**
* Setup MCP tool handlers
*/
setupToolHandlers() {
// List available tools
this.server.setRequestHandler(ListToolsRequestSchema, async () => {
return {
tools: [
{
name: 'extract_virtual_canvas',
description: 'Extract virtual canvas data from current page instead of taking screenshot',
inputSchema: {
type: 'object',
properties: {
url: {
type: 'string',
description: 'URL to navigate to (optional if page already loaded)'
},
wait_for: {
type: 'string',
description: 'CSS selector to wait for before extraction (optional)'
},
include_non_interactive: {
type: 'boolean',
description: 'Include non-interactive elements in extraction',
default: false
}
}
}
},
{
name: 'navigate_browser',
description: 'Navigate browser to a specific URL',
inputSchema: {
type: 'object',
properties: {
url: {
type: 'string',
description: 'URL to navigate to'
}
},
required: ['url']
}
},
{
name: 'input_text',
description: 'Input text into a form field using virtual canvas data',
inputSchema: {
type: 'object',
properties: {
element_id: {
type: 'string',
description: 'Element ID from virtual canvas data'
},
text: {
type: 'string',
description: 'Text to input'
},
clear_first: {
type: 'boolean',
description: 'Clear existing text first',
default: true
}
},
required: ['element_id', 'text']
}
},
{
name: 'scroll_page',
description: 'Scroll the page in specified direction',
inputSchema: {
type: 'object',
properties: {
direction: {
type: 'string',
enum: ['up', 'down', 'left', 'right', 'top', 'bottom'],
description: 'Scroll direction'
},
amount: {
type: 'number',
description: 'Scroll amount in pixels (optional, defaults to viewport size)',
default: null
}
},
required: ['direction']
}
},
{
name: 'start_browser',
description: 'Start browser instance for testing',
inputSchema: {
type: 'object',
properties: {
headless: {
type: 'boolean',
description: 'Run browser in headless mode',
default: false
}
}
}
},
{
name: 'compare_with_screenshot',
description: 'Compare virtual canvas data size with screenshot for performance testing',
inputSchema: {
type: 'object',
properties: {}
}
},
{
name: 'list_navigation_elements',
description: 'Step 1: List all available interactive elements on the page with descriptions',
inputSchema: {
type: 'object',
properties: {
url: {
type: 'string',
description: 'URL to analyze (optional if page already loaded)'
},
group_by: {
type: 'string',
enum: ['purpose', 'type', 'position'],
description: 'How to group the elements',
default: 'purpose'
}
}
}
},
{
name: 'get_element_details',
description: 'Step 2: Get detailed information about specific element for precise action execution',
inputSchema: {
type: 'object',
properties: {
element_id: {
type: 'string',
description: 'Element ID from list_navigation_elements'
},
action_intent: {
type: 'string',
description: 'Intended action (click, input, hover, etc.)'
}
},
required: ['element_id']
}
},
{
name: 'execute_atomic_action',
description: 'Execute an atomic action generated by get_element_details',
inputSchema: {
type: 'object',
properties: {
action: {
type: 'object',
description: 'Atomic action object from get_element_details'
},
text_input: {
type: 'string',
description: 'Text to input (for input_text actions)'
}
},
required: ['action']
}
},
{
name: 'get_performance_metrics',
description: 'Get Browser[X]MCP server performance metrics and statistics',
inputSchema: {
type: 'object',
properties: {
random_string: {
type: 'string',
description: 'Dummy parameter for no-parameter tools'
}
}
}
},
{
name: 'help',
description: 'Get detailed usage instructions and workflow examples for Browser[X]MCP',
inputSchema: {
type: 'object',
properties: {
topic: {
type: 'string',
enum: ['overview', 'workflow', 'tools', 'examples', 'troubleshooting'],
description: 'Specific help topic (optional)',
default: 'overview'
}
}
}
},
{
name: 'batch_actions',
description: 'Execute multiple actions in batch (up to 5 actions for performance)',
inputSchema: {
type: 'object',
properties: {
actions: {
type: 'array',
maxItems: 5,
items: {
type: 'object',
properties: {
action: {
type: 'string',
enum: ['click_element_by_id', 'input_text', 'scroll_page'],
description: 'Type of action to perform'
},
element_id: {
type: 'string',
description: 'Element ID for click/input actions'
},
text: {
type: 'string',
description: 'Text to input (for input_text actions)'
},
clear_first: {
type: 'boolean',
description: 'Clear existing text first (for input_text)',
default: true
},
direction: {
type: 'string',
enum: ['up', 'down', 'left', 'right'],
description: 'Scroll direction (for scroll_page)'
},
distance: {
type: 'number',
description: 'Scroll distance in pixels (for scroll_page)',
default: 300
}
},
required: ['action']
},
description: 'Array of actions to execute in sequence'
}
},
required: ['actions']
}
},
{
name: 'evaluate_in_page',
description: 'Execute JavaScript code in the browser page context',
inputSchema: {
type: 'object',
properties: {
code: {
type: 'string',
description: 'JavaScript code to execute in the page'
}
},
required: ['code']
}
},
{
name: 'click_element_by_xpath',
description: 'Click an element using XPath selector',
inputSchema: {
type: 'object',
properties: {
xpath: {
type: 'string',
description: 'XPath selector for the element to click'
}
},
required: ['xpath']
}
}
]
};
});
// Handle tool calls
this.server.setRequestHandler(CallToolRequestSchema, async (request) => {
const { name, arguments: args } = request.params;
try {
switch (name) {
case 'start_browser':
return await this.startBrowser(args);
case 'navigate_browser':
return await this.navigateBrowser(args);
case 'extract_virtual_canvas':
return await this.extractVirtualCanvas(args);
case 'extract_xml_canvas':
return await this.extractXMLCanvas(args);
case 'click_element_by_id':
return await this.clickElementById(args);
case 'input_text':
return await this.inputText(args);
case 'scroll_page':
return await this.scrollPage(args);
case 'compare_with_screenshot':
return await this.compareWithScreenshot(args);
case 'list_navigation_elements':
return await this.listNavigationElements(args);
case 'get_element_details':
return await this.getElementDetails(args);
case 'execute_atomic_action':
return await this.executeAtomicAction(args);
case 'get_performance_metrics':
return await this.getPerformanceMetrics(args);
case 'help':
return await this.getHelp(args);
case 'batch_actions':
return await this.executeBatchActions(args);
case 'evaluate_in_page':
return await this.evaluateInPage(args);
case 'click_element_by_xpath':
return await this.clickElementByXPath(args);
default:
throw new Error(`Unknown tool: ${name}`);
}
} catch (error) {
return {
content: [
{
type: 'text',
text: `Error executing ${name}: ${error.message}`
}
],
isError: true
};
}
});
}
/**
* Setup error handlers
*/
setupErrorHandlers() {
this.server.onerror = (error) => {
console.error('[MCP Error]:', error);
};
process.on('SIGINT', async () => {
await this.cleanup();
process.exit(0);
});
}
/**
* Start browser instance
* @param {Object} args - Arguments
* @returns {Promise<Object>} MCP response
*/
async startBrowser(args = {}) {
try {
// Check if browser is already running and connected
if (this.browserInstance?.browser && this.browserInstance.connected) {
try {
// Test if browser is still alive
await this.browserInstance.page.evaluate(() => true);
return {
content: [
{
type: 'text',
text: `š Browser already running (headless: ${args.headless ?? false})`
}
]
};
} catch {
// Browser is dead, clean up
this.browserInstance = null;
}
}
// Close any existing browser before starting new one
if (this.browserInstance?.browser) {
await this.browserInstance.browser.close();
}
const browser = await chromium.launch({
headless: args.headless ?? false,
args: [
'--no-sandbox',
'--disable-web-security',
'--disable-blink-features=AutomationControlled',
'--disable-features=VizDisplayCompositor',
'--disable-dev-shm-usage',
'--no-first-run',
'--no-default-browser-check',
'--disable-background-timer-throttling',
'--disable-backgrounding-occluded-windows',
'--disable-renderer-backgrounding',
'--disable-field-trial-config',
'--disable-back-forward-cache',
'--disable-ipc-flooding-protection'
]
});
const page = await browser.newPage({
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'
});
// Remove webdriver property
await page.addInitScript(() => {
Object.defineProperty(navigator, 'webdriver', {
get: () => undefined,
});
// Add plugins to appear more realistic
Object.defineProperty(navigator, 'plugins', {
get: () => [1, 2, 3, 4, 5],
});
// Add languages
Object.defineProperty(navigator, 'languages', {
get: () => ['en-US', 'en'],
});
// Override permissions
const originalQuery = window.navigator.permissions.query;
window.navigator.permissions.query = (parameters) => (
parameters.name === 'notifications' ?
Promise.resolve({ state: Notification.permission }) :
originalQuery(parameters)
);
});
this.browserInstance = {
browser,
page,
connected: true
};
return {
content: [
{
type: 'text',
text: `ā
Browser started successfully (headless: ${args.headless ?? false})`
}
]
};
} catch (error) {
throw new Error(`Failed to start browser: ${error.message}`);
}
}
/**
* Navigate browser to URL
* @param {Object} args - Arguments
* @returns {Promise<Object>} MCP response
*/
async navigateBrowser(args) {
if (!this.browserInstance?.page) {
throw new Error('Browser not started. Call start_browser first.');
}
try {
await this.browserInstance.page.goto(args.url, {
waitUntil: 'networkidle'
});
return {
content: [
{
type: 'text',
text: `ā
Navigated to: ${args.url}`
}
]
};
} catch (error) {
throw new Error(`Failed to navigate: ${error.message}`);
}
}
/**
* Extract virtual canvas data from current page
* @param {Object} args - Arguments
* @returns {Promise<Object>} MCP response
*/
async extractVirtualCanvas(args = {}) {
if (!this.browserInstance?.page) {
throw new Error('Browser not started. Call start_browser first.');
}
try {
// Navigate if URL provided
if (args.url) {
await this.browserInstance.page.goto(args.url, {
waitUntil: 'networkidle'
});
}
// Wait for specific element if requested
if (args.wait_for) {
await this.browserInstance.page.waitForSelector(args.wait_for, {
timeout: 10000
});
}
// Inject our VirtualCanvasExtractor
const extractorCode = readFileSync(
join(__dirname, '../extractor/VirtualCanvasExtractor.js'),
'utf-8'
);
// Execute extraction in browser context
const canvasData = await this.browserInstance.page.evaluate((extractorCode) => {
// Inject the extractor class
eval(extractorCode);
// Create instance and extract data
const extractor = new VirtualCanvasExtractor();
const data = extractor.extract();
const sizeComparison = extractor.getDataSizeComparison();
return {
canvas_data: data,
size_comparison: sizeComparison,
extraction_timestamp: Date.now()
};
}, extractorCode);
return {
content: [
{
type: 'text',
text: `šÆ Virtual Canvas Extracted Successfully!\n\n` +
`š STATS:\n` +
`ā¢ Total elements: ${canvasData.canvas_data.stats.total_elements}\n` +
`ā¢ Interactive elements: ${canvasData.canvas_data.stats.interactive_elements}\n` +
`ā¢ Primary actions: ${canvasData.canvas_data.stats.primary_actions}\n` +
`ā¢ Page type: ${canvasData.canvas_data.context.page_type}\n\n` +
`š¾ SIZE EFFICIENCY:\n` +
`ā¢ Canvas data: ${canvasData.size_comparison.canvas_data_size} bytes\n` +
`ā¢ Screenshot estimate: ${canvasData.size_comparison.estimated_screenshot_size} bytes\n` +
`ā¢ Reduction factor: ${canvasData.size_comparison.size_reduction}x\n` +
`ā¢ Efficiency gain: ${canvasData.size_comparison.efficiency_gain}\n\n` +
`šÆ KEY INTERACTIVE ELEMENTS:\n` +
canvasData.canvas_data.visible_elements
.filter(el => el.interactive)
.slice(0, 10) // Show first 10
.map((el, i) => `${i+1}. ${el.type}: "${el.content}" at [${el.rect.join(',')}] (${el.action})`)
.join('\n')
},
{
type: 'text',
text: `\nš COMPLETE VIRTUAL CANVAS DATA:\n\`\`\`json\n${JSON.stringify(canvasData.canvas_data, null, 2)}\`\`\`
`
}
]
};
} catch (error) {
throw new Error(`Failed to extract virtual canvas: ${error.message}`);
}
}
/**
* Extract XML canvas representation (clean & efficient)
* @param {Object} args - Arguments
* @returns {Promise<Object>} MCP response with minified XML
*/
async extractXMLCanvas(args = {}) {
if (!this.browserInstance?.page) {
throw new Error('Browser not started. Call start_browser first.');
}
try {
// Navigate if URL provided
if (args.url) {
await this.browserInstance.page.goto(args.url, {
waitUntil: 'networkidle'
});
}
// Wait for specific element if requested
if (args.wait_for) {
await this.browserInstance.page.waitForSelector(args.wait_for, {
timeout: 10000
});
}
// Inject our XML Canvas Extractor
const extractorCode = readFileSync(
join(__dirname, '../extractor/XMLCanvasExtractor.js'),
'utf-8'
);
// Execute extraction in browser context
const xmlData = await this.browserInstance.page.evaluate((extractorCode) => {
// Inject the extractor class
eval(extractorCode);
// Create instance and extract data
const extractor = new XMLCanvasExtractor();
return extractor.extract();
}, extractorCode);
return {
content: [
{
type: 'text',
text: `šÆ Compact XML Canvas Extracted Successfully!\n\n` +
`š STATS:\n` +
`ā¢ Total elements: ${xmlData.stats.total_elements}\n` +
`ā¢ Interactive elements: ${xmlData.stats.interactive_elements}\n` +
`ā¢ Format: ID-based Compact XML\n\n` +
`š¾ SIZE EFFICIENCY:\n` +
`ā¢ XML size: ${xmlData.stats.xml_size} bytes\n` +
`ā¢ Coordinate map: ${xmlData.stats.coordinate_map_size} bytes\n` +
`ā¢ Total size: ${xmlData.stats.total_size} bytes\n` +
`ā¢ Screenshot estimate: ${xmlData.stats.estimated_screenshot_size} bytes\n` +
`ā¢ Compression ratio: ${xmlData.stats.compression_ratio}x\n` +
`ā¢ Efficiency gain: ${xmlData.stats.efficiency_gain}%\n\n` +
`š SCROLL INFO:\n` +
`ā¢ Scrollable: ${xmlData.scroll.scrollable ? 'Yes' : 'No'}\n` +
`ā¢ Current Y: ${xmlData.scroll.current_y}\n` +
`ā¢ Max Y: ${xmlData.scroll.max_y}\n` +
`ā¢ Progress: ${Math.round(xmlData.scroll.progress * 100)}%\n\n` +
`š² ELEMENT IDs: ${Object.keys(xmlData.element_index).join(', ')}`
},
{
type: 'text',
text: `\nš COMPACT XML:\n${xmlData.xml}`
}
]
};
} catch (error) {
throw new Error(`Failed to extract SVG canvas: ${error.message}`);
}
}
/**
* Click element by ID (using coordinate lookup)
* @param {Object} args - Arguments containing element_id
* @returns {Promise<Object>} MCP response
*/
async clickElementById(args = {}) {
if (!this.browserInstance?.page) {
throw new Error('Browser not started. Call start_browser first.');
}
const { element_id } = args;
if (!element_id) {
throw new Error('element_id parameter is required');
}
try {
// Extract current canvas to get coordinate map
const extractorCode = readFileSync(
join(__dirname, '../extractor/XMLCanvasExtractor.js'),
'utf-8'
);
// Get coordinates for the element ID
const coordinates = await this.browserInstance.page.evaluate(({extractorCode, elementId}) => {
// Inject the extractor class
eval(extractorCode);
// Create instance and get coordinates
const extractor = new XMLCanvasExtractor();
extractor.extractElements(); // Build coordinate map
return extractor.getClickCoordinates(elementId);
}, {extractorCode, elementId: element_id});
// Scroll element to center
await this.browserInstance.page.evaluate((coords) => {
const targetY = coords.y - window.innerHeight / 2;
window.scrollTo(0, Math.max(0, targetY));
}, coordinates);
// Wait for scroll to complete
await this.browserInstance.page.waitForTimeout(50);
// Click at the center coordinates
await this.browserInstance.page.mouse.click(coordinates.x, coordinates.y);
return {
content: [
{
type: 'text',
text: `šÆ Element clicked successfully!\n\n` +
`š Element ID: ${element_id}\n` +
`š Click coordinates: (${coordinates.x}, ${coordinates.y})\n` +
`š Element bounds: ${coordinates.bounds.width}x${coordinates.bounds.height} at (${coordinates.bounds.left}, ${coordinates.bounds.top})`
}
]
};
} catch (error) {
console.error('ā Click element by ID error:', error);
throw new Error(`Failed to click element ${element_id}: ${error.message}`);
}
}
/**
* Extract virtual canvas data for internal use
*/
async extractVirtualCanvasData() {
const extractorCode = readFileSync(
join(__dirname, '../extractor/VirtualCanvasExtractor.js'),
'utf-8'
);
return await this.browserInstance.page.evaluate((extractorCode) => {
eval(extractorCode);
const extractor = new VirtualCanvasExtractor();
return extractor.extractData();
}, extractorCode);
}
/**
* Get performance metrics
* @param {Object} args - Arguments
* @returns {Promise<Object>} MCP response
*/
async getPerformanceMetrics(args) {
try {
const successRate = performanceMetrics.totalActions > 0 ?
(performanceMetrics.actionSuccess / performanceMetrics.totalActions * 100).toFixed(1) : 0;
const fallbackRate = performanceMetrics.totalActions > 0 ?
(performanceMetrics.fallbackUsed / performanceMetrics.totalActions * 100).toFixed(1) : 0;
return {
content: [
{
type: "text",
text: `š BROWSER[X]MCP PERFORMANCE METRICS
š Success Rate: ${successRate}% (${performanceMetrics.actionSuccess}/${performanceMetrics.totalActions})
š Failure Rate: ${(100 - successRate).toFixed(1)}% (${performanceMetrics.actionFailure}/${performanceMetrics.totalActions})
š Fallback Usage: ${fallbackRate}% (${performanceMetrics.fallbackUsed}/${performanceMetrics.totalActions})
ā±ļø Average Response Time: ${performanceMetrics.averageResponseTime.toFixed(0)}ms
š¢ Total Actions: ${performanceMetrics.totalActions}`
}
]
};
} catch (error) {
console.error('ā Error getting performance metrics:', error);
return {
content: [
{
type: "text",
text: `ā Error getting performance metrics: ${error.message}`
}
]
};
}
}
/**
* Click element using virtual canvas data
* @param {Object} args - Arguments
* @returns {Promise<Object>} MCP response
*/
/**
* Input text using virtual canvas data
* @param {Object} args - Arguments
* @returns {Promise<Object>} MCP response
*/
async inputText(args) {
if (!this.browserInstance?.page) {
throw new Error('Browser not started. Call start_browser first.');
}
try {
const { element_id, text, clear_first = true } = args;
if (!element_id || !text) {
throw new Error('Both element_id and text are required');
}
// Use existing click_element_by_id to get coordinates
const clickResult = await this.clickElementById({ element_id });
if (!clickResult || clickResult.isError) {
throw new Error(`Failed to locate element "${element_id}": ${clickResult?.content?.[0]?.text || 'Unknown error'}`);
}
// Extract coordinates from click result
const resultText = clickResult.content[0].text;
const coordMatch = resultText.match(/Click coordinates: \((\d+), (\d+)\)/);
if (!coordMatch) {
throw new Error(`Failed to extract coordinates from click result for element "${element_id}"`);
}
const x = parseInt(coordMatch[1]);
const y = parseInt(coordMatch[2]);
console.log(`āØļø Inputting text into element "${element_id}" at coordinates [${x}, ${y}]`);
// Click on the element first to focus it
await this.browserInstance.page.mouse.click(x, y);
await this.browserInstance.page.waitForTimeout(100); // Increased pause for focus
// Clear existing text if requested
if (clear_first) {
// More robust clearing method
await this.browserInstance.page.evaluate((elementId) => {
const element = document.getElementById(elementId);
if (element) {
element.value = '';
element.focus();
element.select();
// Trigger events to notify frameworks
element.dispatchEvent(new Event('input', { bubbles: true }));
element.dispatchEvent(new Event('change', { bubbles: true }));
}
}, element_id);
await this.browserInstance.page.waitForTimeout(50); // Increased pause after JS clear
// Fallback: keyboard clear
await this.browserInstance.page.keyboard.press('Control+a');
await this.browserInstance.page.waitForTimeout(30); // Pause after select all
await this.browserInstance.page.keyboard.press('Delete');
await this.browserInstance.page.waitForTimeout(50); // Increased pause after delete
}
// Type the text
await this.browserInstance.page.keyboard.type(text, { delay: 20 }); // Added typing delay
await this.browserInstance.page.waitForTimeout(100); // Increased final pause
return {
content: [
{
type: 'text',
text: `ā
Input text "${text}" into element "${element_id}" at coordinates [${x}, ${y}]`
}
]
};
} catch (error) {
console.error('ā Input text error:', error);
throw new Error(`Failed to input text: ${error.message}`);
}
}
/**
* Scroll page in specified direction
* @param {Object} args - Arguments containing direction and optional amount
* @returns {Promise<Object>} MCP response
*/
async scrollPage(args) {
if (!this.browserInstance?.page) {
throw new Error('Browser not started. Call start_browser first.');
}
try {
const { direction, amount } = args;
if (!direction) {
throw new Error('Direction parameter is required');
}
console.log(`š Scrolling ${direction}${amount ? ` by ${amount}px` : ''}`);
// Get current scroll info and document dimensions
const scrollInfo = await this.browserInstance.page.evaluate(() => {
const body = document.body;
const html = document.documentElement;
const documentHeight = Math.max(
body.scrollHeight, body.offsetHeight,
html.clientHeight, html.scrollHeight, html.offsetHeight
);
const documentWidth = Math.max(
body.scrollWidth, body.offsetWidth,
html.clientWidth, html.scrollWidth, html.offsetWidth
);
const viewportHeight = window.innerHeight;
const viewportWidth = window.innerWidth;
return {
current: { x: window.scrollX, y: window.scrollY },
max: {
x: Math.max(0, documentWidth - viewportWidth),
y: Math.max(0, documentHeight - viewportHeight)
},
viewport: { width: viewportWidth, height: viewportHeight }
};
});
let newScrollY = scrollInfo.current.y;
let newScrollX = scrollInfo.current.x;
switch (direction) {
case 'up':
newScrollY = Math.max(0, scrollInfo.current.y - (amount || scrollInfo.viewport.height));
break;
case 'down':
newScrollY = Math.min(scrollInfo.max.y, scrollInfo.current.y + (amount || scrollInfo.viewport.height));
break;
case 'left':
newScrollX = Math.max(0, scrollInfo.current.x - (amount || scrollInfo.viewport.width));
break;
case 'right':
newScrollX = Math.min(scrollInfo.max.x, scrollInfo.current.x + (amount || scrollInfo.viewport.width));
break;
case 'top':
newScrollY = 0;
newScrollX = 0;
break;
case 'bottom':
newScrollY = scrollInfo.max.y;
break;
default:
throw new Error(`Unknown scroll direction: ${direction}. Valid directions: up, down, left, right, top, bottom`);
}
// Perform the scroll
await this.browserInstance.page.evaluate((newScrollX, newScrollY) => {
window.scrollTo(newScrollX, newScrollY);
}, newScrollX, newScrollY);
// Wait for scroll to complete
await this.browserInstance.page.waitForTimeout(100);
return {
content: [
{
type: 'text',
text: `ā
Scrolled ${direction}${amount ? ` by ${amount}px` : ''}\nFrom: [${scrollInfo.current.x}, ${scrollInfo.current.y}] to [${newScrollX}, ${newScrollY}]`
}
]
};
} catch (error) {
console.error('ā Scroll page error:', error);
throw new Error(`Failed to scroll page: ${error.message}`);
}
}
/**
* Compare virtual canvas with screenshot for performance testing
* @param {Object} args - Arguments
* @returns {Promise<Object>} MCP response
*/
async compareWithScreenshot(args) {
if (!this.browserInstance?.page) {
throw new Error('Browser not started. Call start_browser first.');
}
try {
// Take screenshot for comparison
const screenshot = await this.browserInstance.page.screenshot({
type: 'png',
fullPage: false
});
// Extract virtual canvas data
const extractorCode = readFileSync(
join(__dirname, '../extractor/VirtualCanvasExtractor.js'),
'utf-8'
);
const canvasData = await this.browserInstance.page.evaluate((extractorCode) => {
eval(extractorCode);
const extractor = new VirtualCanvasExtractor();
return extractor.getDataSizeComparison();
}, extractorCode);
const screenshotSize = screenshot.length;
const canvasSize = canvasData.canvas_data_size;
const actualReduction = Math.round((screenshotSize / canvasSize) * 100) / 100;
const actualEfficiency = Math.round(((screenshotSize - canvasSize) / screenshotSize) * 100);
return {
content: [
{
type: 'text',
text: `š¬ BROWSER[X]MCP PERFORMANCE TEST RESULTS:\n\n` +
`šø ACTUAL SCREENSHOT:\n` +
`ā¢ Size: ${screenshotSize} bytes (${Math.round(screenshotSize/1024)} KB)\n\n` +
`šÆ VIRTUAL CANVAS:\n` +
`ā¢ Size: ${canvasSize} bytes (${Math.round(canvasSize/1024)} KB)\n\n` +
`ā” PERFORMANCE GAINS:\n` +
`ā¢ Actual reduction: ${actualReduction}x smaller\n` +
`ā¢ Actual efficiency: ${actualEfficiency}% reduction\n` +
`ā¢ Estimated reduction: ${canvasData.size_reduction}x\n` +
`ā¢ Estimated efficiency: ${canvasData.efficiency_gain}\n\n` +
`ā
Virtual Canvas is ${actualReduction}x smaller than actual screenshot!`
}
]
};
} catch (error) {
throw new Error(`Failed to compare with screenshot: ${error.message}`);
}
}
/**
* List navigation elements (simplified overview for Step 1)
* @param {Object} args - Arguments
* @returns {Promise<Object>} MCP response
*/
async listNavigationElements(args) {
try {
if (!this.browserInstance?.page) {
throw new Error('Browser not started. Call start_browser first.');
}
const { url, group_by = 'purpose' } = args;
// Navigate to URL if provided
if (url && this.browserInstance.page.url() !== url) {
console.log(`š Navigating to: ${url}`);
await this.browserInstance.page.goto(url, { waitUntil: 'networkidle2', timeout: 30000 });
}
// Extract virtual canvas data
const extractorCode = readFileSync(
join(__dirname, '../extractor/VirtualCanvasExtractor.js'),
'utf-8'
);
const canvasData = await this.browserInstance.page.evaluate((extractorCode) => {
eval(extractorCode);
const extractor = new VirtualCanvasExtractor();
return extractor.extractData();
}, extractorCode);
// Filter and simplify elements for overview
const interactiveElements = canvasData.visible_elements
.filter(el => el.interactive)
.map(el => ({
id: el.id,
type: el.type,
content: el.content?.substring(0, 50) + (el.content?.length > 50 ? '...' : ''),
action: el.action,
primary: el.primary
}));
// Group elements based on group_by parameter
let groupedElements = {};
if (group_by === 'purpose') {
groupedElements = {
'Navigation': interactiveElements.filter(el =>
/home|menu|guide|nav|back|forward/i.test(el.content) || el.type === 'nav'
),
'Search & Input': interactiveElements.filter(el =>
el.action === 'input_text' || /search|input|text/i.test(el.content)
),
'Actions & Buttons': interactiveElements.filter(el =>
el.action === 'click' && el.type === 'button'
),
'Links': interactiveElements.filter(el =>
el.type === 'link' || el.type === 'a'
),
'Other Interactive': interactiveElements.filter(el =>
!['Navigation', 'Search & Input', 'Actions & Buttons', 'Links'].includes(el.category)
)
};
} else if (group_by === 'type') {
groupedElements = interactiveElements.reduce((acc, el) => {
const type = el.type.charAt(0).toUpperCase() + el.type.slice(1);
if (!acc[type]) acc[type] = [];
acc[type].push(el);
return acc;
}, {});
} else {
// group_by === 'position' - simplified without exact coordinates
groupedElements = {
'Top Area': interactiveElements.filter(el => el.primary),
'Content Area': interactiveElements.filter(el => !el.primary)
};
}
// Format output
let output = `š INTERACTIVE ELEMENTS OVERVIEW (${interactiveElements.length} found)\n\n`;
for (const [groupName, elements] of Object.entries(groupedElements)) {
if (elements.length > 0) {
output += `š ${groupName} (${elements.length}):\n`;
elements.forEach((el, idx) => {
output += ` ${idx + 1}. [${el.id}] ${el.type.toUpperCase()}: "${el.content}"\n`;
});
output += '\n';
}
}
output += `š§ NEXT STEP: Use 'get_element_details' with element_id to get precise action details.\n`;
output += `Example: get_element_details(element_id="button_1", action_intent="click")`;
return {
content: [
{
type: 'text',
text: output
}
]
};
} catch (error) {
console.error('ā Error listing navigation elements:', error);