@puberty-labs/clits
Version:
CLiTS (Chrome Logging and Inspection Tool Suite) is a powerful Node.js library for automated Chrome browser testing, logging, and inspection. It provides a comprehensive suite of tools for monitoring network requests, console logs, DOM mutations, and more
1,151 lines (1,125 loc) • 71.6 kB
JavaScript
// BSD: Chrome DevTools Protocol automation for navigation, interaction, and scripted automation tasks.
// Provides browser automation capabilities including navigation, element interaction, and screenshot capture.
import CDP from 'chrome-remote-interface';
import { writeFileSync, readFileSync } from 'fs';
import { ChromeExtractor } from './chrome-extractor.js';
import { createLogger, format, transports } from 'winston';
import fetch from 'node-fetch';
const logger = createLogger({
level: 'info',
format: format.combine(format.timestamp(), format.json()),
transports: [
new transports.Console({
format: format.combine(format.colorize(), format.simple())
})
]
});
export class ChromeAutomation {
constructor(port = ChromeAutomation.DEFAULT_PORT, host = ChromeAutomation.DEFAULT_HOST) {
this.port = port;
this.host = host;
}
async navigate(options) {
const client = await this.connectToChrome();
try {
const { Page, Runtime } = client;
await Page.enable();
await Runtime.enable();
logger.info(`Navigating to: ${options.url}`);
// Get current URL before navigation for comparison
const beforeNavigation = await Runtime.evaluate({
expression: 'window.location.href'
});
const initialUrl = beforeNavigation.result.value;
await Page.navigate({ url: options.url });
// Wait for page load
await Page.loadEventFired();
// CRITICAL FIX: Verify actual URL after navigation
const afterNavigation = await Runtime.evaluate({
expression: 'window.location.href'
});
const actualUrl = afterNavigation.result.value;
// Parse expected path from target URL for comparison
const targetUrl = new URL(options.url);
const actualUrlObj = new URL(actualUrl);
// Check if navigation was successful
const navigationSuccessful =
// Either the URL changed to the target
(actualUrl !== initialUrl &&
(actualUrlObj.pathname === targetUrl.pathname ||
actualUrl.includes(targetUrl.pathname) ||
actualUrl === options.url)) ||
// Or we were already at the target URL
(actualUrl === options.url ||
actualUrlObj.pathname === targetUrl.pathname ||
actualUrl.includes(targetUrl.pathname));
if (!navigationSuccessful) {
throw new Error(`Navigation verification failed. Expected URL containing "${targetUrl.pathname}", but got "${actualUrl}". Initial URL was "${initialUrl}".`);
}
logger.info(`Navigation verified: ${actualUrl}`);
// Wait for specific selector if provided
if (options.waitForSelector) {
await this.waitForSelector(client, options.waitForSelector, options.timeout);
}
// Take screenshot if requested
if (options.screenshotPath) {
await this.takeScreenshot(client, options.screenshotPath);
}
logger.info('Navigation completed successfully');
return { actualUrl, success: true };
}
finally {
await client.close();
}
}
async interact(options) {
const client = await this.connectToChrome();
const networkLogs = [];
const result = {
success: false,
timestamp: new Date().toISOString()
};
try {
const { Page, Runtime, Network, DOM, Input } = client;
await Page.enable();
await Runtime.enable();
await DOM.enable();
// Note: Input domain doesn't require enable() - it's ready to use immediately
// Input is used in private methods like clickElement() and typeInElement()
// Enable network monitoring if requested
if (options.captureNetwork) {
await Network.enable();
Network.requestWillBeSent((params) => {
networkLogs.push({
type: 'request',
timestamp: Date.now(),
...params
});
});
Network.responseReceived((params) => {
networkLogs.push({
type: 'response',
timestamp: Date.now(),
...params
});
});
}
// Perform click interaction
if (options.clickSelector) {
if (options.useJavaScriptExpression && options.jsExpression) {
// Use JavaScript expression for element selection
await this.clickElementByJavaScript(client, options.jsExpression);
}
else {
await this.clickElement(client, options.clickSelector);
}
}
// Perform type interaction
if (options.typeSelector && options.typeText) {
await this.typeInElement(client, options.typeSelector, options.typeText);
}
// Perform toggle interaction
if (options.toggleSelector) {
await this.toggleElement(client, options.toggleSelector);
}
// Ensure Input is referenced to avoid linter warning
void Input;
// Wait for selector after interaction
if (options.waitForSelector) {
await this.waitForSelector(client, options.waitForSelector, options.timeout);
}
// Take screenshot if requested
if (options.screenshotPath) {
await this.takeScreenshot(client, options.screenshotPath);
}
// Handle tab discovery
if (options.discoverTabs) {
const tabs = await this.discoverTabLabels(client);
// If a specific tab label or pattern is specified, click it
if (options.clickSelector && tabs.length > 0) {
let targetTab = null;
if (options.tabLabelPattern) {
const regex = new RegExp(options.tabLabelPattern, 'i');
targetTab = tabs.find(tab => regex.test(tab.label));
}
else if (options.clickSelector) {
targetTab = tabs.find(tab => tab.label.toLowerCase().includes(options.clickSelector.toLowerCase()));
}
if (targetTab) {
await this.clickElement(client, targetTab.selector);
logger.info(`Clicked tab: ${targetTab.label}`);
}
}
// Output discovered tabs
console.log(JSON.stringify({
success: true,
tabCount: tabs.length,
tabs: tabs
}, null, 2));
}
// Handle save button discovery
if (options.findSaveButton) {
const saveButton = await this.findSaveButton(client, options.customSavePatterns);
console.log(JSON.stringify({
success: true,
saveButton: saveButton
}, null, 2));
}
// Enhanced screenshot and visual features
if (options.takeScreenshot || options.screenshotPath) {
const screenshotData = await this.takeEnhancedScreenshot(client, options);
result.screenshotPath = screenshotData.path;
result.screenshotBase64 = screenshotData.base64;
}
// Generate selector map if requested
if (options.selectorMap) {
result.selectorMap = await this.generateSelectorMap(client);
}
// Collect metadata if requested
if (options.withMetadata) {
result.metadata = await this.collectPageMetadata(client);
}
// Handle new tab management commands
if (options.switchTabIndex !== undefined) {
await this.switchToTab(options.switchTabIndex);
logger.info(`Switched to tab ${options.switchTabIndex}`);
}
if (options.tabNext) {
await this.switchToNextTab();
logger.info('Switched to next tab');
}
if (options.tabPrev) {
await this.switchToPrevTab();
logger.info('Switched to previous tab');
}
// Handle keyboard commands
if (options.keyCommand) {
await this.sendKeyCommand(client, options.keyCommand);
logger.info(`Sent keyboard command: ${options.keyCommand}`);
}
// Include network logs in result
if (options.captureNetwork && networkLogs.length > 0) {
result.networkLogs = networkLogs;
logger.info(`Captured ${networkLogs.length} network events during interaction`);
}
result.success = true;
logger.info('Interaction completed successfully');
return result;
}
catch (error) {
result.error = error instanceof Error ? error.message : String(error);
result.success = false;
logger.error(`Interaction failed: ${result.error}`);
return result;
}
finally {
await client.close();
}
}
async runAutomation(options) {
const script = JSON.parse(readFileSync(options.scriptPath, 'utf8'));
const result = {
success: false,
completedSteps: 0,
totalSteps: script.steps.length,
screenshots: [],
networkLogs: [],
monitoringData: [],
timestamp: new Date().toISOString()
};
const client = await this.connectToChrome();
try {
const { Page, Runtime, Network, DOM, Input } = client;
await Page.enable();
await Runtime.enable();
await DOM.enable();
// Note: Input domain doesn't require enable() - it's ready to use immediately
// Enable monitoring if requested
if (options.monitor || script.options?.monitor) {
this.chromeExtractor = new ChromeExtractor({
port: options.chromePort || this.port,
host: options.chromeHost || this.host,
includeNetwork: script.options?.captureNetwork !== false,
includeConsole: true
});
}
// Enable network monitoring if needed
if (script.options?.captureNetwork !== false) {
await Network.enable();
Network.requestWillBeSent((params) => {
result.networkLogs.push({
type: 'request',
timestamp: Date.now(),
...params
});
});
Network.responseReceived((params) => {
result.networkLogs.push({
type: 'response',
timestamp: Date.now(),
...params
});
});
}
// Ensure Input is referenced to avoid linter warning
void Input;
// Execute each step
for (let i = 0; i < script.steps.length; i++) {
const step = script.steps[i];
logger.info(`Executing step ${i + 1}/${script.steps.length}: ${step.action}`);
try {
await this.executeStep(client, step, result);
result.completedSteps++;
}
catch (error) {
result.error = `Failed at step ${i + 1}: ${error instanceof Error ? error.message : String(error)}`;
logger.error(result.error);
break;
}
}
result.success = result.completedSteps === result.totalSteps;
// Save results if requested
if (options.saveResultsPath) {
writeFileSync(options.saveResultsPath, JSON.stringify(result, null, 2));
logger.info(`Results saved to: ${options.saveResultsPath}`);
}
logger.info(`Automation completed: ${result.completedSteps}/${result.totalSteps} steps successful`);
return result;
}
catch (error) {
result.error = error instanceof Error ? error.message : String(error);
logger.error(`Automation failed: ${result.error}`);
return result;
}
finally {
await client.close();
}
}
async connectToChrome() {
try {
// Auto-launch Chrome if needed (using same logic as working clits-inspect)
await this.launchChromeIfNeeded();
// Smart target selection with priority logic (using same logic as working clits-inspect)
const target = await this.autoSelectTarget();
// Connect to the selected target using its webSocketDebuggerUrl
const client = await CDP({
target: target.webSocketDebuggerUrl || target.id
});
return client;
}
catch (error) {
throw new Error(`Failed to connect to Chrome: ${error instanceof Error ? error.message : String(error)}`);
}
}
async checkChromeConnection() {
try {
const response = await fetch(`http://${this.host}:${this.port}/json/version`);
if (response.ok) {
const version = await response.json();
logger.info(`✅ Existing Chrome debugging session detected: ${version.Browser || 'Chrome'}`);
return true;
}
return false;
}
catch (error) {
logger.debug(`No Chrome debugging session found on port ${this.port}: ${error instanceof Error ? error.message : String(error)}`);
return false;
}
}
async launchChromeIfNeeded() {
// First check if Chrome is already running with debugging
const isChrome = await this.checkChromeConnection();
if (isChrome) {
logger.info('✅ Using existing Chrome debugging session');
return;
}
// Check if Chrome process exists but not responding to debugging
if (process.platform === 'darwin') {
try {
const { execSync } = await import('child_process');
const chromeProcesses = execSync('ps aux | grep Chrome | grep remote-debugging-port', { encoding: 'utf8' });
if (chromeProcesses.trim().length > 0) {
logger.warn('Chrome process with remote debugging found but not responding. Waiting for it to be ready...');
// Wait longer for existing Chrome to become ready
for (let i = 0; i < 10; i++) {
await new Promise(resolve => setTimeout(resolve, 1000));
const isReady = await this.checkChromeConnection();
if (isReady) {
logger.info('✅ Existing Chrome session is now ready');
return;
}
}
logger.warn('Existing Chrome session did not become ready, will launch new instance');
}
}
catch (error) {
logger.debug('Could not check for existing Chrome processes:', error);
}
}
// Only launch if no Chrome debugging session exists
logger.info('No Chrome debugging session found. Launching Chrome with debugging enabled...');
if (process.platform === 'darwin') {
const { spawn } = await import('child_process');
const chromePath = '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
const args = [
`--remote-debugging-port=${this.port}`,
'--user-data-dir=/tmp/chrome-debug-clits',
'--no-first-run',
'--no-default-browser-check',
'--disable-web-security',
'--disable-features=VizDisplayCompositor'
];
const chromeProcess = spawn(chromePath, args, {
detached: true,
stdio: 'ignore'
});
chromeProcess.unref();
logger.info(`Chrome launched with PID: ${chromeProcess.pid}`);
// Wait for Chrome to start and verify connection
for (let i = 0; i < 15; i++) {
await new Promise(resolve => setTimeout(resolve, 1000));
const isReady = await this.checkChromeConnection();
if (isReady) {
logger.info('✅ New Chrome debugging session is ready');
return;
}
logger.debug(`Waiting for Chrome to be ready... attempt ${i + 1}/15`);
}
throw new Error('Chrome launched but debugging session did not become available within 15 seconds');
}
else {
throw new Error('Chrome is not running with remote debugging enabled. Start Chrome with --remote-debugging-port=9222');
}
}
async autoSelectTarget() {
const response = await fetch(`http://${this.host}:${this.port}/json/list`);
const targets = await response.json();
const pageTargets = targets.filter((t) => t.type === 'page');
if (pageTargets.length === 0) {
throw new Error('No Chrome page targets found. Please open a tab in Chrome with --remote-debugging-port=9222');
}
if (pageTargets.length === 1) {
return pageTargets[0];
}
// Smart target selection - prefer localhost/development URLs (same logic as working clits-inspect)
const localTargets = pageTargets.filter(t => t.url.includes('localhost') ||
t.url.includes('127.0.0.1') ||
t.url.includes('local') ||
t.url.startsWith('http://localhost') ||
t.url.startsWith('https://localhost'));
if (localTargets.length > 0) {
return localTargets[0];
}
// Filter out chrome:// URLs as fallback
const nonChromeTargets = pageTargets.filter(t => !t.url.startsWith('chrome://'));
if (nonChromeTargets.length > 0) {
return nonChromeTargets[0];
}
// Last resort: return first available
return pageTargets[0];
}
async executeStep(client, step, result) {
const timeout = step.timeout || ChromeAutomation.DEFAULT_TIMEOUT;
switch (step.action) {
case 'navigate': {
if (!step.url)
throw new Error('Navigate step requires url');
// Use internal navigation logic for consistency
const tempOptions = {
url: step.url,
timeout: timeout
};
// Call navigate method but handle return internally
const tempAutomation = new ChromeAutomation(this.port, this.host);
await tempAutomation.navigate(tempOptions);
break;
}
case 'wait':
if (!step.selector)
throw new Error('Wait step requires selector');
await this.waitForSelector(client, step.selector, timeout);
break;
case 'click':
if (!step.selector)
throw new Error('Click step requires selector');
await this.clickElement(client, step.selector);
break;
case 'type':
if (!step.selector || !step.text)
throw new Error('Type step requires selector and text');
await this.typeInElement(client, step.selector, step.text);
break;
case 'toggle':
if (!step.selector)
throw new Error('Toggle step requires selector');
await this.toggleElement(client, step.selector);
break;
case 'screenshot':
if (!step.path)
throw new Error('Screenshot step requires path');
await this.takeScreenshot(client, step.path);
result.screenshots.push(step.path);
break;
case 'discover_links': {
// Add discover_links action support
const links = await this.discoverAllLinks(client);
// Save links data to results
if (!result.monitoringData)
result.monitoringData = [];
result.monitoringData.push({
type: 'discovered_links',
timestamp: new Date().toISOString(),
data: links
});
// If path is specified, save to file
if (step.path) {
writeFileSync(step.path, JSON.stringify(links, null, 2));
}
break;
}
case 'interact': {
// Add interact action support - handle visual selection methods like CLI does
let clickSelector = step.selector;
let useJavaScriptExpression = false;
let jsExpression = '';
if (step.clickText) {
jsExpression = await this.findElementByText(step.clickText);
useJavaScriptExpression = true;
clickSelector = '__JS_EXPRESSION__';
}
else if (step.clickColor) {
jsExpression = await this.findElementByColor(step.clickColor);
useJavaScriptExpression = true;
clickSelector = '__JS_EXPRESSION__';
}
else if (step.clickRegion) {
jsExpression = await this.findElementByRegion(step.clickRegion);
useJavaScriptExpression = true;
clickSelector = '__JS_EXPRESSION__';
}
else if (step.clickDescription) {
jsExpression = await this.findElementByDescription(step.clickDescription);
useJavaScriptExpression = true;
clickSelector = '__JS_EXPRESSION__';
}
const interactionOptions = {
clickSelector,
screenshotPath: step.screenshotPath,
takeScreenshot: !!step.screenshotPath,
timeout: step.timeout,
chromePort: this.port,
chromeHost: this.host,
useJavaScriptExpression,
jsExpression
};
const interactResult = await this.interact(interactionOptions);
// Add screenshot to results if taken
if (interactResult.screenshotPath) {
result.screenshots.push(interactResult.screenshotPath);
}
// Add network logs if captured
if (interactResult.networkLogs) {
if (!result.monitoringData)
result.monitoringData = [];
result.monitoringData.push({
type: 'network_logs',
timestamp: new Date().toISOString(),
data: interactResult.networkLogs
});
}
if (!interactResult.success) {
throw new Error(interactResult.error || 'Interaction failed');
}
break;
}
case 'click-text': {
if (!step.text)
throw new Error('Click-text step requires text');
const jsExpression = await this.findElementByText(step.text);
await this.clickElementByJavaScript(client, jsExpression);
// Wait if specified
if (step.wait) {
await new Promise(resolve => setTimeout(resolve, step.wait));
}
// Take screenshot if requested
if (step.screenshotPath) {
await this.takeScreenshot(client, step.screenshotPath);
result.screenshots.push(step.screenshotPath);
}
break;
}
case 'click-region': {
if (!step.region)
throw new Error('Click-region step requires region');
const jsExpression = await this.findElementByRegion(step.region);
await this.clickElementByJavaScript(client, jsExpression);
// Wait if specified
if (step.wait) {
await new Promise(resolve => setTimeout(resolve, step.wait));
}
// Take screenshot if requested
if (step.screenshotPath) {
await this.takeScreenshot(client, step.screenshotPath);
result.screenshots.push(step.screenshotPath);
}
break;
}
case 'key': {
if (!step.keyCommand)
throw new Error('Key step requires keyCommand');
await this.sendKeyCommand(client, step.keyCommand);
// Wait if specified
if (step.wait) {
await new Promise(resolve => setTimeout(resolve, step.wait));
}
// Take screenshot if requested
if (step.screenshotPath) {
await this.takeScreenshot(client, step.screenshotPath);
result.screenshots.push(step.screenshotPath);
}
break;
}
case 'switch-tab': {
if (step.tabIndex === undefined)
throw new Error('Switch-tab step requires tabIndex');
await this.switchToTab(step.tabIndex);
// Wait if specified
if (step.wait) {
await new Promise(resolve => setTimeout(resolve, step.wait));
}
// Take screenshot if requested
if (step.screenshotPath) {
await this.takeScreenshot(client, step.screenshotPath);
result.screenshots.push(step.screenshotPath);
}
break;
}
case 'tab-next': {
await this.switchToNextTab();
// Wait if specified
if (step.wait) {
await new Promise(resolve => setTimeout(resolve, step.wait));
}
// Take screenshot if requested
if (step.screenshotPath) {
await this.takeScreenshot(client, step.screenshotPath);
result.screenshots.push(step.screenshotPath);
}
break;
}
case 'tab-prev': {
await this.switchToPrevTab();
// Wait if specified
if (step.wait) {
await new Promise(resolve => setTimeout(resolve, step.wait));
}
// Take screenshot if requested
if (step.screenshotPath) {
await this.takeScreenshot(client, step.screenshotPath);
result.screenshots.push(step.screenshotPath);
}
break;
}
default:
throw new Error(`Unknown step action: ${step.action}`);
}
}
escapeSelector(selector) {
// Escape backslashes, single quotes, and double quotes for safe JavaScript string interpolation
return selector.replace(/\\/g, '\\\\').replace(/'/g, "\\'").replace(/"/g, '\\"');
}
async findElementWithFallback(client, selector) {
logger.debug(`Finding element with selector: ${selector}`);
// CRITICAL FIX: Use the same approach as the working clickElementDirect method
const result = await client.Runtime.evaluate({
expression: `
JSON.stringify((() => {
const selector = '${selector.replace(/'/g, "\\'")}';
let element = null;
// Strategy 1: Direct CSS selector (this is what was failing before)
try {
element = document.querySelector(selector);
} catch (e) {
// Invalid CSS selector, skip
}
// Strategy 2: Attribute-based search for common patterns
if (!element && !selector.includes(':')) {
const patterns = [
'[data-testid="' + selector + '"]',
'[aria-label*="' + selector + '"]',
'[class*="' + selector + '"]'
];
for (const pattern of patterns) {
try {
element = document.querySelector(pattern);
if (element) break;
} catch (e) {
// Invalid selector, skip
}
}
}
// Strategy 3: Text search for simple text
if (!element && selector.length < 50 && !selector.includes('[') && !selector.includes('.') && !selector.includes('#')) {
const clickables = Array.from(document.querySelectorAll('a, button, [role="button"], [onclick]'));
element = clickables.find(el => el.textContent && el.textContent.includes(selector));
}
if (!element) {
return { error: 'Element not found' };
}
const rect = element.getBoundingClientRect();
if (rect.width === 0 || rect.height === 0) {
return { error: 'Element not visible' };
}
return {
x: rect.left + rect.width / 2,
y: rect.top + rect.height / 2,
width: rect.width,
height: rect.height
};
})())
`
});
if (result.result?.value) {
const elementData = JSON.parse(result.result.value);
if (elementData.error) {
throw new Error(`Element not found: "${selector}". ${elementData.error}`);
}
logger.info(`✅ Element found: ${selector}`);
return elementData;
}
throw new Error(`Element not found: "${selector}". No result from evaluation.`);
}
async waitForSelector(client, selector, timeout = ChromeAutomation.DEFAULT_TIMEOUT) {
const startTime = Date.now();
let attempts = 0;
const maxAttempts = Math.min(50, timeout / 500); // Limit attempts based on timeout
while (Date.now() - startTime < timeout && attempts < maxAttempts) {
try {
const elementInfo = await this.findElementWithFallback(client, selector);
if (elementInfo) {
logger.info(`✅ Element found: ${selector} (attempt ${attempts + 1})`);
return;
}
}
catch (error) {
// CRITICAL FIX: Don't retry on specific errors that won't resolve
if (error instanceof Error && error.message.includes('Element not found:')) {
throw error; // Immediately fail for definitive not-found errors
}
logger.debug(`⏳ Attempt ${attempts + 1}: ${error instanceof Error ? error.message : String(error)}`);
}
attempts++;
await new Promise(resolve => setTimeout(resolve, 500)); // Longer delay between attempts
}
throw new Error(`Timeout waiting for selector: "${selector}" after ${attempts} attempts in ${Date.now() - startTime}ms. Element may not exist or be accessible.`);
}
async clickElement(client, selector) {
// First ensure element exists
await this.waitForSelector(client, selector);
// Try to find element with fallback strategies
const elementInfo = await this.findElementWithFallback(client, selector);
if (!elementInfo) {
throw new Error(`Element not found with any strategy: ${selector}`);
}
const { x, y } = elementInfo;
// Perform click
await client.Input.dispatchMouseEvent({
type: 'mousePressed',
x,
y,
button: 'left',
clickCount: 1
});
await client.Input.dispatchMouseEvent({
type: 'mouseReleased',
x,
y,
button: 'left',
clickCount: 1
});
logger.info(`Clicked element: ${selector}`);
}
async clickElementByJavaScript(client, jsExpression) {
logger.info(`Attempting to click element using JavaScript expression`);
try {
// Evaluate the provided JavaScript expression directly
const result = await client.Runtime.evaluate({
expression: jsExpression
});
if (!result.result?.value) {
throw new Error('No result from element evaluation');
}
// Parse JSON result if the expression returns JSON
let resultData = result.result.value;
if (typeof resultData === 'string') {
try {
resultData = JSON.parse(resultData);
}
catch (e) {
// If it's not JSON, treat it as a simple value
logger.error('Failed to parse result as JSON:', resultData);
throw new Error('Invalid result format from JavaScript expression');
}
}
if (resultData.error) {
throw new Error(`Element operation failed: ${resultData.error}`);
}
if (resultData.success) {
logger.info(`Successfully performed ${resultData.method} operation: "${resultData.text}"`);
if (resultData.url) {
logger.info(`Navigated to: ${resultData.url}`);
}
// Handle coordinate-based clicking if needed
if (resultData.method === 'coordinates' && resultData.x && resultData.y) {
await client.Input.dispatchMouseEvent({
type: 'mousePressed',
x: resultData.x,
y: resultData.y,
button: 'left',
clickCount: 1
});
await client.Input.dispatchMouseEvent({
type: 'mouseReleased',
x: resultData.x,
y: resultData.y,
button: 'left',
clickCount: 1
});
logger.info(`Performed coordinate click at (${resultData.x}, ${resultData.y})`);
}
return;
}
throw new Error('Unexpected result from element operation');
}
catch (error) {
logger.error('Click operation failed:', error);
throw error;
}
}
async typeInElement(client, selector, text) {
// First click on the element to focus it
await this.clickElement(client, selector);
// Clear existing content
await client.Input.dispatchKeyEvent({
type: 'keyDown',
key: 'Control'
});
await client.Input.dispatchKeyEvent({
type: 'char',
text: 'a'
});
await client.Input.dispatchKeyEvent({
type: 'keyUp',
key: 'Control'
});
// Type the new text
for (const char of text) {
await client.Input.dispatchKeyEvent({
type: 'char',
text: char
});
}
logger.info(`Typed text in element: ${selector}`);
}
async toggleElement(client, selector) {
// Simply click the toggle element
await this.clickElement(client, selector);
logger.info(`Toggled element: ${selector}`);
}
async takeScreenshot(client, path) {
const screenshot = await client.Page.captureScreenshot({
format: 'png',
fullPage: true
});
writeFileSync(path, screenshot.data, 'base64');
logger.info(`Screenshot saved: ${path}`);
}
/**
* Discovers all tab labels in a dialog or tabbed interface
* @param client CDP client instance
* @returns Array of tab labels with their selectors
*/
async discoverTabLabels(client) {
const result = await client.Runtime.evaluate({
expression: `
JSON.stringify((function() {
const tabs = Array.from(document.querySelectorAll('.MuiTab-root, [role="tab"], .MuiTabs-root .MuiTab-root'));
return tabs.map((tab, index) => ({
label: tab.textContent?.trim() || tab.getAttribute('aria-label') || tab.getAttribute('title') || '',
selector: tab.getAttribute('data-testid') ||
(tab.getAttribute('aria-label') ? '[aria-label="' + tab.getAttribute('aria-label') + '"]' : '') ||
(tab.textContent?.trim() ? '[role="tab"]:nth-child(' + (index + 1) + ')' : '') ||
'.MuiTab-root:nth-child(' + (index + 1) + ')',
index: index,
isActive: tab.getAttribute('aria-selected') === 'true' || tab.classList.contains('Mui-selected'),
isDisabled: tab.getAttribute('aria-disabled') === 'true' || tab.classList.contains('Mui-disabled')
})).filter(tab => tab.label);
})())
`
});
if (result.result.value) {
return JSON.parse(result.result.value);
}
return [];
}
/**
* Finds the best save button in a dialog using multiple strategies
* @param client CDP client instance
* @param customText Optional custom text patterns for save buttons
* @returns Save button element info or null
*/
async findSaveButton(client, customText) {
const savePatterns = customText || ['save', 'update', 'apply', 'ok', 'done', 'submit', 'confirm'];
const result = await client.Runtime.evaluate({
expression: `
JSON.stringify((function() {
const savePatterns = ${JSON.stringify(savePatterns)};
const patternRegex = /${savePatterns.join('|')}/i;
// Strategy 1: Find by text content
const buttonsByText = Array.from(document.querySelectorAll('.MuiDialog-root button, .MuiModal-root button, [role="dialog"] button'))
.filter(btn => patternRegex.test(btn.textContent?.trim() || ''));
if (buttonsByText.length > 0) {
const rect = buttonsByText[0].getBoundingClientRect();
return {
x: rect.left + rect.width / 2,
y: rect.top + rect.height / 2,
strategy: 'text-content',
text: buttonsByText[0].textContent?.trim()
};
}
// Strategy 2: Find by type="submit"
const submitButtons = Array.from(document.querySelectorAll('.MuiDialog-root button[type="submit"], .MuiModal-root button[type="submit"], .MuiDialogActions-root button[type="submit"]'));
if (submitButtons.length > 0) {
const rect = submitButtons[0].getBoundingClientRect();
return {
x: rect.left + rect.width / 2,
y: rect.top + rect.height / 2,
strategy: 'submit-type',
text: submitButtons[0].textContent?.trim()
};
}
// Strategy 3: Find by aria-label or title
const ariaButtons = Array.from(document.querySelectorAll('.MuiDialog-root button, .MuiModal-root button'))
.filter(btn => patternRegex.test(btn.getAttribute('aria-label') || '') || patternRegex.test(btn.getAttribute('title') || ''));
if (ariaButtons.length > 0) {
const rect = ariaButtons[0].getBoundingClientRect();
return {
x: rect.left + rect.width / 2,
y: rect.top + rect.height / 2,
strategy: 'aria-label',
ariaLabel: ariaButtons[0].getAttribute('aria-label'),
title: ariaButtons[0].getAttribute('title')
};
}
// Strategy 4: Find icon buttons with save-like icons
const iconButtons = Array.from(document.querySelectorAll('.MuiDialog-root .MuiIconButton-root, .MuiDialog-root button'))
.filter(btn => {
const svg = btn.querySelector('.MuiSvgIcon-root, svg');
if (!svg) return false;
const ariaLabel = btn.getAttribute('aria-label') || '';
const title = btn.getAttribute('title') || '';
return patternRegex.test(ariaLabel) || patternRegex.test(title);
});
if (iconButtons.length > 0) {
const rect = iconButtons[0].getBoundingClientRect();
return {
x: rect.left + rect.width / 2,
y: rect.top + rect.height / 2,
strategy: 'icon-button',
ariaLabel: iconButtons[0].getAttribute('aria-label')
};
}
// Strategy 5: Find in action areas (last resort)
const actionButtons = Array.from(document.querySelectorAll('.MuiDialogActions-root button:not([disabled]), .modal-footer button:not([disabled])'));
if (actionButtons.length === 1) {
const rect = actionButtons[0].getBoundingClientRect();
return {
x: rect.left + rect.width / 2,
y: rect.top + rect.height / 2,
strategy: 'single-action-button',
text: actionButtons[0].textContent?.trim(),
warning: 'Only one enabled button found in action area - assuming it is the save button'
};
}
// Strategy 6: Primary button in action area
const primaryButtons = Array.from(document.querySelectorAll('.MuiDialogActions-root .MuiButton-containedPrimary, .MuiDialogActions-root .MuiButton-root[color="primary"]'));
if (primaryButtons.length > 0) {
const rect = primaryButtons[0].getBoundingClientRect();
return {
x: rect.left + rect.width / 2,
y: rect.top + rect.height / 2,
strategy: 'primary-button',
text: primaryButtons[0].textContent?.trim()
};
}
return { error: 'No save button found with any strategy' };
})())
`
});
if (result.result.value) {
const buttonInfo = JSON.parse(result.result.value);
if (buttonInfo.error) {
logger.warn(`Save button detection failed: ${buttonInfo.error}`);
return null;
}
logger.info(`Save button found using strategy: ${buttonInfo.strategy}`, buttonInfo);
return buttonInfo;
}
return null;
}
// Enhanced screenshot capabilities with base64 support
async takeEnhancedScreenshot(client, options) {
const screenshotOptions = {
format: 'png'
};
if (options.fullPageScreenshot) {
screenshotOptions.fullPage = true;
}
const screenshot = await client.Page.captureScreenshot(screenshotOptions);
const result = {};
if (options.base64Output) {
result.base64 = screenshot.data;
}
if (options.screenshotPath && !options.base64Output) {
writeFileSync(options.screenshotPath, screenshot.data, 'base64');
result.path = options.screenshotPath;
logger.info(`Enhanced screenshot saved: ${options.screenshotPath}`);
}
return result;
}
// Generate map of clickable elements with coordinates
async generateSelectorMap(client) {
const result = await client.Runtime.evaluate({
expression: `
JSON.stringify((function() {
const selectors = [
'button', 'a', 'input[type="button"]', 'input[type="submit"]',
'[role="button"]', '[onclick]', '.MuiButton-root', '.MuiIconButton-root',
'.MuiFab-root', '.MuiToggleButton-root', '[data-testid]',
'input[type="checkbox"]', 'input[type="radio"]', 'select'
];
const elements = [];
selectors.forEach(selector => {
const nodes = document.querySelectorAll(selector);
nodes.forEach((element, index) => {
const rect = element.getBoundingClientRect();
const style = getComputedStyle(element);
// Only include visible elements
if (rect.width > 0 && rect.height > 0 &&
style.display !== 'none' && style.visibility !== 'hidden') {
elements.push({
selector: selector + ':nth-of-type(' + (index + 1) + ')',
text: element.textContent?.trim() ||
element.getAttribute('aria-label') ||
element.getAttribute('title') ||
element.getAttribute('alt') || '',
coordinates: {
x: rect.left + rect.width / 2,
y: rect.top + rect.height / 2
},
boundingBox: {
x: rect.left,
y: rect.top,
width: rect.width,
height: rect.height
}
});
}
});
});
return elements;
})())
`
});
if (result.result.value) {
return JSON.parse(result.result.value);
}
return [];
}
// Collect page metadata
async collectPageMetadata(client) {
const result = await client.Runtime.evaluate({
expression: `
JSON.stringify({
viewport: {
width: window.innerWidth,
height: window.innerHeight
},
url: window.location.href,
title: document.title,
elementCount: document.querySelectorAll('*').length
})
`
});
if (result.result.value) {
return JSON.parse(result.result.value);
}
return {
viewport: { width: 0, height: 0 },
url: '',
title: '',
elementCount: 0
};
}
// Visual element selection methods - FIXED IMPLEMENTATIONS
async findElementByText(text) {
// Return JavaScript expression using the proven working method from chrome-control
return `JSON.stringify((() => {
const searchText = '${text}';
let element = null;
// Strategy: Text search in clickable elements (proven to work)
const clickables = Array.from(document.querySelectorAll('a, button, [role="button"], [onclick]'));
element = clickables.find(el => el.textContent && el.textContent.includes(searchText));
if (!element) {
return { error: 'Element not found' };
}
const rect = element.getBoundingClientRect();
if (rect.width === 0 || rect.height === 0) {
return { error: 'Element not visible' };
}
// For links, navigate directly (this is the key difference!)
if (element.tagName.toLowerCase() === 'a' && element.href) {
window.location.href = element.href;
return {
success: true,
method: 'navigation',
url: element.href,
text: element.textContent?.trim() || ''
};
}
// For other elements, try direct click
try {
element.click();
return {
success: true,
method: 'click',
text: element.textContent?.trim() || ''
};
} catch (e) {
return {
success: true,
method: 'coordinates',
x: rect.left + rect.width / 2,
y: rect.top + rect.height / 2,
text: element.textContent?.trim() || ''
};
}
})())`;
}
async findElementByColor(color) {
// Return JavaScript expression using the proven working approach for color-based selection
return `JSON.stringify((() => {
const targetColor = '${color}';
let element = null;
// Strategy: Find clickable elements with matching colors
const clickables = Array.from(document.querySelectorAll('a, button, [role="button"], [onclick]'));
for (const el of clickables) {
const style = getComputedStyle(el);
const rect = el.getBoundingClientRect();
// Only check visible elements
if (rect.width > 0 && rect.height > 0) {
if (style.color === targetColor ||
style.backgroundColor === targetColor ||
style.borderColor === targetColor ||
style.fill === targetColor) {
element = el;