@puberty-labs/clits
Version:
CLiTS (Chrome Logging and Inspection Tool Suite) is a powerful Node.js library for AI-controlled Chrome browser automation, testing, and inspection. Features enhanced CSS selector support (:contains(), XPath), dry-run mode, element discovery tools, and co
1,113 lines (1,091 loc) • 79.6 kB
JavaScript
// BSD: Chrome DevTools Protocol automation for navigation, interaction, and scripted automation tasks.
// Provides browser automation capabilities including navigation, element interaction, and screenshot capture.
import CDP from 'chrome-remote-interface';
import { writeFileSync, readFileSync } from 'fs';
import { ChromeExtractor } from './chrome-extractor.js';
import { createLogger, format, transports } from 'winston';
import fetch from 'node-fetch';
const logger = createLogger({
level: 'info',
format: format.combine(format.timestamp(), format.json()),
transports: [
new transports.Console({
format: format.combine(format.colorize(), format.simple())
})
]
});
export class ChromeAutomation {
constructor(port = ChromeAutomation.DEFAULT_PORT, host = ChromeAutomation.DEFAULT_HOST) {
this.port = port;
this.host = host;
}
async navigate(options) {
const client = await this.connectToChrome();
try {
const { Page, Runtime } = client;
await Page.enable();
await Runtime.enable();
logger.info(`Navigating to: ${options.url}`);
// Get current URL before navigation for comparison
const beforeNavigation = await Runtime.evaluate({
expression: 'window.location.href'
});
const initialUrl = beforeNavigation.result.value;
await Page.navigate({ url: options.url });
// Wait for page load
await Page.loadEventFired();
// CRITICAL FIX: Verify actual URL after navigation
const afterNavigation = await Runtime.evaluate({
expression: 'window.location.href'
});
const actualUrl = afterNavigation.result.value;
// Parse expected path from target URL for comparison
const targetUrl = new URL(options.url);
const actualUrlObj = new URL(actualUrl);
// Check if navigation was successful
const navigationSuccessful =
// Either the URL changed to the target
(actualUrl !== initialUrl &&
(actualUrlObj.pathname === targetUrl.pathname ||
actualUrl.includes(targetUrl.pathname) ||
actualUrl === options.url)) ||
// Or we were already at the target URL
(actualUrl === options.url ||
actualUrlObj.pathname === targetUrl.pathname ||
actualUrl.includes(targetUrl.pathname));
if (!navigationSuccessful) {
throw new Error(`Navigation verification failed. Expected URL containing "${targetUrl.pathname}", but got "${actualUrl}". Initial URL was "${initialUrl}".`);
}
logger.info(`Navigation verified: ${actualUrl}`);
// Wait for specific selector if provided
if (options.waitForSelector) {
await this.waitForSelector(client, options.waitForSelector, options.timeout);
}
// Take screenshot if requested
if (options.screenshotPath) {
await this.takeScreenshot(client, options.screenshotPath);
}
logger.info('Navigation completed successfully');
return { actualUrl, success: true };
}
finally {
await client.close();
}
}
async interact(options) {
const client = await this.connectToChrome();
const networkLogs = [];
const result = {
success: false,
timestamp: new Date().toISOString()
};
try {
const { Page, Runtime, Network, DOM, Input } = client;
await Page.enable();
await Runtime.enable();
await DOM.enable();
// Note: Input domain doesn't require enable() - it's ready to use immediately
// Input is used in private methods like clickElement() and typeInElement()
// Enable network monitoring if requested
if (options.captureNetwork) {
await Network.enable();
Network.requestWillBeSent((params) => {
networkLogs.push({
type: 'request',
timestamp: Date.now(),
...params
});
});
Network.responseReceived((params) => {
networkLogs.push({
type: 'response',
timestamp: Date.now(),
...params
});
});
}
// Perform click interaction
if (options.clickSelector) {
if (options.dryRun) {
// Dry-run mode: just find the element and report what would be clicked
logger.info(`🔍 DRY-RUN: Testing selector "${options.clickSelector}"`);
try {
const elementInfo = await this.findElementWithFallback(client, options.clickSelector);
if (elementInfo) {
logger.info(`✅ DRY-RUN: Element found and would be clicked`);
logger.info(` Strategy: ${elementInfo.strategy}`);
logger.info(` Tag: <${elementInfo.tagName}>`);
logger.info(` Text: "${elementInfo.text}"`);
logger.info(` Position: (${elementInfo.x}, ${elementInfo.y})`);
logger.info(` Size: ${elementInfo.width}x${elementInfo.height}`);
if (elementInfo.attributes) {
const attrs = Object.entries(elementInfo.attributes)
.filter(([_, v]) => v)
.map(([k, v]) => `${k}="${v}"`)
.join(', ');
if (attrs)
logger.info(` Attributes: ${attrs}`);
}
result.dryRunResult = {
elementFound: true,
strategy: elementInfo.strategy,
tagName: elementInfo.tagName,
text: elementInfo.text,
position: { x: elementInfo.x, y: elementInfo.y },
size: { width: elementInfo.width || 0, height: elementInfo.height || 0 },
attributes: elementInfo.attributes
};
}
}
catch (error) {
logger.error(`❌ DRY-RUN: Element not found - ${error instanceof Error ? error.message : String(error)}`);
result.dryRunResult = {
elementFound: false,
error: error instanceof Error ? error.message : String(error)
};
}
}
else {
// Normal mode: actually click the element
if (options.useJavaScriptExpression && options.jsExpression) {
// Use JavaScript expression for element selection
await this.clickElementByJavaScript(client, options.jsExpression);
}
else {
await this.clickElement(client, options.clickSelector);
}
}
}
// Perform type interaction
if (options.typeSelector && options.typeText) {
await this.typeInElement(client, options.typeSelector, options.typeText);
}
// Perform toggle interaction
if (options.toggleSelector) {
await this.toggleElement(client, options.toggleSelector);
}
// Ensure Input is referenced to avoid linter warning
void Input;
// Wait for selector after interaction
if (options.waitForSelector) {
await this.waitForSelector(client, options.waitForSelector, options.timeout);
}
// Take screenshot if requested
if (options.screenshotPath) {
await this.takeScreenshot(client, options.screenshotPath);
}
// Handle tab discovery
if (options.discoverTabs) {
const tabs = await this.discoverTabLabels(client);
// If a specific tab label or pattern is specified, click it
if (options.clickSelector && tabs.length > 0) {
let targetTab = null;
if (options.tabLabelPattern) {
const regex = new RegExp(options.tabLabelPattern, 'i');
targetTab = tabs.find(tab => regex.test(tab.label));
}
else if (options.clickSelector) {
targetTab = tabs.find(tab => tab.label.toLowerCase().includes(options.clickSelector.toLowerCase()));
}
if (targetTab) {
await this.clickElement(client, targetTab.selector);
logger.info(`Clicked tab: ${targetTab.label}`);
}
}
// Output discovered tabs
console.log(JSON.stringify({
success: true,
tabCount: tabs.length,
tabs: tabs
}, null, 2));
}
// Handle save button discovery
if (options.findSaveButton) {
const saveButton = await this.findSaveButton(client, options.customSavePatterns);
console.log(JSON.stringify({
success: true,
saveButton: saveButton
}, null, 2));
}
// Enhanced screenshot and visual features
if (options.takeScreenshot || options.screenshotPath) {
const screenshotData = await this.takeEnhancedScreenshot(client, options);
result.screenshotPath = screenshotData.path;
result.screenshotBase64 = screenshotData.base64;
}
// Generate selector map if requested
if (options.selectorMap) {
result.selectorMap = await this.generateSelectorMap(client);
}
// Collect metadata if requested
if (options.withMetadata) {
result.metadata = await this.collectPageMetadata(client);
}
// Handle new tab management commands
if (options.switchTabIndex !== undefined) {
await this.switchToTab(options.switchTabIndex);
logger.info(`Switched to tab ${options.switchTabIndex}`);
}
if (options.tabNext) {
await this.switchToNextTab();
logger.info('Switched to next tab');
}
if (options.tabPrev) {
await this.switchToPrevTab();
logger.info('Switched to previous tab');
}
// Handle keyboard commands
if (options.keyCommand) {
await this.sendKeyCommand(client, options.keyCommand);
logger.info(`Sent keyboard command: ${options.keyCommand}`);
}
// Include network logs in result
if (options.captureNetwork && networkLogs.length > 0) {
result.networkLogs = networkLogs;
logger.info(`Captured ${networkLogs.length} network events during interaction`);
}
result.success = true;
logger.info('Interaction completed successfully');
return result;
}
catch (error) {
result.error = error instanceof Error ? error.message : String(error);
result.success = false;
logger.error(`Interaction failed: ${result.error}`);
return result;
}
finally {
await client.close();
}
}
async runAutomation(options) {
const script = JSON.parse(readFileSync(options.scriptPath, 'utf8'));
const result = {
success: false,
completedSteps: 0,
totalSteps: script.steps.length,
screenshots: [],
networkLogs: [],
monitoringData: [],
timestamp: new Date().toISOString()
};
const client = await this.connectToChrome();
try {
const { Page, Runtime, Network, DOM, Input } = client;
await Page.enable();
await Runtime.enable();
await DOM.enable();
// Note: Input domain doesn't require enable() - it's ready to use immediately
// Enable monitoring if requested
if (options.monitor || script.options?.monitor) {
this.chromeExtractor = new ChromeExtractor({
port: options.chromePort || this.port,
host: options.chromeHost || this.host,
includeNetwork: script.options?.captureNetwork !== false,
includeConsole: true
});
}
// Enable network monitoring if needed
if (script.options?.captureNetwork !== false) {
await Network.enable();
Network.requestWillBeSent((params) => {
result.networkLogs.push({
type: 'request',
timestamp: Date.now(),
...params
});
});
Network.responseReceived((params) => {
result.networkLogs.push({
type: 'response',
timestamp: Date.now(),
...params
});
});
}
// Ensure Input is referenced to avoid linter warning
void Input;
// Execute each step
for (let i = 0; i < script.steps.length; i++) {
const step = script.steps[i];
logger.info(`Executing step ${i + 1}/${script.steps.length}: ${step.action}`);
try {
await this.executeStep(client, step, result);
result.completedSteps++;
}
catch (error) {
result.error = `Failed at step ${i + 1}: ${error instanceof Error ? error.message : String(error)}`;
logger.error(result.error);
break;
}
}
result.success = result.completedSteps === result.totalSteps;
// Save results if requested
if (options.saveResultsPath) {
writeFileSync(options.saveResultsPath, JSON.stringify(result, null, 2));
logger.info(`Results saved to: ${options.saveResultsPath}`);
}
logger.info(`Automation completed: ${result.completedSteps}/${result.totalSteps} steps successful`);
return result;
}
catch (error) {
result.error = error instanceof Error ? error.message : String(error);
logger.error(`Automation failed: ${result.error}`);
return result;
}
finally {
await client.close();
}
}
async connectToChrome() {
try {
// Auto-launch Chrome if needed (using same logic as working clits-inspect)
await this.launchChromeIfNeeded();
// Smart target selection with priority logic (using same logic as working clits-inspect)
const target = await this.autoSelectTarget();
// Connect to the selected target using its webSocketDebuggerUrl
const client = await CDP({
target: target.webSocketDebuggerUrl || target.id
});
return client;
}
catch (error) {
throw new Error(`Failed to connect to Chrome: ${error instanceof Error ? error.message : String(error)}`);
}
}
async checkChromeConnection() {
try {
const response = await fetch(`http://${this.host}:${this.port}/json/version`);
if (response.ok) {
const version = await response.json();
logger.info(`✅ Existing Chrome debugging session detected: ${version.Browser || 'Chrome'}`);
return true;
}
return false;
}
catch (error) {
logger.debug(`No Chrome debugging session found on port ${this.port}: ${error instanceof Error ? error.message : String(error)}`);
return false;
}
}
async launchChromeIfNeeded() {
// First check if Chrome is already running with debugging
const isChrome = await this.checkChromeConnection();
if (isChrome) {
logger.info('✅ Using existing Chrome debugging session');
return;
}
// Check if Chrome process exists but not responding to debugging
if (process.platform === 'darwin') {
try {
const { execSync } = await import('child_process');
const chromeProcesses = execSync('ps aux | grep Chrome | grep remote-debugging-port', { encoding: 'utf8' });
if (chromeProcesses.trim().length > 0) {
logger.warn('Chrome process with remote debugging found but not responding. Waiting for it to be ready...');
// Wait longer for existing Chrome to become ready
for (let i = 0; i < 10; i++) {
await new Promise(resolve => setTimeout(resolve, 1000));
const isReady = await this.checkChromeConnection();
if (isReady) {
logger.info('✅ Existing Chrome session is now ready');
return;
}
}
logger.warn('Existing Chrome session did not become ready, will launch new instance');
}
}
catch (error) {
logger.debug('Could not check for existing Chrome processes:', error);
}
}
// Only launch if no Chrome debugging session exists
logger.info('No Chrome debugging session found. Launching Chrome with debugging enabled...');
if (process.platform === 'darwin') {
const { spawn } = await import('child_process');
const chromePath = '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
const args = [
`--remote-debugging-port=${this.port}`,
'--user-data-dir=/tmp/chrome-debug-clits',
'--no-first-run',
'--no-default-browser-check',
'--disable-web-security',
'--disable-features=VizDisplayCompositor'
];
const chromeProcess = spawn(chromePath, args, {
detached: true,
stdio: 'ignore'
});
chromeProcess.unref();
logger.info(`Chrome launched with PID: ${chromeProcess.pid}`);
// Wait for Chrome to start and verify connection
for (let i = 0; i < 15; i++) {
await new Promise(resolve => setTimeout(resolve, 1000));
const isReady = await this.checkChromeConnection();
if (isReady) {
logger.info('✅ New Chrome debugging session is ready');
return;
}
logger.debug(`Waiting for Chrome to be ready... attempt ${i + 1}/15`);
}
throw new Error('Chrome launched but debugging session did not become available within 15 seconds');
}
else {
throw new Error('Chrome is not running with remote debugging enabled. Start Chrome with --remote-debugging-port=9222');
}
}
async autoSelectTarget() {
const response = await fetch(`http://${this.host}:${this.port}/json/list`);
const targets = await response.json();
const pageTargets = targets.filter((t) => t.type === 'page');
if (pageTargets.length === 0) {
throw new Error('No Chrome page targets found. Please open a tab in Chrome with --remote-debugging-port=9222');
}
if (pageTargets.length === 1) {
return pageTargets[0];
}
// Smart target selection - prefer localhost/development URLs (same logic as working clits-inspect)
const localTargets = pageTargets.filter(t => t.url.includes('localhost') ||
t.url.includes('127.0.0.1') ||
t.url.includes('local') ||
t.url.startsWith('http://localhost') ||
t.url.startsWith('https://localhost'));
if (localTargets.length > 0) {
return localTargets[0];
}
// Filter out chrome:// URLs as fallback
const nonChromeTargets = pageTargets.filter(t => !t.url.startsWith('chrome://'));
if (nonChromeTargets.length > 0) {
return nonChromeTargets[0];
}
// Last resort: return first available
return pageTargets[0];
}
async executeStep(client, step, result) {
const timeout = step.timeout || ChromeAutomation.DEFAULT_TIMEOUT;
switch (step.action) {
case 'navigate': {
if (!step.url)
throw new Error('Navigate step requires url');
// Use internal navigation logic for consistency
const tempOptions = {
url: step.url,
timeout: timeout
};
// Call navigate method but handle return internally
const tempAutomation = new ChromeAutomation(this.port, this.host);
await tempAutomation.navigate(tempOptions);
break;
}
case 'wait':
if (!step.selector)
throw new Error('Wait step requires selector');
await this.waitForSelector(client, step.selector, timeout);
break;
case 'click':
if (!step.selector)
throw new Error('Click step requires selector');
await this.clickElement(client, step.selector);
break;
case 'type':
if (!step.selector || !step.text)
throw new Error('Type step requires selector and text');
await this.typeInElement(client, step.selector, step.text);
break;
case 'toggle':
if (!step.selector)
throw new Error('Toggle step requires selector');
await this.toggleElement(client, step.selector);
break;
case 'screenshot':
if (!step.path)
throw new Error('Screenshot step requires path');
await this.takeScreenshot(client, step.path);
result.screenshots.push(step.path);
break;
case 'discover_links': {
// Add discover_links action support
const links = await this.discoverAllLinks(client);
// Save links data to results
if (!result.monitoringData)
result.monitoringData = [];
result.monitoringData.push({
type: 'discovered_links',
timestamp: new Date().toISOString(),
data: links
});
// If path is specified, save to file
if (step.path) {
writeFileSync(step.path, JSON.stringify(links, null, 2));
}
break;
}
case 'interact': {
// Add interact action support - handle visual selection methods like CLI does
let clickSelector = step.selector;
let useJavaScriptExpression = false;
let jsExpression = '';
if (step.clickText) {
jsExpression = await this.findElementByText(step.clickText);
useJavaScriptExpression = true;
clickSelector = '__JS_EXPRESSION__';
}
else if (step.clickColor) {
jsExpression = await this.findElementByColor(step.clickColor);
useJavaScriptExpression = true;
clickSelector = '__JS_EXPRESSION__';
}
else if (step.clickRegion) {
jsExpression = await this.findElementByRegion(step.clickRegion);
useJavaScriptExpression = true;
clickSelector = '__JS_EXPRESSION__';
}
else if (step.clickDescription) {
jsExpression = await this.findElementByDescription(step.clickDescription);
useJavaScriptExpression = true;
clickSelector = '__JS_EXPRESSION__';
}
const interactionOptions = {
clickSelector,
screenshotPath: step.screenshotPath,
takeScreenshot: !!step.screenshotPath,
timeout: step.timeout,
chromePort: this.port,
chromeHost: this.host,
useJavaScriptExpression,
jsExpression
};
const interactResult = await this.interact(interactionOptions);
// Add screenshot to results if taken
if (interactResult.screenshotPath) {
result.screenshots.push(interactResult.screenshotPath);
}
// Add network logs if captured
if (interactResult.networkLogs) {
if (!result.monitoringData)
result.monitoringData = [];
result.monitoringData.push({
type: 'network_logs',
timestamp: new Date().toISOString(),
data: interactResult.networkLogs
});
}
if (!interactResult.success) {
throw new Error(interactResult.error || 'Interaction failed');
}
break;
}
case 'click-text': {
if (!step.text)
throw new Error('Click-text step requires text');
const jsExpression = await this.findElementByText(step.text);
await this.clickElementByJavaScript(client, jsExpression);
// Wait if specified
if (step.wait) {
await new Promise(resolve => setTimeout(resolve, step.wait));
}
// Take screenshot if requested
if (step.screenshotPath) {
await this.takeScreenshot(client, step.screenshotPath);
result.screenshots.push(step.screenshotPath);
}
break;
}
case 'click-region': {
if (!step.region)
throw new Error('Click-region step requires region');
const jsExpression = await this.findElementByRegion(step.region);
await this.clickElementByJavaScript(client, jsExpression);
// Wait if specified
if (step.wait) {
await new Promise(resolve => setTimeout(resolve, step.wait));
}
// Take screenshot if requested
if (step.screenshotPath) {
await this.takeScreenshot(client, step.screenshotPath);
result.screenshots.push(step.screenshotPath);
}
break;
}
case 'key': {
if (!step.keyCommand)
throw new Error('Key step requires keyCommand');
await this.sendKeyCommand(client, step.keyCommand);
// Wait if specified
if (step.wait) {
await new Promise(resolve => setTimeout(resolve, step.wait));
}
// Take screenshot if requested
if (step.screenshotPath) {
await this.takeScreenshot(client, step.screenshotPath);
result.screenshots.push(step.screenshotPath);
}
break;
}
case 'switch-tab': {
if (step.tabIndex === undefined)
throw new Error('Switch-tab step requires tabIndex');
await this.switchToTab(step.tabIndex);
// Wait if specified
if (step.wait) {
await new Promise(resolve => setTimeout(resolve, step.wait));
}
// Take screenshot if requested
if (step.screenshotPath) {
await this.takeScreenshot(client, step.screenshotPath);
result.screenshots.push(step.screenshotPath);
}
break;
}
case 'tab-next': {
await this.switchToNextTab();
// Wait if specified
if (step.wait) {
await new Promise(resolve => setTimeout(resolve, step.wait));
}
// Take screenshot if requested
if (step.screenshotPath) {
await this.takeScreenshot(client, step.screenshotPath);
result.screenshots.push(step.screenshotPath);
}
break;
}
case 'tab-prev': {
await this.switchToPrevTab();
// Wait if specified
if (step.wait) {
await new Promise(resolve => setTimeout(resolve, step.wait));
}
// Take screenshot if requested
if (step.screenshotPath) {
await this.takeScreenshot(client, step.screenshotPath);
result.screenshots.push(step.screenshotPath);
}
break;
}
default:
throw new Error(`Unknown step action: ${step.action}`);
}
}
escapeSelector(selector) {
// Escape backslashes, single quotes, and double quotes for safe JavaScript string interpolation
return selector.replace(/\\/g, '\\\\').replace(/'/g, "\\'").replace(/"/g, '\\"');
}
async findElementWithFallback(client, selector) {
logger.debug(`Finding element with selector: ${selector}`);
// Enhanced element finding with comprehensive CSS selector support
const result = await client.Runtime.evaluate({
expression: `
JSON.stringify((() => {
const selector = '${selector.replace(/'/g, "\\'")}';
let element = null;
let strategy = '';
// Helper function to handle :contains() pseudo-selector
function findByContains(containsSelector) {
const match = containsSelector.match(/^(.+?):contains\\(['"](.+?)['"]\\)$/);
if (!match) return null;
const baseSelector = match[1] || '*';
const text = match[2];
try {
const candidates = document.querySelectorAll(baseSelector);
return Array.from(candidates).find(el =>
el.textContent && el.textContent.includes(text)
);
} catch (e) {
return null;
}
}
// Helper function to evaluate XPath
function findByXPath(xpath) {
try {
const result = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
return result.singleNodeValue;
} catch (e) {
return null;
}
}
// Strategy 1: Handle :contains() pseudo-selector
if (selector.includes(':contains(')) {
element = findByContains(selector);
if (element) strategy = 'contains';
}
// Strategy 2: Handle XPath expressions (starting with // or /)
if (!element && (selector.startsWith('//') || selector.startsWith('/'))) {
element = findByXPath(selector);
if (element) strategy = 'xpath';
}
// Strategy 3: Direct CSS selector with comprehensive support
if (!element) {
try {
element = document.querySelector(selector);
if (element) strategy = 'css-direct';
} catch (e) {
// Invalid CSS selector, continue to fallbacks
}
}
// Strategy 4: Enhanced attribute-based search
if (!element && !selector.includes(':') && !selector.startsWith('//') && !selector.startsWith('/')) {
const patterns = [
// Data attributes
'[data-testid="' + selector + '"]',
'[data-test="' + selector + '"]',
'[data-cy="' + selector + '"]',
'[data-id="' + selector + '"]',
// ARIA attributes
'[aria-label*="' + selector + '"]',
'[aria-labelledby*="' + selector + '"]',
'[role="' + selector + '"]',
// Class and ID patterns
'[class*="' + selector + '"]',
'[id*="' + selector + '"]',
// Input attributes
'[name="' + selector + '"]',
'[placeholder*="' + selector + '"]',
'[title*="' + selector + '"]',
'[alt*="' + selector + '"]',
// Button/link specific
'[type="' + selector + '"]'
];
for (const pattern of patterns) {
try {
element = document.querySelector(pattern);
if (element) {
strategy = 'attribute-' + pattern.split('=')[0].slice(1);
break;
}
} catch (e) {
// Invalid selector, skip
}
}
}
// Strategy 5: Complex CSS selector combinations
if (!element && selector.includes('.')) {
try {
// Handle multiple classes
element = document.querySelector(selector);
if (element) strategy = 'css-complex';
} catch (e) {
// Try with escaped special characters
try {
const escapedSelector = selector.replace(/:/g, '\\\\:').replace(/\\./g, '\\\\.');
element = document.querySelector(escapedSelector);
if (element) strategy = 'css-escaped';
} catch (e2) {
// Continue to text search
}
}
}
// Strategy 6: Enhanced text search with fuzzy matching
if (!element && selector.length < 100 && !selector.includes('[') && !selector.includes('//')) {
const clickables = Array.from(document.querySelectorAll(
'a, button, [role="button"], [onclick], input[type="button"], input[type="submit"], ' +
'[class*="btn"], [class*="button"], [aria-label], [data-testid]'
));
// Exact text match first
element = clickables.find(el =>
el.textContent && el.textContent.trim() === selector.trim()
);
// Then partial text match
if (!element) {
element = clickables.find(el =>
el.textContent && el.textContent.includes(selector)
);
}
// Then attribute text match
if (!element) {
element = clickables.find(el => {
const ariaLabel = el.getAttribute('aria-label') || '';
const title = el.getAttribute('title') || '';
const placeholder = el.getAttribute('placeholder') || '';
return ariaLabel.includes(selector) || title.includes(selector) || placeholder.includes(selector);
});
}
if (element) strategy = 'text-search';
}
if (!element) {
return {
error: 'Element not found with any strategy',
attemptedStrategies: ['contains', 'xpath', 'css-direct', 'attribute-patterns', 'css-complex', 'text-search'],
selector: selector
};
}
const rect = element.getBoundingClientRect();
if (rect.width === 0 || rect.height === 0) {
return {
error: 'Element found but not visible',
strategy: strategy,
selector: selector
};
}
return {
x: rect.left + rect.width / 2,
y: rect.top + rect.height / 2,
width: rect.width,
height: rect.height,
strategy: strategy,
tagName: element.tagName.toLowerCase(),
text: element.textContent?.trim().substring(0, 50) || '',
attributes: {
id: element.id || '',
className: element.className || '',
'data-testid': element.getAttribute('data-testid') || '',
'aria-label': element.getAttribute('aria-label') || ''
}
};
})())
`
});
if (result.result?.value) {
const elementData = JSON.parse(result.result.value);
if (elementData.error) {
// Enhanced error message with attempted strategies
let errorMsg = `Element not found: "${selector}". ${elementData.error}`;
if (elementData.attemptedStrategies) {
errorMsg += ` (Tried: ${elementData.attemptedStrategies.join(', ')})`;
}
throw new Error(errorMsg);
}
logger.info(`✅ Element found: ${selector} (strategy: ${elementData.strategy}, tag: ${elementData.tagName})`);
if (elementData.text) {
logger.info(` Text: "${elementData.text}"`);
}
return elementData;
}
throw new Error(`Element not found: "${selector}". No result from evaluation.`);
}
async waitForSelector(client, selector, timeout = ChromeAutomation.DEFAULT_TIMEOUT) {
const startTime = Date.now();
let attempts = 0;
const maxAttempts = Math.min(50, timeout / 500); // Limit attempts based on timeout
while (Date.now() - startTime < timeout && attempts < maxAttempts) {
try {
const elementInfo = await this.findElementWithFallback(client, selector);
if (elementInfo) {
logger.info(`✅ Element found: ${selector} (attempt ${attempts + 1})`);
return;
}
}
catch (error) {
// CRITICAL FIX: Don't retry on specific errors that won't resolve
if (error instanceof Error && error.message.includes('Element not found:')) {
throw error; // Immediately fail for definitive not-found errors
}
logger.debug(`⏳ Attempt ${attempts + 1}: ${error instanceof Error ? error.message : String(error)}`);
}
attempts++;
await new Promise(resolve => setTimeout(resolve, 500)); // Longer delay between attempts
}
throw new Error(`Timeout waiting for selector: "${selector}" after ${attempts} attempts in ${Date.now() - startTime}ms. Element may not exist or be accessible.`);
}
async clickElement(client, selector) {
// First ensure element exists
await this.waitForSelector(client, selector);
// Try to find element with fallback strategies
const elementInfo = await this.findElementWithFallback(client, selector);
if (!elementInfo) {
throw new Error(`Element not found with any strategy: ${selector}`);
}
const { x, y } = elementInfo;
// Perform click
await client.Input.dispatchMouseEvent({
type: 'mousePressed',
x,
y,
button: 'left',
clickCount: 1
});
await client.Input.dispatchMouseEvent({
type: 'mouseReleased',
x,
y,
button: 'left',
clickCount: 1
});
logger.info(`Clicked element: ${selector}`);
}
async clickElementByJavaScript(client, jsExpression) {
logger.info(`Attempting to click element using JavaScript expression`);
try {
// Evaluate the provided JavaScript expression directly
const result = await client.Runtime.evaluate({
expression: jsExpression
});
if (!result.result?.value) {
throw new Error('No result from element evaluation');
}
// Parse JSON result if the expression returns JSON
let resultData = result.result.value;
if (typeof resultData === 'string') {
try {
resultData = JSON.parse(resultData);
}
catch (e) {
// If it's not JSON, treat it as a simple value
logger.error('Failed to parse result as JSON:', resultData);
throw new Error('Invalid result format from JavaScript expression');
}
}
if (resultData.error) {
throw new Error(`Element operation failed: ${resultData.error}`);
}
if (resultData.success) {
logger.info(`Successfully performed ${resultData.method} operation: "${resultData.text}"`);
if (resultData.url) {
logger.info(`Navigated to: ${resultData.url}`);
}
// Handle coordinate-based clicking if needed
if (resultData.method === 'coordinates' && resultData.x && resultData.y) {
await client.Input.dispatchMouseEvent({
type: 'mousePressed',
x: resultData.x,
y: resultData.y,
button: 'left',
clickCount: 1
});
await client.Input.dispatchMouseEvent({
type: 'mouseReleased',
x: resultData.x,
y: resultData.y,
button: 'left',
clickCount: 1
});
logger.info(`Performed coordinate click at (${resultData.x}, ${resultData.y})`);
}
return;
}
throw new Error('Unexpected result from element operation');
}
catch (error) {
logger.error('Click operation failed:', error);
throw error;
}
}
async typeInElement(client, selector, text) {
// First click on the element to focus it
await this.clickElement(client, selector);
// Clear existing content
await client.Input.dispatchKeyEvent({
type: 'keyDown',
key: 'Control'
});
await client.Input.dispatchKeyEvent({
type: 'char',
text: 'a'
});
await client.Input.dispatchKeyEvent({
type: 'keyUp',
key: 'Control'
});
// Type the new text
for (const char of text) {
await client.Input.dispatchKeyEvent({
type: 'char',
text: char
});
}
logger.info(`Typed text in element: ${selector}`);
}
async toggleElement(client, selector) {
// Simply click the toggle element
await this.clickElement(client, selector);
logger.info(`Toggled element: ${selector}`);
}
async takeScreenshot(client, path) {
const screenshot = await client.Page.captureScreenshot({
format: 'png',
fullPage: true
});
writeFileSync(path, screenshot.data, 'base64');
logger.info(`Screenshot saved: ${path}`);
}
/**
* Discovers all tab labels in a dialog or tabbed interface
* @param client CDP client instance
* @returns Array of tab labels with their selectors
*/
async discoverTabLabels(client) {
const result = await client.Runtime.evaluate({
expression: `
JSON.stringify((function() {
const tabs = Array.from(document.querySelectorAll('.MuiTab-root, [role="tab"], .MuiTabs-root .MuiTab-root'));
return tabs.map((tab, index) => ({
label: tab.textContent?.trim() || tab.getAttribute('aria-label') || tab.getAttribute('title') || '',
selector: tab.getAttribute('data-testid') ||
(tab.getAttribute('aria-label') ? '[aria-label="' + tab.getAttribute('aria-label') + '"]' : '') ||
(tab.textContent?.trim() ? '[role="tab"]:nth-child(' + (index + 1) + ')' : '') ||
'.MuiTab-root:nth-child(' + (index + 1) + ')',
index: index,
isActive: tab.getAttribute('aria-selected') === 'true' || tab.classList.contains('Mui-selected'),
isDisabled: tab.getAttribute('aria-disabled') === 'true' || tab.classList.contains('Mui-disabled')
})).filter(tab => tab.label);
})())
`
});
if (result.result.value) {
return JSON.parse(result.result.value);
}
return [];
}
/**
* Finds the best save button in a dialog using multiple strategies
* @param client CDP client instance
* @param customText Optional custom text patterns for save buttons
* @returns Save button element info or null
*/
async findSaveButton(client, customText) {
const savePatterns = customText || ['save', 'update', 'apply', 'ok', 'done', 'submit', 'confirm'];
const result = await client.Runtime.evaluate({
expression: `
JSON.stringify((function() {
const savePatterns = ${JSON.stringify(savePatterns)};
const patternRegex = /${savePatterns.join('|')}/i;
// Strategy 1: Find by text content
const buttonsByText = Array.from(document.querySelectorAll('.MuiDialog-root button, .MuiModal-root button, [role="dialog"] button'))
.filter(btn => patternRegex.test(btn.textContent?.trim() || ''));
if (buttonsByText.length > 0) {
const rect = buttonsByText[0].getBoundingClientRect();
return {
x: rect.left + rect.width / 2,
y: rect.top + rect.height / 2,
strategy: 'text-content',
text: buttonsByText[0].textContent?.trim()
};
}
// Strategy 2: Find by type="submit"
const submitButtons = Array.from(document.querySelectorAll('.MuiDialog-root button[type="submit"], .MuiModal-root button[type="submit"], .MuiDialogActions-root button[type="submit"]'));
if (submitButtons.length > 0) {
const rect = submitButtons[0].getBoundingClientRect();
return {
x: rect.left + rect.width / 2,
y: rect.top + rect.height / 2,
strategy: 'submit-type',
text: submitButtons[0].textContent?.trim()
};
}
// Strategy 3: Find by aria-label or title
const ariaButtons = Array.from(document.querySelectorAll('.MuiDialog-root button, .MuiModal-root button'))
.filter(btn => patternRegex.test(btn.getAttribute('aria-label') || '') || patternRegex.test(btn.getAttribute('title') || ''));
if (ariaButtons.length > 0) {
const rect = ariaButtons[0].getBoundingClientRect();
return {
x: rect.left + rect.width / 2,
y: rect.top + rect.height / 2,
strategy: 'aria-label',
ariaLabel: ariaButtons[0].getAttribute('aria-label'),
title: ariaButtons[0].getAttribute('title')
};
}
// Strategy 4: Find icon buttons with save-like icons
const iconButtons = Array.from(document.querySelectorAll('.MuiDialog-root .MuiIconButton-root, .MuiDialog-root button'))
.filter(btn => {
const svg = btn.querySelector('.MuiSvgIcon-root, svg');
if (!svg) return false;
const ariaLabel = btn.getAttribute('aria-label') || '';
const title = btn.getAttribute('title') || '';
return patternRegex.test(ariaLabel) || patternRegex.test(title);
});
if (iconButtons.length > 0) {
const rect = iconButtons[0].getBoundingClientRect();
return {
x: rect.left + rect.width / 2,
y: rect.top + rect.height / 2,
strategy: 'icon-but