browser-use-typescript
Version:
A TypeScript-based browser automation framework
1,152 lines (1,134 loc) • 60.9 kB
JavaScript
import { randomUUID } from "crypto";
import { Browser } from "./browserService";
import { BrowserError, BrowserState, TabInfo, URLNotAllowedError } from "./type";
import { DomService } from "../../domTypes/DomService";
import { DOMElementNode } from "../../domTypes/domClass";
import path from "path";
import * as fs from "fs";
import { S3Client, PutObjectCommand, GetObjectCommand, } from "@aws-sdk/client-s3";
// Debug utility with different levels
const DEBUG = {
ERROR: false, // Always show errors
WARN: false, // Always show warnings
INFO: false, // Show general information
DEBUG: false, // Show detailed debugging information
TRACE: false, // Show very detailed trace information (can be verbose)
error: (message, ...args) => {
if (DEBUG.ERROR) {
console.error(message, ...args);
}
},
warn: (message, ...args) => {
if (DEBUG.WARN) {
console.warn(message, ...args);
}
},
info: (message, ...args) => {
if (DEBUG.INFO) {
console.info(message, ...args);
}
},
debug: (message, ...args) => {
if (DEBUG.DEBUG) {
console.debug(message, ...args);
}
},
trace: (message, ...args) => {
if (DEBUG.TRACE) {
console.trace(message, ...args);
}
}
};
const s3 = new S3Client();
class BrowserContextWindowSize {
width;
height;
constructor(width, height) {
this.width = width;
this.height = height;
}
}
export class BrowserContextConfig {
cookies_file;
s3Storage;
minimum_wait_page_load_time;
wait_for_network_idle_page_load_time;
maximum_wait_page_load_time;
wait_between_actions;
browser_window_size;
no_viewport;
save_recording_path;
save_downloads_path;
trace_path;
locale;
user_agent;
highlight_elements;
viewport_expansion;
allowed_domains;
include_dynamic_attributes;
disable_security;
_force_keep_context_alive;
constructor() {
this.s3Storage = false;
this.cookies_file = null;
this.minimum_wait_page_load_time = 0.5;
this.wait_for_network_idle_page_load_time = 1.0;
this.maximum_wait_page_load_time = 5.0;
this.wait_between_actions = 0.5;
this.browser_window_size = new BrowserContextWindowSize(1280, 1100);
this.no_viewport = null;
this.save_recording_path = null;
this.save_downloads_path = null;
this.trace_path = null;
this.locale = null;
this.user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36';
this.highlight_elements = true;
this.viewport_expansion = 500;
this.allowed_domains = null;
this.include_dynamic_attributes = true;
this.disable_security = true;
this._force_keep_context_alive = false;
}
}
class BrowserSession {
context;
cached_state;
constructor(context, cached_state) {
this.context = context;
this.cached_state = cached_state;
}
}
class BrowserContextState {
target_id = null;
constructor(target_id) {
this.target_id = target_id ?? null;
}
}
export class BrowserContext {
context_id = randomUUID().toString();
config = new BrowserContextConfig();
current_state;
browser = new Browser();
state = new BrowserContextState();
session = null;
_initialized = null;
constructor(param) {
// Start the initialization but don't await it - will be awaited on first use
this.config = param?.config ? param.config : new BrowserContextConfig();
this._initialized = this._initialize_session();
}
// Ensure the session is initialized before any method that uses it
async _ensure_initialized() {
if (this._initialized) {
await this._initialized;
}
}
async __aexit__() {
await this._ensure_initialized();
await this.close();
}
async close() {
await this._ensure_initialized();
if (!this.session) {
return;
}
try {
if (!this.config._force_keep_context_alive && this.session.context) {
await this.session.context.close();
}
await this.save_cookies();
}
catch (e) {
DEBUG.error('Failed to close browser context', e);
}
this.session = null;
}
async save_cookies(local) {
if (this.session && this.session.context && this.config.cookies_file) {
try {
const cookies = await this.session.context.cookies();
if (local) {
const dirname = path.dirname(this.config.cookies_file);
if (dirname) {
// Ensure the directory exists
fs.mkdirSync(dirname, { recursive: true });
}
// Write cookies to the file
fs.writeFileSync(this.config.cookies_file, JSON.stringify(cookies, null, 2));
}
else {
// Upload to S3
await s3.send(new PutObjectCommand({
Bucket: process.env.AWS_S3_BUCKET_NAME,
Key: path.basename(this.config.cookies_file),
Body: JSON.stringify(cookies, null, 2),
ContentType: 'application/json'
}));
}
}
catch (e) {
DEBUG.error('Failed to save cookies', e);
}
}
}
async _initialize_session() {
DEBUG.info('Initializing browser session');
try {
DEBUG.debug('Getting Playwright browser instance');
const playwright_browser = await this.browser.get_playwright_browser();
DEBUG.debug('Creating browser context');
const context = await this._create_context(playwright_browser);
DEBUG.debug('Getting pages from context');
const pages = context.pages();
DEBUG.debug(`Found ${pages.length} existing pages`);
this.session = new BrowserSession(context, null);
let active_page = null;
if (pages && pages.length > 0) {
active_page = pages[0];
DEBUG.debug('Using existing page', { pageUrl: active_page.url() });
}
else {
DEBUG.debug('Creating new page');
active_page = await context.newPage();
DEBUG.debug('New page created');
}
DEBUG.debug('Bringing active page to front');
await active_page.bringToFront();
DEBUG.debug('Waiting for page load state');
await active_page.waitForLoadState('load');
DEBUG.info('Browser context initialized successfully');
return this.session;
}
catch (error) {
DEBUG.error('Failed to initialize session', error);
throw error;
}
}
async _add_new_page_listener(context) {
const on_page = async (page) => {
await page.waitForLoadState();
if (this.session !== null) {
this.state.target_id = null;
}
};
context.on('page', on_page);
}
async get_session() {
await this._ensure_initialized();
if (this.session === null) {
return await this._initialize_session();
}
return this.session;
}
async get_current_page() {
await this._ensure_initialized();
DEBUG.debug('Getting current page');
try {
const session = await this.get_session();
DEBUG.debug('Session retrieved successfully');
const page = await this._get_current_page(session);
DEBUG.debug('Current page retrieved successfully', { url: page.url() });
return page;
}
catch (error) {
DEBUG.error('Failed to get current page', error);
throw error;
}
}
async _get_current_page(session) {
await this._ensure_initialized();
DEBUG.debug('Getting pages from browser context');
try {
const pages = session.context.pages();
DEBUG.debug(`Found ${pages.length} pages in context`);
if (pages.length === 0) {
DEBUG.error('No pages found in context, creating a new page');
const newPage = await session.context.newPage();
return newPage;
}
const currentPage = pages[pages.length - 1];
DEBUG.debug('Selected current page', {
url: currentPage.url(),
index: pages.length - 1,
totalPages: pages.length
});
return currentPage;
}
catch (error) {
DEBUG.error('Error getting current page from session', error);
throw error;
}
}
async _create_context(browser) {
if (this.browser.config.browser_instance_path && (await browser.contexts()).length > 0) {
return (await browser.contexts())[0];
}
else {
const context = await browser.newContext({
viewport: this.config.browser_window_size,
javaScriptEnabled: true,
bypassCSP: this.config.disable_security,
ignoreHTTPSErrors: this.config.disable_security,
recordVideo: { dir: this.config.save_recording_path || "null",
size: this.config.browser_window_size
},
locale: this.config.locale || undefined,
userAgent: this.config.user_agent
});
if (this.config.trace_path) {
(await context).tracing.start({ screenshots: true, snapshots: true, sources: true });
}
//Add Cookies
if (this.config.cookies_file) {
//Load cookies from s3
if (this.config.s3Storage) {
const cookiesFromS3 = await s3.send(new GetObjectCommand({
Bucket: process.env.AWS_S3_BUCKET_NAME,
Key: path.basename(this.config.cookies_file)
}));
const cookies = JSON.parse((await cookiesFromS3)?.Body?.toString() || '');
await (await context).addCookies(cookies);
}
//Or locally
else {
const cookies = fs.readFileSync(this.config.cookies_file, 'utf8');
await (await context).addCookies(JSON.parse(cookies));
}
}
//Expose Anti-Detection Scripts
(await context).addInitScript({
content: `
// Webdriver property
Object.defineProperty(navigator, 'webdriver', {
get: () => undefined
});
// Languages
Object.defineProperty(navigator, 'languages', {
get: () => ['en-US']
});
// Plugins
Object.defineProperty(navigator, 'plugins', {
get: () => [1, 2, 3, 4, 5]
});
// Chrome runtime
window.chrome = { runtime: {} };
// Permissions
const originalQuery = window.navigator.permissions.query;
window.navigator.permissions.query = (parameters) => (
parameters.name === 'notifications' ?
Promise.resolve({ state: Notification.permission }) :
originalQuery(parameters)
);
(function () {
const originalAttachShadow = Element.prototype.attachShadow;
Element.prototype.attachShadow = function attachShadow(options) {
return originalAttachShadow.call(this, { ...options, mode: "open" });
};
})();
`
});
return context;
}
}
async _wait_for_stable_network() {
const relevant_resource_types = new Set(['document', 'script', 'stylesheet', 'xhr', 'fetch', 'other']);
const relevant_content_types = new Set(['text/html', 'application/javascript', 'text/css']);
const ignored_url_patterns = new Set(['.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico', '.woff', '.woff2', '.ttf']);
const page = await this.get_current_page();
const pending_requests = new Set();
let last_activity = Date.now();
// Function to handle request events
const on_request = (request) => {
const url = request.url().toLowerCase();
if (!relevant_resource_types.has(request.resourceType())) {
return;
}
if (ignored_url_patterns.has(url)) {
return;
}
pending_requests.add(request);
last_activity = Date.now();
};
// Function to handle response events
const on_response = (response) => {
const request = response.request();
if (!pending_requests.has(request)) {
return;
}
const content_type = response.headers()['content-type']?.toLowerCase() || '';
if (!relevant_content_types.has(content_type)) {
pending_requests.delete(request);
return;
}
pending_requests.delete(request);
last_activity = Date.now();
};
// Attach event listeners to the page
page.on('request', on_request);
page.on('response', on_response);
try {
// Wait for the network to stabilize
let timeout = false;
while (!timeout) {
await new Promise(resolve => setTimeout(resolve, 100)); // Sleep for 100ms
const now = Date.now();
if (pending_requests.size === 0 && (now - last_activity) >= this.config.wait_for_network_idle_page_load_time * 1000) {
break;
}
if (now - last_activity > this.config.maximum_wait_page_load_time * 1000) {
timeout = true;
break;
}
}
}
finally {
// Clean up event listeners
page.removeListener('request', on_request);
page.removeListener('response', on_response);
}
}
async _wait_for_page_and_frames_load(timeout_overwrite = null) {
await this._ensure_initialized();
const start_time = Date.now();
try {
await this._wait_for_stable_network();
const page = await this.get_current_page();
await this._check_and_handle_navigation(page);
}
catch (e) {
DEBUG.error('Error waiting for page and frames load', e);
}
const elapsed = (Date.now() - start_time) / 1000;
const remaining = Math.max((timeout_overwrite || this.config.minimum_wait_page_load_time) - elapsed, 0);
if (remaining > 0) {
await new Promise(resolve => setTimeout(resolve, remaining * 1000));
}
}
async _is_url_allowed(url) {
await this._ensure_initialized();
if (!this.config.allowed_domains) {
return true;
}
try {
const parsed_url = new URL(url);
const domain = parsed_url.hostname.toLowerCase();
return this.config.allowed_domains.some(allowed_domain => {
return domain === allowed_domain.toLowerCase() || domain.endsWith(`.${allowed_domain.toLowerCase()}`);
});
}
catch (e) {
DEBUG.error('Error checking URL allowed', e);
return false;
}
}
async _check_and_handle_navigation(page) {
await this._ensure_initialized();
DEBUG.debug('Starting navigation check', { url: page.url() });
try {
if (!page) {
DEBUG.error('No page provided for navigation check');
throw new Error('No page provided for navigation check');
}
DEBUG.debug('Waiting for navigation to complete');
await page.waitForLoadState('domcontentloaded');
await page.waitForLoadState('load', { timeout: 10000 }).catch(e => {
DEBUG.warn('Load state timeout', e);
});
DEBUG.debug('Checking current URL', { url: page.url() });
const url = page.url();
// Check if URL is allowed
if (!this._is_url_allowed(url)) {
DEBUG.warn(`Navigation to non-allowed URL detected: ${url}`);
try {
await this.go_back();
}
catch (e) {
DEBUG.error(`Failed to go back after detecting non-allowed URL: ${e}`);
}
throw new URLNotAllowedError(`Navigation to non-allowed URL: ${url}`);
}
// Handle various navigation scenarios
if (url.startsWith('chrome-error://')) {
DEBUG.warn('Chrome error page detected', { url });
// Handle chrome error
}
DEBUG.debug('Navigation check completed successfully');
}
catch (error) {
if (!(error instanceof URLNotAllowedError)) {
DEBUG.error('Error during navigation check', error);
}
throw error;
}
}
async handle_page_load() {
await this._ensure_initialized();
DEBUG.info('Starting page load handling');
const start_time = Date.now();
try {
DEBUG.debug('Waiting for page to load and become stable');
await this._wait_for_page_and_frames_load();
DEBUG.debug('Page is stable, updating state');
const page = await this.get_current_page();
DEBUG.debug('Checking for navigation events');
await this._check_and_handle_navigation(page);
await this._wait_for_stable_network();
DEBUG.info('Navigation check completed successfully');
}
catch (e) {
DEBUG.error('Page load failed', e);
// Don't return anything for void function
}
const end_time = Date.now();
DEBUG.debug(`Page load handling completed in ${end_time - start_time}ms`);
}
async navigate_to(url) {
await this._ensure_initialized();
if (!this._is_url_allowed(url)) {
throw new Error(`Navigation to non-allowed URL: ${url}`);
}
const page = await this.get_current_page();
await page.goto(url);
await page.waitForLoadState();
}
async refresh_page() {
await this._ensure_initialized();
const page = await this.get_current_page();
await page.reload();
await page.waitForLoadState();
}
async go_back() {
await this._ensure_initialized();
const page = await this.get_current_page();
try {
await page.goBack({ timeout: 10, waitUntil: 'domcontentloaded' });
}
catch (e) {
DEBUG.error('Failed to go back', e);
}
}
async go_forward() {
await this._ensure_initialized();
const page = await this.get_current_page();
try {
await page.goForward({ timeout: 10, waitUntil: 'domcontentloaded' });
}
catch (e) {
DEBUG.error('Failed to go forward', e);
}
}
async close_current_tab() {
await this._ensure_initialized();
const session = this.session;
const page = await this._get_current_page(session);
await page.close();
if (session?.context?.pages) {
await this.switch_to_tab(0);
}
}
async get_page_html() {
await this._ensure_initialized();
const page = await this.get_current_page();
return await page.content();
}
async execute_javascript(script) {
await this._ensure_initialized();
const page = await this.get_current_page();
return await page.evaluate(script);
}
async get_page_structure() {
await this._ensure_initialized();
const debug_script = `
(() => {
function getPageStructure(element = document, depth = 0, maxDepth = 10) {
if (depth >= maxDepth) return '';
const indent = ' '.repeat(depth);
let structure = '';
// Skip certain elements that clutter the output
const skipTags = new Set(['script', 'style', 'link', 'meta', 'noscript']);
// Add current element info if it's not the document
if (element !== document) {
const tagName = element.tagName.toLowerCase();
// Skip uninteresting elements
if (skipTags.has(tagName)) return '';
const id = element.id ? \`#\${element.id}\` : '';
const classes = element.className && typeof element.className === 'string' ?
\`.\${element.className.split(' ').filter(c => c).join('.')}\` : '';
// Get additional useful attributes
const attrs = [];
if (element.getAttribute('role')) attrs.push(\`role="\${element.getAttribute('role')}"\`);
if (element.getAttribute('aria-label')) attrs.push(\`aria-label="\${element.getAttribute('aria-label')}"\`);
if (element.getAttribute('type')) attrs.push(\`type="\${element.getAttribute('type')}"\`);
if (element.getAttribute('name')) attrs.push(\`name="\${element.getAttribute('name')}"\`);
if (element.getAttribute('src')) {
const src = element.getAttribute('src');
attrs.push(\`src="\${src.substring(0, 50)}\${src.length > 50 ? '...' : ''}"\`);
}
// Add element info
structure += \`\${indent}\${tagName}\${id}\${classes}\${attrs.length ? ' [' + attrs.join(', ') + ']' : ''}\\n\`;
// Handle iframes specially
if (tagName === 'iframe') {
try {
const iframeDoc = element.contentDocument || element.contentWindow?.document;
if (iframeDoc) {
structure += \`\${indent} [IFRAME CONTENT]:\\n\`;
structure += getPageStructure(iframeDoc, depth + 2, maxDepth);
} else {
structure += \`\${indent} [IFRAME: No access - likely cross-origin]\\n\`;
}
} catch (e) {
}
}
}
// Get all child elements
const children = element.children || element.childNodes;
for (const child of children) {
if (child.nodeType === 1) { // Element nodes only
structure += getPageStructure(child, depth + 1, maxDepth);
}
}
return structure;
}
return getPageStructure();
})()
`;
return (await this.get_current_page()).evaluate(debug_script);
}
async get_state() {
await this._ensure_initialized();
await this._wait_for_page_and_frames_load();
const session = await this.get_session();
session.cached_state = await this._update_state();
if (this.config.cookies_file) {
await this.save_cookies(true);
}
else if (this.config.s3Storage) {
await this.save_cookies(false);
}
return session.cached_state;
}
async _update_state(focus_element = -1) {
await this._ensure_initialized();
const session = await this.get_session();
let page = undefined;
try {
page = await this.get_current_page();
await page.evaluate('1+1');
}
catch (e) {
const pages = await session.context.pages();
if (pages) {
this.state.target_id = null;
page = await this.get_current_page();
}
else {
throw new BrowserError(`Browser closed: no valid pages available ${e}`);
}
}
try {
await this.remove_highlights();
const dom_service = new DomService(page);
const content = await dom_service.getClickableElements(this.config.highlight_elements, focus_element, this.config.viewport_expansion);
const screenshot_b64 = await this.take_screenshot();
const [pixels_above, pixels_below] = await this.get_scroll_info(page);
this.current_state = new BrowserState(content.elementTree, content.selectorMap, await page.url(), await page.title(), await this.get_tabs_info(), screenshot_b64, pixels_above, pixels_below);
return this.current_state;
}
catch (e) {
// Return last known good state if available
if (this.current_state) {
return this.current_state;
}
throw e;
}
}
async take_screenshot(full_page = false) {
await this._ensure_initialized();
const page = await this.get_current_page();
await page.bringToFront();
await page.waitForLoadState();
const screenshot = await page.screenshot({
fullPage: full_page,
animations: 'disabled'
});
const screenshot_b64 = screenshot.toString('base64');
return screenshot_b64;
}
async remove_highlights() {
await this._ensure_initialized();
try {
const page = await this.get_current_page();
await page.evaluate(`
try {
// Remove the highlight container and all its contents
const container = document.getElementById('playwright-highlight-container');
if (container) {
container.remove();
}
// Remove highlight attributes from elements
const highlightedElements = document.querySelectorAll('[browser-user-highlight-id^="playwright-highlight-"]');
highlightedElements.forEach(el => {
el.removeAttribute('browser-user-highlight-id');
});
} catch (e) {
}
`);
}
catch (e) {
DEBUG.error('Error removing highlights', e);
// Don't raise the error since this is not critical functionality
}
}
async _convert_simple_xpath_to_css_selector(xpath) {
/**
* Converts simple XPath expressions to CSS selectors.
*
* @param xpath - The XPath expression to convert.
* @returns A CSS selector string.
*/
if (!xpath) {
return '';
}
// Remove leading slash if present
xpath = xpath.replace(/^\//, '');
// Split into parts
const parts = xpath.split('/');
const css_parts = [];
for (const part of parts) {
if (!part) {
continue;
}
// Handle custom elements with colons by escaping them
if (part.includes(':') && !part.includes('[')) {
const base_part = part.replace(/:/g, '\\:');
css_parts.push(base_part);
continue;
}
// Handle index notation [n]
if (part.includes('[')) {
let base_part = part.slice(0, part.indexOf('['));
// Handle custom elements with colons in the base part
if (base_part.includes(':')) {
base_part = base_part.replace(/:/g, '\\:');
}
const index_part = part.slice(part.indexOf('['));
// Handle multiple indices
const indices = index_part.split(']').slice(0, -1).map(i => i.replace(/\[|\]/g, ''));
for (const idx of indices) {
try {
// Handle numeric indices
if (!isNaN(Number(idx))) {
const index = parseInt(idx, 10) - 1;
base_part += `:nth-of-type(${index + 1})`;
// Handle last() function
}
else if (idx === 'last()') {
base_part += ':last-of-type';
// Handle position() functions
}
else if (idx.includes('position()')) {
if (idx.includes('>1')) {
base_part += ':nth-of-type(n+2)';
}
}
}
catch (e) {
DEBUG.error('Error enhancing CSS selector', e);
continue;
}
}
css_parts.push(base_part);
}
else {
css_parts.push(part);
}
}
return css_parts.join(' > ');
}
async _enhanced_css_selector_for_element(element, include_dynamic_attributes = true) {
/**
* Creates a CSS selector for a DOM element, handling various edge cases and special characters.
*
* @param element - The DOM element to create a selector for.
* @param include_dynamic_attributes - Whether to include dynamic attributes in the selector.
* @returns A valid CSS selector string.
*/
try {
// Get base selector from XPath
let css_selector = await this._convert_simple_xpath_to_css_selector(element.xpath);
// Handle class attributes
if ('class' in element.attributes && element.attributes['class'] && include_dynamic_attributes) {
// Define a regex pattern for valid class names in CSS
const valid_class_name_pattern = /^[a-zA-Z_][a-zA-Z0-9_-]*$/;
// Iterate through the class attribute values
const classes = element.attributes['class'].split(' ');
for (const class_name of classes) {
// Skip empty class names
if (!class_name.trim()) {
continue;
}
// Check if the class name is valid
if (valid_class_name_pattern.test(class_name)) {
// Append the valid class name to the CSS selector
css_selector += `.${class_name}`;
}
}
}
// Expanded set of safe attributes that are stable and useful for selection
const SAFE_ATTRIBUTES = new Set([
'id', 'name', 'type', 'placeholder',
'aria-label', 'aria-labelledby', 'aria-describedby', 'role',
'for', 'autocomplete', 'required', 'readonly',
'alt', 'title', 'src', 'href', 'target',
]);
if (include_dynamic_attributes) {
const dynamic_attributes = ['data-id', 'data-qa', 'data-cy', 'data-testid'];
dynamic_attributes.forEach(attr => SAFE_ATTRIBUTES.add(attr));
}
// Handle other attributes
for (const [attribute, value] of Object.entries(element.attributes)) {
if (attribute === 'class') {
continue;
}
// Skip invalid attribute names
if (!attribute.trim()) {
continue;
}
if (!SAFE_ATTRIBUTES.has(attribute)) {
continue;
}
// Escape special characters in attribute names
const safe_attribute = attribute.replace(/:/g, '\\:');
// Handle different value cases
if (value === '') {
css_selector += `[${safe_attribute}]`;
}
else if (/["'<>`\n\r\t]/.test(value)) {
// Use contains for values with special characters
const collapsed_value = value.replace(/\s+/g, ' ').trim();
const safe_value = collapsed_value.replace(/"/g, '\\"');
css_selector += `[${safe_attribute}*="${safe_value}"]`;
}
else {
css_selector += `[${safe_attribute}="${value}"]`;
}
}
return css_selector;
}
catch (e) {
DEBUG.error('Error enhancing CSS selector, Falling back to basic one', e);
// Fallback to a more basic selector if something goes wrong
const tag_name = element.tagName || '*';
return `${tag_name}[highlight_index='${element.highlightIndex}']`;
}
}
async get_locate_element(element) {
await this._ensure_initialized();
if (!element) {
DEBUG.error('Element is null or undefined in get_locate_element');
return null;
}
let current_frame = await (this.get_current_page());
let current = element;
const parent = [];
while (current.parent !== null) {
parent.push(current.parent);
current = current.parent;
}
parent.reverse();
const iframes = parent.filter(item => item.tagName === 'iframe');
for (const iframe of iframes) {
const css_selector = await this._convert_simple_xpath_to_css_selector(iframe.xpath);
current_frame = current_frame.frameLocator(css_selector);
}
// Generate CSS selector from XPath
const css_selector = await this._convert_simple_xpath_to_css_selector(element.xpath);
DEBUG.info(`Attempting to locate element with xpath: ${element.xpath}`);
DEBUG.info(`Converted CSS selector: ${css_selector}`);
// Try multiple strategies to locate the element
try {
// STRATEGY 1: Try locating with the converted CSS selector
if ('frameLocator' in current_frame && typeof current_frame.frameLocator === 'function') {
// Try with locator first
DEBUG.info(`Strategy 1: Trying to locate using frameLocator.locator(${css_selector})`);
try {
const element_handle = await current_frame.locator(css_selector).elementHandle({ timeout: 2000 });
if (element_handle) {
DEBUG.info(`Successfully found element with locator`);
return element_handle;
}
}
catch (locatorErr) {
DEBUG.warn(`Failed to locate using CSS selector: ${locatorErr.message}`);
}
// STRATEGY 2: Try using enhanced CSS selector with attributes
DEBUG.info('Strategy 2: Trying enhanced CSS selector with attributes');
try {
const enhanced_selector = await this._enhanced_css_selector_for_element(element, true);
if (enhanced_selector !== css_selector) {
DEBUG.info(`Enhanced CSS selector: ${enhanced_selector}`);
const element_handle = await current_frame.locator(enhanced_selector).elementHandle({ timeout: 2000 });
if (element_handle) {
DEBUG.info(`Successfully found element with enhanced CSS selector`);
return element_handle;
}
}
}
catch (enhancedSelectorErr) {
DEBUG.warn(`Failed to locate using enhanced CSS selector: ${enhancedSelectorErr.message}`);
}
// STRATEGY 3: Try using text content if available
if (element.attributes && 'textContent' in element.attributes && element.attributes['textContent'] && element.attributes['textContent'].trim()) {
DEBUG.info(`Strategy 3: Trying to locate using text content: "${element.attributes['textContent'].trim()}"`);
try {
const trimmedText = element.attributes['textContent'].trim().substring(0, 50); // Use first 50 chars to avoid overly long text
const element_handle = await current_frame.locator(`text=${trimmedText}`).elementHandle({ timeout: 2000 });
if (element_handle) {
DEBUG.info(`Successfully found element with text content`);
return element_handle;
}
}
catch (textErr) {
DEBUG.warn(`Failed to locate using text content: ${textErr.message}`);
}
}
// STRATEGY 4: Try using nth child as a fallback if highlight index is present
if (element.highlightIndex !== null && element.highlightIndex !== undefined) {
DEBUG.info(`Strategy 4: Trying to locate using highlightIndex: ${element.highlightIndex}`);
try {
// Try by tag name and index
const tagSelector = `${element.tagName}:nth-child(${element.highlightIndex})`;
const element_handle = await current_frame.locator(tagSelector).first().elementHandle({ timeout: 2000 });
if (element_handle) {
DEBUG.info(`Successfully found element with tag selector: ${tagSelector}`);
return element_handle;
}
}
catch (indexErr) {
DEBUG.warn(`Failed to locate using highlightIndex: ${indexErr.message}`);
}
// Try locating by attribute
try {
const attributeSelector = `[highlight_index='${element.highlightIndex}']`;
const element_handle = await current_frame.locator(attributeSelector).elementHandle({ timeout: 2000 });
if (element_handle) {
DEBUG.info(`Successfully found element with highlight_index attribute`);
return element_handle;
}
}
catch (attrErr) {
DEBUG.warn(`Failed to locate using highlight_index attribute: ${attrErr.message}`);
}
}
DEBUG.error(`All strategies failed to locate element with xpath: ${element.xpath}`);
return null;
}
else {
// STRATEGY 1: Try querySelector with CSS selector
DEBUG.info(`Strategy 1: Trying to locate using page.querySelector(${css_selector})`);
try {
const element_handle = await current_frame.querySelector(css_selector);
if (element_handle) {
await element_handle.scrollIntoViewIfNeeded();
DEBUG.info(`Successfully found element with querySelector`);
return element_handle;
}
}
catch (querySelectorErr) {
DEBUG.warn(`Failed to locate using querySelector: ${querySelectorErr.message}`);
}
// STRATEGY 2: Try using enhanced CSS selector with attributes
DEBUG.info('Strategy 2: Trying enhanced CSS selector with attributes');
try {
const enhanced_selector = await this._enhanced_css_selector_for_element(element, true);
if (enhanced_selector !== css_selector) {
DEBUG.info(`Enhanced CSS selector: ${enhanced_selector}`);
const element_handle = await current_frame.querySelector(enhanced_selector);
if (element_handle) {
await element_handle.scrollIntoViewIfNeeded();
DEBUG.info(`Successfully found element with enhanced CSS selector`);
return element_handle;
}
}
}
catch (enhancedSelectorErr) {
DEBUG.warn(`Failed to locate using enhanced CSS selector: ${enhancedSelectorErr.message}`);
}
// STRATEGY 3: Try evaluating xpath directly
DEBUG.info(`Strategy 3: Trying to evaluate XPath directly: ${element.xpath}`);
try {
const elements = await current_frame.$$eval(`xpath=${element.xpath}`, (nodes) => nodes.length > 0);
if (elements) {
const element_handle = await current_frame.$(`xpath=${element.xpath}`);
if (element_handle) {
await element_handle.scrollIntoViewIfNeeded();
DEBUG.info(`Successfully found element with direct XPath evaluation`);
return element_handle;
}
}
}
catch (xpathErr) {
DEBUG.warn(`Failed to locate using direct XPath evaluation: ${xpathErr.message}`);
}
// STRATEGY 4: Try using nth child as a fallback if highlight index is present
if (element.highlightIndex !== null && element.highlightIndex !== undefined) {
DEBUG.info(`Strategy 4: Trying to locate using highlightIndex: ${element.highlightIndex}`);
try {
// Try by tag name and index
const tagSelector = `${element.tagName}:nth-child(${element.highlightIndex})`;
const element_handle = await current_frame.querySelector(tagSelector);
if (element_handle) {
await element_handle.scrollIntoViewIfNeeded();
DEBUG.info(`Successfully found element with tag selector: ${tagSelector}`);
return element_handle;
}
}
catch (indexErr) {
DEBUG.warn(`Failed to locate using highlightIndex: ${indexErr.message}`);
}
// Try locating by attribute
try {
const attributeSelector = `[highlight_index='${element.highlightIndex}']`;
const element_handle = await current_frame.querySelector(attributeSelector);
if (element_handle) {
await element_handle.scrollIntoViewIfNeeded();
DEBUG.info(`Successfully found element with highlight_index attribute`);
return element_handle;
}
}
catch (attrErr) {
DEBUG.warn(`Failed to locate using highlight_index attribute: ${attrErr.message}`);
}
}
DEBUG.error(`All strategies failed to locate element with xpath: ${element.xpath}`);
return null;
}
}
catch (e) {
DEBUG.error(`Error in get_locate_element: ${e.message}`);
DEBUG.error(`Element details - tagName: ${element.tagName}, xpath: ${element.xpath}, highlightIndex: ${element.highlightIndex}`);
return null;
}
}
async get_locate_element_by_xpath(xpath) {
await this._ensure_initialized();
DEBUG.debug(`Locating element by XPath: ${xpath}`);
const currentFrame = await this.get_current_page();
try {
// Use XPath to locate the element
const elementHandle = await currentFrame.locator(`xpath=${xpath}`).elementHandle();
if (elementHandle) {
await elementHandle.scrollIntoViewIfNeeded();
return elementHandle;
}
return null;
}
catch (e) {
DEBUG.error(`Failed to locate element by XPath ${xpath}: ${e}`);
return null;
}
}
async get_locate_element_by_css_selector(css_selector) {
await this._ensure_initialized();
DEBUG.debug(`Locating element by CSS selector: ${css_selector}`);
const currentFrame = await this.get_current_page();
try {
// Use CSS selector to locate the element
const elementHandle = await currentFrame.locator(css_selector).elementHandle();
if (elementHandle) {
await elementHandle.scrollIntoViewIfNeeded();
return elementHandle;
}
return null;
}
catch (e) {
DEBUG.error(`Failed to locate element by CSS selector ${css_selector}: ${e}`);
return null;
}
}
async get_locate_element_by_text(text, nth = 0) {
await this._ensure_initialized();
DEBUG.debug(`Locating element by text: "${text}" (nth: ${nth})`);
const currentFrame = await this.get_current_page();
try {
const elements = await currentFrame.locator(`text=${text}`).elementHandles();
// Filter to get only visible elements
const visibleElements = [];
for (const el of elements) {
if (await el.isVisible()) {
visibleElements.push(el);
}
}
if (visibleElements.length === 0) {
DEBUG.error(`No visible element with text '${text}' found.`);
return null;
}
if (nth !== null && nth !== undefined) {
if (0 <= nth && nth < visibleElements.length) {
const elementHandle = visibleElements[nth];
await elementHandle.scrollIntoViewIfNeeded();
return elementHandle;
}
else {
DEBUG.error(`Visible element with text '${text}' not found at index ${nth}.`);
return null;
}
}
else {
const elementHandle = visibleElements[0];
await elementHandle.scrollIntoViewIfNeeded();
return elementHandle;
}
}
catch (e) {
DEBUG.error(`Failed to locate element by text '${text}': ${e}`);
return null;
}
}
async _input_text_element_node(element_node, text) {
await this._ensure_initialized();
try {
await this._update_state(element_node.highlightIndex || undefined);
const element_handle = await this.get_locate_element(element_node);
if (element_handle === null) {
throw new BrowserError(`Element: ${element_node} not found`);
}
try {
await element_handle.waitForElementState('stable', { timeout: 1000 });
await element_handle.scrollIntoViewIfNeeded({ timeout: 1000 });
}
catch (scrollError) {
DEBUG.warn(`Unable to stabilize or scroll element: ${scrollError.message}`);
}
const tag_handle = await element_handle.getProperty('tagName');
const tag_name = (await tag_handle.jsonValue()).toLowerCase();
const is_contenteditable = await element_handle.getProperty('isContentEditable');
const readonly_handle = await element_handle.getProperty('readOnly');
const disabled_handle = await element_handle.getProperty('disabled');
const readonly = (await readonly_handle.jsonValue());
const disabled = (await disabled_handle.jsonValue());
// Fix: Properly await and separate the content editable check
const isContentEditable = await is_contenteditable.jsonValue();
DEBUG.info(`Element input details: tag=${tag_name}, contentEditable=${isContentEditable}, readonly=${readonly}, disabled=${disabled}`);
if ((isContentEditable || tag_name === 'input') && !(readonly || disabled)) {
await element_handle.evaluate('el=>el.textContent=""');
await element_handle.type(text, { delay: 5 });
}
else {
await element_handle.fill(text);
}
}
catch (e) {
DEBUG.error(`Input text error for element index ${element_node.highlightIndex}:`, e);
throw new BrowserError(`Failed to input text into element index ${element_node.highlightIndex}: ${e.message}`);
}
}
async _click_element_node(element_node) {
await this._ensure_initialized();
const page = await this.get_current_page();
try {
// First update state and highlight the element
await this._update_state(element_node.highlightIndex || undefined);
// Locate the element with more detailed diagnostics
DEBUG.info(`Attempting to locate element for clicking with xpath: ${element_node.xpath}`);
const element_handle = await this.get_locate_element(elemen