@ui-tars/operator-browser
Version:
Native-browser operator for UI-TARS
1 lines • 36.6 kB
Source Map (JSON)
{"version":3,"file":"ui-helper.mjs","sources":["webpack://@ui-tars/operator-browser/./src/ui-helper.ts"],"sourcesContent":["/*\n * Copyright (c) 2025 Bytedance, Inc. and its affiliates.\n * SPDX-License-Identifier: Apache-2.0\n */\nimport { Page } from '@agent-infra/browser';\nimport { Logger } from '@agent-infra/logger';\nimport { ParsedPrediction } from './types';\n\n/**\n * Helper class for UI interactions in the browser\n * Provides visual feedback for actions and information display\n */\nexport class UIHelper {\n private styleId = 'gui-agent-helper-styles';\n private containerId = 'gui-agent-helper-container';\n private highlightClass = 'gui-agent-clickable-highlight';\n private waterFlowId = 'gui-agent-water-flow';\n\n /**\n * Creates a new UIHelper instance\n * @param getCurrentPage Function that returns the current active page\n */\n constructor(\n private getCurrentPage: () => Promise<Page>,\n private logger: Logger,\n ) {\n this.logger = logger.spawn('[UIHelper]');\n }\n\n /**\n * Injects required CSS styles into the page\n * Creates styling for action indicators and information panels\n */\n private async injectStyles() {\n const page = await this.getCurrentPage();\n await page.evaluate((styleId: string) => {\n if (document.getElementById(styleId)) return;\n\n const style = document.createElement('style');\n style.id = styleId;\n style.textContent = `\n #gui-agent-helper-container {\n position: fixed;\n top: 20px;\n right: 20px;\n background: rgba(0, 0, 0, 0.85); \n color: white;\n padding: 15px 20px;\n border-radius: 12px;\n font-family: system-ui;\n z-index: 999999;\n max-width: 320px;\n backdrop-filter: blur(8px);\n box-shadow: 0 8px 32px rgba(0, 0, 0, 0.2);\n border: 1px solid rgba(255, 255, 255, 0.1);\n }\n\n .gui-agent-title {\n font-size: 14px;\n font-weight: 600;\n margin-bottom: 10px;\n color: #00ff9d;\n text-transform: uppercase;\n letter-spacing: 0.5px;\n }\n\n .gui-agent-content {\n font-size: 13px;\n line-height: 1.5;\n color: rgba(255, 255, 255, 0.9);\n }\n\n .gui-agent-coords {\n margin-top: 8px;\n font-size: 12px;\n color: #00ff9d;\n opacity: 0.8;\n }\n\n .gui-agent-thought {\n margin-top: 10px;\n padding-top: 10px;\n border-top: 1px solid rgba(255, 255, 255, 0.15);\n font-style: italic;\n color: rgba(255, 255, 255, 0.7);\n font-size: 12px;\n }\n\n .gui-agent-click-indicator {\n position: fixed;\n pointer-events: none;\n width: 60px;\n height: 60px;\n border-radius: 50%;\n border: 4px solid #00ff9d;\n background: rgba(0, 255, 157, 0.3);\n transform: translate(-50%, -50%);\n animation: click-pulse 1.2s ease-out;\n z-index: 2147483647;\n }\n\n .gui-agent-click-indicator::before {\n content: '';\n position: absolute;\n top: 50%;\n left: 50%;\n width: 12px;\n height: 12px;\n background: #00ff9d;\n border-radius: 50%;\n transform: translate(-50%, -50%);\n }\n\n .gui-agent-drag-indicator {\n position: fixed;\n pointer-events: none;\n z-index: 2147483647;\n }\n\n .gui-agent-drag-start {\n width: 40px;\n height: 40px;\n border-radius: 50%;\n border: 3px solid #ff6b00;\n background: rgba(255, 107, 0, 0.4);\n transform: translate(-50%, -50%);\n position: absolute;\n }\n\n .gui-agent-drag-end {\n width: 40px;\n height: 40px;\n border-radius: 50%;\n border: 3px solid #00c3ff;\n background: rgba(0, 195, 255, 0.4);\n transform: translate(-50%, -50%);\n position: absolute;\n }\n\n .gui-agent-drag-path {\n position: absolute;\n height: 6px;\n background: linear-gradient(to right, #ff6b00, #00c3ff);\n border-radius: 3px;\n transform-origin: left center;\n opacity: 0.7;\n }\n\n .gui-agent-drag-arrow {\n position: absolute;\n width: 0;\n height: a0;\n border-top: 12px solid transparent;\n border-bottom: 12px solid transparent;\n border-left: 16px solid #00c3ff;\n transform-origin: left center;\n right: -16px;\n top: -9px;\n }\n\n .gui-agent-clickable-highlight {\n outline: 3px solid rgba(0, 155, 255, 0.7) !important;\n box-shadow: 0 0 0 3px rgba(0, 155, 255, 0.3) !important;\n background-color: rgba(0, 155, 255, 0.05) !important;\n transition: all 0.2s ease-in-out !important;\n z-index: 999 !important;\n position: relative !important;\n }\n\n .gui-agent-clickable-highlight:hover {\n outline: 4px solid rgba(0, 155, 255, 0.9) !important;\n background-color: rgba(0, 155, 255, 0.1) !important;\n }\n\n .gui-agent-clickable-highlight.gui-highlight-button {\n outline: 3px solid rgba(255, 64, 129, 0.8) !important;\n box-shadow: 0 0 0 3px rgba(255, 64, 129, 0.3) !important;\n background-color: rgba(255, 64, 129, 0.05) !important;\n }\n\n .gui-agent-clickable-highlight.gui-highlight-button:hover {\n outline: 4px solid rgba(255, 64, 129, 0.9) !important;\n background-color: rgba(255, 64, 129, 0.1) !important;\n }\n\n .gui-agent-clickable-highlight.gui-highlight-link {\n outline: 3px solid rgba(124, 77, 255, 0.8) !important;\n box-shadow: 0 0 0 3px rgba(124, 77, 255, 0.3) !important;\n background-color: rgba(124, 77, 255, 0.05) !important;\n }\n\n .gui-agent-clickable-highlight.gui-highlight-link:hover {\n outline: 4px solid rgba(124, 77, 255, 0.9) !important;\n background-color: rgba(124, 77, 255, 0.1) !important;\n }\n\n .gui-agent-clickable-highlight.gui-highlight-input {\n outline: 3px solid rgba(0, 230, 118, 0.8) !important;\n box-shadow: 0 0 0 3px rgba(0, 230, 118, 0.3) !important;\n background-color: rgba(0, 230, 118, 0.05) !important;\n }\n\n .gui-agent-clickable-highlight.gui-highlight-input:hover {\n outline: 4px solid rgba(0, 230, 118, 0.9) !important;\n background-color: rgba(0, 230, 118, 0.1) !important;\n }\n\n .gui-agent-clickable-highlight.gui-highlight-other {\n outline: 3px solid rgba(255, 171, 0, 0.8) !important;\n box-shadow: 0 0 0 3px rgba(255, 171, 0, 0.3) !important;\n background-color: rgba(255, 171, 0, 0.05) !important;\n }\n\n .gui-agent-clickable-highlight.gui-highlight-other:hover {\n outline: 4px solid rgba(255, 171, 0, 0.9) !important;\n background-color: rgba(255, 171, 0, 0.1) !important;\n }\n\n .gui-agent-legend {\n position: fixed;\n bottom: 20px;\n left: 20px;\n background: rgba(0, 0, 0, 0.85);\n color: white;\n padding: 12px 18px;\n border-radius: 10px;\n font-family: system-ui;\n font-size: 12px;\n z-index: 999999;\n backdrop-filter: blur(8px);\n box-shadow: 0 4px 16px rgba(0, 0, 0, 0.2);\n border: 1px solid rgba(255, 255, 255, 0.1);\n display: flex;\n flex-direction: column;\n gap: 8px;\n }\n\n .gui-agent-legend-title {\n font-weight: 600;\n margin-bottom: 4px;\n font-size: 13px;\n }\n\n .gui-agent-legend-item {\n display: flex;\n align-items: center;\n gap: 8px;\n }\n\n .gui-agent-legend-icon {\n display: inline-block;\n width: 14px;\n height: 14px;\n border-radius: 3px;\n border: 2px solid rgba(255, 255, 255, 0.8);\n }\n\n .gui-legend-button {\n background: rgba(255, 64, 129, 0.7);\n }\n\n .gui-legend-link {\n background: rgba(124, 77, 255, 0.7);\n }\n\n .gui-legend-input {\n background: rgba(0, 230, 118, 0.7);\n }\n\n .gui-legend-other {\n background: rgba(255, 171, 0, 0.7);\n }\n\n @keyframes click-pulse {\n 0% {\n transform: translate(-50%, -50%) scale(0.8);\n opacity: 1;\n }\n 100% {\n transform: translate(-50%, -50%) scale(2.5);\n opacity: 0;\n }\n }\n `;\n document.head.appendChild(style);\n }, this.styleId);\n }\n\n /**\n * Displays information about the current action being performed\n * @param prediction The parsed prediction containing action details\n */\n async showActionInfo(prediction: ParsedPrediction) {\n this.logger.info('Showing action info ...');\n await this.injectStyles();\n\n const { action_type, action_inputs, thought } = prediction;\n const page = await this.getCurrentPage();\n\n await page.evaluate(\n (params) => {\n const { containerId, action_type, action_inputs, thought } = params;\n\n let container = document.getElementById(containerId);\n if (!container) {\n container = document.createElement('div');\n container.id = containerId;\n document.body.appendChild(container);\n }\n\n const actionMap = {\n click: '🖱️ Single Click',\n left_click: '🖱️ Single Click',\n left_single: '🖱️ Single Click',\n double_click: '🖱️ Double Click',\n left_double: '🖱️ Double Click',\n right_click: '🖱️ Right Click',\n type: `⌨️ Type: \"${action_inputs.content}\"`,\n navigate: `🌐 Navigate to: ${action_inputs.content}`,\n hotkey: `⌨️ Hotkey: ${action_inputs.key || action_inputs.hotkey}`,\n scroll: `📜 Scroll ${action_inputs.direction}`,\n wait: '⏳ Wait',\n };\n\n // @ts-expect-error\n const actionText = actionMap[action_type] || action_type;\n\n container.innerHTML = `\n <div class=\"gui-agent-title\">Next Action</div>\n <div class=\"gui-agent-content\">${actionText}</div>\n ${thought ? `<div class=\"gui-agent-thought\">${thought}</div>` : ''}\n `;\n },\n { containerId: this.containerId, action_type, action_inputs, thought },\n );\n this.logger.info('Showing action info done.');\n }\n\n /**\n * Shows a visual click indicator at the specified coordinates\n * @param x X coordinate for the click\n * @param y Y coordinate for the click\n */\n async showClickIndicator(x: number, y: number) {\n this.logger.info('Showing click indicator...');\n await this.injectStyles();\n const page = await this.getCurrentPage();\n\n await page.evaluate(\n // eslint-disable-next-line no-shadow\n ({ x, y, containerId }) => {\n // Remove any existing indicators\n const existingIndicators = document.querySelectorAll(\n '.gui-agent-click-indicator',\n );\n existingIndicators.forEach((el) => el.remove());\n\n // Create new indicator\n const indicator = document.createElement('div');\n indicator.className = 'gui-agent-click-indicator';\n indicator.style.left = `${x}px`;\n indicator.style.top = `${y}px`;\n document.body.appendChild(indicator);\n\n // Update coords in container\n const container = document.getElementById(containerId);\n if (container) {\n const coordsDiv = document.createElement('div');\n coordsDiv.className = 'gui-agent-coords';\n coordsDiv.textContent = `Click at: (${Math.round(x)}, ${Math.round(y)})`;\n\n const existingCoords = container.querySelector('.gui-agent-coords');\n if (existingCoords) {\n existingCoords.remove();\n }\n\n container.appendChild(coordsDiv);\n }\n\n // Remove indicator after animation\n setTimeout(() => {\n indicator.remove();\n }, 1200);\n },\n { x, y, containerId: this.containerId },\n );\n this.logger.info('Showing click indicator done.');\n }\n\n async showWaterFlow() {\n this.logger.info('Showing water flow effect...');\n\n await this.injectStyles();\n const page = await this.getCurrentPage();\n\n await page.evaluate((waterFlowId: string) => {\n if (document.getElementById(waterFlowId)) return;\n\n const waterFlow = document.createElement('div');\n waterFlow.id = waterFlowId;\n waterFlow.style.cssText = `\n position: fixed;\n top: 0;\n left: 0;\n right: 0;\n bottom: 0;\n pointer-events: none;\n z-index: 2147483647;\n `;\n\n const style = document.createElement('style');\n style.textContent = `\n #${waterFlowId}::before {\n content: \"\";\n position: fixed;\n top: 0; right: 0; bottom: 0; left: 0;\n pointer-events: none;\n z-index: 9999;\n background:\n linear-gradient(to right, rgba(30, 144, 255, 0.4), transparent 50%) left,\n linear-gradient(to left, rgba(30, 144, 255, 0.4), transparent 50%) right,\n linear-gradient(to bottom, rgba(30, 144, 255, 0.4), transparent 50%) top,\n linear-gradient(to top, rgba(30, 144, 255, 0.4), transparent 50%) bottom;\n background-repeat: no-repeat;\n background-size: 10% 100%, 10% 100%, 100% 10%, 100% 10%;\n animation: waterflow 5s cubic-bezier(0.4, 0, 0.6, 1) infinite;\n filter: blur(8px);\n }\n\n @keyframes waterflow {\n 0%, 100% {\n background-image:\n linear-gradient(to right, rgba(30, 144, 255, 0.4), transparent 50%),\n linear-gradient(to left, rgba(30, 144, 255, 0.4), transparent 50%),\n linear-gradient(to bottom, rgba(30, 144, 255, 0.4), transparent 50%),\n linear-gradient(to top, rgba(30, 144, 255, 0.4), transparent 50%);\n transform: scale(1);\n }\n 25% {\n background-image:\n linear-gradient(to right, rgba(30, 144, 255, 0.39), transparent 52%),\n linear-gradient(to left, rgba(30, 144, 255, 0.39), transparent 52%),\n linear-gradient(to bottom, rgba(30, 144, 255, 0.39), transparent 52%),\n linear-gradient(to top, rgba(30, 144, 255, 0.39), transparent 52%);\n transform: scale(1.03);\n }\n 50% {\n background-image:\n linear-gradient(to right, rgba(30, 144, 255, 0.38), transparent 55%),\n linear-gradient(to left, rgba(30, 144, 255, 0.38), transparent 55%),\n linear-gradient(to bottom, rgba(30, 144, 255, 0.38), transparent 55%),\n linear-gradient(to top, rgba(30, 144, 255, 0.38), transparent 55%);\n transform: scale(1.05);\n }\n 75% {\n background-image:\n linear-gradient(to right, rgba(30, 144, 255, 0.39), transparent 52%),\n linear-gradient(to left, rgba(30, 144, 255, 0.39), transparent 52%),\n linear-gradient(to bottom, rgba(30, 144, 255, 0.39), transparent 52%),\n linear-gradient(to top, rgba(30, 144, 255, 0.39), transparent 52%);\n transform: scale(1.03);\n }\n }\n `;\n\n document.head.appendChild(style);\n document.body.appendChild(waterFlow);\n }, this.waterFlowId);\n\n this.logger.info('Water flow effect shown.');\n }\n\n async hideWaterFlow() {\n this.logger.info('Hiding water flow effect...');\n\n const page = await this.getCurrentPage();\n\n await page.evaluate((waterFlowId: string) => {\n const waterFlow = document.getElementById(waterFlowId);\n if (waterFlow) {\n waterFlow.remove();\n }\n }, this.waterFlowId);\n\n this.logger.info('Water flow effect hidden.');\n }\n\n /**\n * Highlights all clickable elements on the page using SoM-inspired approach\n * Should be called before taking a screenshot to show interactive elements\n */\n async highlightClickableElements() {\n this.logger.info('Highlighting clickable elements...');\n await this.injectStyles();\n const page = await this.getCurrentPage();\n\n // Remove any existing highlights first\n await this.removeClickableHighlights();\n\n await page.evaluate((highlightClass) => {\n // Create a legend to explain the highlighting\n const createLegend = () => {\n const legend = document.createElement('div');\n legend.className = 'gui-agent-legend';\n legend.id = 'gui-agent-clickable-legend';\n legend.innerHTML = `\n <div class=\"gui-agent-legend-title\">Clickable Elements</div>\n <div class=\"gui-agent-legend-item\">\n <span class=\"gui-agent-legend-icon gui-legend-button\"></span>\n <span>Buttons</span>\n </div>\n <div class=\"gui-agent-legend-item\">\n <span class=\"gui-agent-legend-icon gui-legend-link\"></span>\n <span>Links</span>\n </div>\n <div class=\"gui-agent-legend-item\">\n <span class=\"gui-agent-legend-icon gui-legend-input\"></span>\n <span>Input Fields</span>\n </div>\n <div class=\"gui-agent-legend-item\">\n <span class=\"gui-agent-legend-icon gui-legend-other\"></span>\n <span>Other Clickables</span>\n </div>\n `;\n document.body.appendChild(legend);\n };\n\n createLegend();\n\n // Common clickable selectors\n const buttonSelectors = [\n 'button',\n '[role=\"button\"]',\n '.btn',\n '.button',\n '[type=\"button\"]',\n '[type=\"submit\"]',\n '[type=\"reset\"]',\n ];\n\n const linkSelectors = ['a', '[role=\"link\"]', '.nav-item'];\n\n const inputSelectors = [\n 'input',\n 'select',\n 'textarea',\n '[role=\"checkbox\"]',\n '[role=\"radio\"]',\n '[role=\"textbox\"]',\n '[contenteditable=\"true\"]',\n ];\n\n const otherSelectors = [\n '[role=\"tab\"]',\n '[role=\"menuitem\"]',\n '[role=\"option\"]',\n '[onclick]',\n '[tabindex=\"0\"]',\n '.clickable',\n '.selectable',\n 'summary',\n 'details',\n 'label',\n ];\n\n // Helper function to add highlight class with specific type\n const highlightElements = (selectors: string[], typeClass: string) => {\n const selector = selectors.join(', ');\n const elements = Array.from(document.querySelectorAll(selector));\n\n // Filter out hidden or disabled elements\n const visibleElements = elements.filter((el) => {\n const rect = el.getBoundingClientRect();\n const style = window.getComputedStyle(el);\n const isVisible =\n rect.width > 0 &&\n rect.height > 0 &&\n style.display !== 'none' &&\n style.visibility !== 'hidden' &&\n style.opacity !== '0';\n\n // Check if element or its ancestor has pointer-events: none\n let current = el as HTMLElement;\n let hasPointerEvents = true;\n while (current && current !== document.body) {\n if (window.getComputedStyle(current).pointerEvents === 'none') {\n hasPointerEvents = false;\n break;\n }\n current = current.parentElement as HTMLElement;\n }\n\n // Check if element is disabled\n const isDisabled =\n (el as HTMLElement).hasAttribute('disabled') ||\n (el as HTMLElement).getAttribute('aria-disabled') === 'true';\n\n return isVisible && hasPointerEvents && !isDisabled;\n });\n\n // Add highlight class to visible clickable elements\n visibleElements.forEach((el) => {\n el.classList.add(highlightClass);\n el.classList.add(typeClass);\n });\n\n return visibleElements.length;\n };\n\n // Highlight different types of elements\n const buttonCount = highlightElements(\n buttonSelectors,\n 'gui-highlight-button',\n );\n const linkCount = highlightElements(linkSelectors, 'gui-highlight-link');\n const inputCount = highlightElements(\n inputSelectors,\n 'gui-highlight-input',\n );\n const otherCount = highlightElements(\n otherSelectors,\n 'gui-highlight-other',\n );\n\n // Return stats for logging\n return {\n buttons: buttonCount,\n links: linkCount,\n inputs: inputCount,\n others: otherCount,\n total: buttonCount + linkCount + inputCount + otherCount,\n };\n }, this.highlightClass);\n this.logger.info('Highlighting clickable elements done.');\n }\n\n /**\n * Removes highlighting from clickable elements\n */\n async removeClickableHighlights() {\n this.logger.info('Removing clickable highlights...');\n try {\n const page = await this.getCurrentPage();\n await page.evaluate((highlightClass) => {\n // Remove all highlight classes\n const highlightedElements = document.querySelectorAll(\n `.${highlightClass}`,\n );\n highlightedElements.forEach((el) => {\n el.classList.remove(highlightClass);\n el.classList.remove('gui-highlight-button');\n el.classList.remove('gui-highlight-link');\n el.classList.remove('gui-highlight-input');\n el.classList.remove('gui-highlight-other');\n });\n\n // Remove the legend if it exists\n const legend = document.getElementById('gui-agent-clickable-legend');\n if (legend) {\n legend.remove();\n }\n }, this.highlightClass);\n } catch (error) {\n // Silently handle errors during cleanup\n console.error('Error removing clickable highlights:', error);\n }\n this.logger.info('Removing clickable highlights done.');\n }\n\n async cleanupTemporaryVisuals() {\n try {\n this.logger.info('cleanupTemporaryVisuals up...');\n const page = await this.getCurrentPage();\n await page.evaluate((containerId: string) => {\n const container = document.getElementById(containerId);\n if (container) {\n container.remove();\n }\n }, this.containerId);\n this.logger.info('cleanupTemporaryVisuals up done!');\n } catch (error) {\n // Silently handle errors during cleanup\n console.error('Error during UIHelper cleanup:', error);\n }\n }\n\n /**\n * Removes all UI helper elements from the page\n */\n async cleanup() {\n try {\n this.logger.info('Cleaning up...');\n await this.removeClickableHighlights();\n await this.hideWaterFlow();\n\n const page = await this.getCurrentPage();\n await page.evaluate((containerId: string) => {\n const container = document.getElementById(containerId);\n if (container) {\n container.remove();\n }\n }, this.containerId);\n this.logger.info('Cleaning up done!');\n } catch (error) {\n // Silently handle errors during cleanup\n console.error('Error during UIHelper cleanup:', error);\n }\n }\n\n /**\n * Shows a visual drag indicator from start to end coordinates\n * @param startX Starting X coordinate\n * @param startY Starting Y coordinate\n * @param endX Ending X coordinate\n * @param endY Ending Y coordinate\n */\n async showDragIndicator(\n startX: number,\n startY: number,\n endX: number,\n endY: number,\n ) {\n this.logger.info('Showing drag indicator...');\n await this.injectStyles();\n const page = await this.getCurrentPage();\n\n await page.evaluate(\n // eslint-disable-next-line no-shadow\n ({ startX, startY, endX, endY, containerId }) => {\n // Remove any existing indicators\n const existingIndicators = document.querySelectorAll(\n '.gui-agent-drag-indicator',\n );\n existingIndicators.forEach((el) => el.remove());\n\n // Create container for the drag indicator\n const dragIndicator = document.createElement('div');\n dragIndicator.className = 'gui-agent-drag-indicator';\n document.body.appendChild(dragIndicator);\n\n // Create start point indicator\n const startPoint = document.createElement('div');\n startPoint.className = 'gui-agent-drag-start';\n startPoint.style.left = `${startX}px`;\n startPoint.style.top = `${startY}px`;\n dragIndicator.appendChild(startPoint);\n\n // Create end point indicator\n const endPoint = document.createElement('div');\n endPoint.className = 'gui-agent-drag-end';\n endPoint.style.left = `${endX}px`;\n endPoint.style.top = `${endY}px`;\n dragIndicator.appendChild(endPoint);\n\n // Create drag path\n const dragPath = document.createElement('div');\n dragPath.className = 'gui-agent-drag-path';\n\n // Calculate path position and rotation\n const dx = endX - startX;\n const dy = endY - startY;\n const length = Math.sqrt(dx * dx + dy * dy);\n const angle = Math.atan2(dy, dx) * (180 / Math.PI);\n\n dragPath.style.width = `${length}px`;\n dragPath.style.left = `${startX}px`;\n dragPath.style.top = `${startY}px`;\n dragPath.style.transform = `rotate(${angle}deg)`;\n\n // Add arrow for direction\n const arrow = document.createElement('div');\n arrow.className = 'gui-agent-drag-arrow';\n dragPath.appendChild(arrow);\n\n dragIndicator.appendChild(dragPath);\n\n // Update coords in container\n const container = document.getElementById(containerId);\n if (container) {\n const coordsDiv = document.createElement('div');\n coordsDiv.className = 'gui-agent-coords';\n coordsDiv.textContent = `Drag from: (${Math.round(startX)}, ${Math.round(startY)}) to (${Math.round(endX)}, ${Math.round(endY)})`;\n\n const existingCoords = container.querySelector('.gui-agent-coords');\n if (existingCoords) {\n existingCoords.remove();\n }\n\n container.appendChild(coordsDiv);\n }\n\n // Remove indicator after animation\n setTimeout(() => {\n dragIndicator.remove();\n }, 3000);\n },\n { startX, startY, endX, endY, containerId: this.containerId },\n );\n this.logger.info('Showing drag indicator done.');\n }\n}\n"],"names":["UIHelper","page","styleId","document","style","prediction","action_type","action_inputs","thought","params","containerId","container","actionMap","actionText","x","y","existingIndicators","el","indicator","coordsDiv","Math","existingCoords","setTimeout","waterFlowId","waterFlow","highlightClass","createLegend","legend","buttonSelectors","linkSelectors","inputSelectors","otherSelectors","highlightElements","selectors","typeClass","selector","elements","Array","visibleElements","rect","window","isVisible","current","hasPointerEvents","isDisabled","buttonCount","linkCount","inputCount","otherCount","highlightedElements","error","console","startX","startY","endX","endY","dragIndicator","startPoint","endPoint","dragPath","dx","dy","length","angle","arrow","getCurrentPage","logger"],"mappings":";;;;AAGC;;;;;;;;;;AASM,MAAMA;IAqBX,MAAc,eAAe;QAC3B,MAAMC,OAAO,MAAM,IAAI,CAAC,cAAc;QACtC,MAAMA,KAAK,QAAQ,CAAC,CAACC;YACnB,IAAIC,SAAS,cAAc,CAACD,UAAU;YAEtC,MAAME,QAAQD,SAAS,aAAa,CAAC;YACrCC,MAAM,EAAE,GAAGF;YACXE,MAAM,WAAW,GAAG,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAmPrB,CAAC;YACDD,SAAS,IAAI,CAAC,WAAW,CAACC;QAC5B,GAAG,IAAI,CAAC,OAAO;IACjB;IAMA,MAAM,eAAeC,UAA4B,EAAE;QACjD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;QACjB,MAAM,IAAI,CAAC,YAAY;QAEvB,MAAM,EAAEC,WAAW,EAAEC,aAAa,EAAEC,OAAO,EAAE,GAAGH;QAChD,MAAMJ,OAAO,MAAM,IAAI,CAAC,cAAc;QAEtC,MAAMA,KAAK,QAAQ,CACjB,CAACQ;YACC,MAAM,EAAEC,WAAW,EAAEJ,WAAW,EAAEC,aAAa,EAAEC,OAAO,EAAE,GAAGC;YAE7D,IAAIE,YAAYR,SAAS,cAAc,CAACO;YACxC,IAAI,CAACC,WAAW;gBACdA,YAAYR,SAAS,aAAa,CAAC;gBACnCQ,UAAU,EAAE,GAAGD;gBACfP,SAAS,IAAI,CAAC,WAAW,CAACQ;YAC5B;YAEA,MAAMC,YAAY;gBAChB,OAAO;gBACP,YAAY;gBACZ,aAAa;gBACb,cAAc;gBACd,aAAa;gBACb,aAAa;gBACb,MAAM,CAAC,wBAAU,EAAEL,cAAc,OAAO,CAAC,CAAC,CAAC;gBAC3C,UAAU,CAAC,uBAAgB,EAAEA,cAAc,OAAO,EAAE;gBACpD,QAAQ,CAAC,yBAAW,EAAEA,cAAc,GAAG,IAAIA,cAAc,MAAM,EAAE;gBACjE,QAAQ,CAAC,iBAAU,EAAEA,cAAc,SAAS,EAAE;gBAC9C,MAAM;YACR;YAGA,MAAMM,aAAaD,SAAS,CAACN,YAAY,IAAIA;YAE7CK,UAAU,SAAS,GAAG,CAAC;;yCAEU,EAAEE,WAAW;UAC5C,EAAEL,UAAU,CAAC,+BAA+B,EAAEA,QAAQ,MAAM,CAAC,GAAG,GAAG;QACrE,CAAC;QACH,GACA;YAAE,aAAa,IAAI,CAAC,WAAW;YAAEF;YAAaC;YAAeC;QAAQ;QAEvE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;IACnB;IAOA,MAAM,mBAAmBM,CAAS,EAAEC,CAAS,EAAE;QAC7C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;QACjB,MAAM,IAAI,CAAC,YAAY;QACvB,MAAMd,OAAO,MAAM,IAAI,CAAC,cAAc;QAEtC,MAAMA,KAAK,QAAQ,CAEjB,CAAC,EAAEa,CAAC,EAAEC,CAAC,EAAEL,WAAW,EAAE;YAEpB,MAAMM,qBAAqBb,SAAS,gBAAgB,CAClD;YAEFa,mBAAmB,OAAO,CAAC,CAACC,KAAOA,GAAG,MAAM;YAG5C,MAAMC,YAAYf,SAAS,aAAa,CAAC;YACzCe,UAAU,SAAS,GAAG;YACtBA,UAAU,KAAK,CAAC,IAAI,GAAG,GAAGJ,EAAE,EAAE,CAAC;YAC/BI,UAAU,KAAK,CAAC,GAAG,GAAG,GAAGH,EAAE,EAAE,CAAC;YAC9BZ,SAAS,IAAI,CAAC,WAAW,CAACe;YAG1B,MAAMP,YAAYR,SAAS,cAAc,CAACO;YAC1C,IAAIC,WAAW;gBACb,MAAMQ,YAAYhB,SAAS,aAAa,CAAC;gBACzCgB,UAAU,SAAS,GAAG;gBACtBA,UAAU,WAAW,GAAG,CAAC,WAAW,EAAEC,KAAK,KAAK,CAACN,GAAG,EAAE,EAAEM,KAAK,KAAK,CAACL,GAAG,CAAC,CAAC;gBAExE,MAAMM,iBAAiBV,UAAU,aAAa,CAAC;gBAC/C,IAAIU,gBACFA,eAAe,MAAM;gBAGvBV,UAAU,WAAW,CAACQ;YACxB;YAGAG,WAAW;gBACTJ,UAAU,MAAM;YAClB,GAAG;QACL,GACA;YAAEJ;YAAGC;YAAG,aAAa,IAAI,CAAC,WAAW;QAAC;QAExC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;IACnB;IAEA,MAAM,gBAAgB;QACpB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;QAEjB,MAAM,IAAI,CAAC,YAAY;QACvB,MAAMd,OAAO,MAAM,IAAI,CAAC,cAAc;QAEtC,MAAMA,KAAK,QAAQ,CAAC,CAACsB;YACnB,IAAIpB,SAAS,cAAc,CAACoB,cAAc;YAE1C,MAAMC,YAAYrB,SAAS,aAAa,CAAC;YACzCqB,UAAU,EAAE,GAAGD;YACfC,UAAU,KAAK,CAAC,OAAO,GAAG,CAAC;;;;;;;;MAQ3B,CAAC;YAED,MAAMpB,QAAQD,SAAS,aAAa,CAAC;YACrCC,MAAM,WAAW,GAAG,CAAC;SAClB,EAAEmB,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAmDjB,CAAC;YAEDpB,SAAS,IAAI,CAAC,WAAW,CAACC;YAC1BD,SAAS,IAAI,CAAC,WAAW,CAACqB;QAC5B,GAAG,IAAI,CAAC,WAAW;QAEnB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;IACnB;IAEA,MAAM,gBAAgB;QACpB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;QAEjB,MAAMvB,OAAO,MAAM,IAAI,CAAC,cAAc;QAEtC,MAAMA,KAAK,QAAQ,CAAC,CAACsB;YACnB,MAAMC,YAAYrB,SAAS,cAAc,CAACoB;YAC1C,IAAIC,WACFA,UAAU,MAAM;QAEpB,GAAG,IAAI,CAAC,WAAW;QAEnB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;IACnB;IAMA,MAAM,6BAA6B;QACjC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;QACjB,MAAM,IAAI,CAAC,YAAY;QACvB,MAAMvB,OAAO,MAAM,IAAI,CAAC,cAAc;QAGtC,MAAM,IAAI,CAAC,yBAAyB;QAEpC,MAAMA,KAAK,QAAQ,CAAC,CAACwB;YAEnB,MAAMC,eAAe;gBACnB,MAAMC,SAASxB,SAAS,aAAa,CAAC;gBACtCwB,OAAO,SAAS,GAAG;gBACnBA,OAAO,EAAE,GAAG;gBACZA,OAAO,SAAS,GAAG,CAAC;;;;;;;;;;;;;;;;;;QAkBpB,CAAC;gBACDxB,SAAS,IAAI,CAAC,WAAW,CAACwB;YAC5B;YAEAD;YAGA,MAAME,kBAAkB;gBACtB;gBACA;gBACA;gBACA;gBACA;gBACA;gBACA;aACD;YAED,MAAMC,gBAAgB;gBAAC;gBAAK;gBAAiB;aAAY;YAEzD,MAAMC,iBAAiB;gBACrB;gBACA;gBACA;gBACA;gBACA;gBACA;gBACA;aACD;YAED,MAAMC,iBAAiB;gBACrB;gBACA;gBACA;gBACA;gBACA;gBACA;gBACA;gBACA;gBACA;gBACA;aACD;YAGD,MAAMC,oBAAoB,CAACC,WAAqBC;gBAC9C,MAAMC,WAAWF,UAAU,IAAI,CAAC;gBAChC,MAAMG,WAAWC,MAAM,IAAI,CAAClC,SAAS,gBAAgB,CAACgC;gBAGtD,MAAMG,kBAAkBF,SAAS,MAAM,CAAC,CAACnB;oBACvC,MAAMsB,OAAOtB,GAAG,qBAAqB;oBACrC,MAAMb,QAAQoC,OAAO,gBAAgB,CAACvB;oBACtC,MAAMwB,YACJF,KAAK,KAAK,GAAG,KACbA,KAAK,MAAM,GAAG,KACdnC,AAAkB,WAAlBA,MAAM,OAAO,IACbA,AAAqB,aAArBA,MAAM,UAAU,IAChBA,AAAkB,QAAlBA,MAAM,OAAO;oBAGf,IAAIsC,UAAUzB;oBACd,IAAI0B,mBAAmB;oBACvB,MAAOD,WAAWA,YAAYvC,SAAS,IAAI,CAAE;wBAC3C,IAAIqC,AAAmD,WAAnDA,OAAO,gBAAgB,CAACE,SAAS,aAAa,EAAa;4BAC7DC,mBAAmB;4BACnB;wBACF;wBACAD,UAAUA,QAAQ,aAAa;oBACjC;oBAGA,MAAME,aACH3B,GAAmB,YAAY,CAAC,eAChCA,AAAqD,WAArDA,GAAmB,YAAY,CAAC;oBAEnC,OAAOwB,aAAaE,oBAAoB,CAACC;gBAC3C;gBAGAN,gBAAgB,OAAO,CAAC,CAACrB;oBACvBA,GAAG,SAAS,CAAC,GAAG,CAACQ;oBACjBR,GAAG,SAAS,CAAC,GAAG,CAACiB;gBACnB;gBAEA,OAAOI,gBAAgB,MAAM;YAC/B;YAGA,MAAMO,cAAcb,kBAClBJ,iBACA;YAEF,MAAMkB,YAAYd,kBAAkBH,eAAe;YACnD,MAAMkB,aAAaf,kBACjBF,gBACA;YAEF,MAAMkB,aAAahB,kBACjBD,gBACA;YAIF,OAAO;gBACL,SAASc;gBACT,OAAOC;gBACP,QAAQC;gBACR,QAAQC;gBACR,OAAOH,cAAcC,YAAYC,aAAaC;YAChD;QACF,GAAG,IAAI,CAAC,cAAc;QACtB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;IACnB;IAKA,MAAM,4BAA4B;QAChC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;QACjB,IAAI;YACF,MAAM/C,OAAO,MAAM,IAAI,CAAC,cAAc;YACtC,MAAMA,KAAK,QAAQ,CAAC,CAACwB;gBAEnB,MAAMwB,sBAAsB9C,SAAS,gBAAgB,CACnD,CAAC,CAAC,EAAEsB,gBAAgB;gBAEtBwB,oBAAoB,OAAO,CAAC,CAAChC;oBAC3BA,GAAG,SAAS,CAAC,MAAM,CAACQ;oBACpBR,GAAG,SAAS,CAAC,MAAM,CAAC;oBACpBA,GAAG,SAAS,CAAC,MAAM,CAAC;oBACpBA,GAAG,SAAS,CAAC,MAAM,CAAC;oBACpBA,GAAG,SAAS,CAAC,MAAM,CAAC;gBACtB;gBAGA,MAAMU,SAASxB,SAAS,cAAc,CAAC;gBACvC,IAAIwB,QACFA,OAAO,MAAM;YAEjB,GAAG,IAAI,CAAC,cAAc;QACxB,EAAE,OAAOuB,OAAO;YAEdC,QAAQ,KAAK,CAAC,wCAAwCD;QACxD;QACA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;IACnB;IAEA,MAAM,0BAA0B;QAC9B,IAAI;YACF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;YACjB,MAAMjD,OAAO,MAAM,IAAI,CAAC,cAAc;YACtC,MAAMA,KAAK,QAAQ,CAAC,CAACS;gBACnB,MAAMC,YAAYR,SAAS,cAAc,CAACO;gBAC1C,IAAIC,WACFA,UAAU,MAAM;YAEpB,GAAG,IAAI,CAAC,WAAW;YACnB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;QACnB,EAAE,OAAOuC,OAAO;YAEdC,QAAQ,KAAK,CAAC,kCAAkCD;QAClD;IACF;IAKA,MAAM,UAAU;QACd,IAAI;YACF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;YACjB,MAAM,IAAI,CAAC,yBAAyB;YACpC,MAAM,IAAI,CAAC,aAAa;YAExB,MAAMjD,OAAO,MAAM,IAAI,CAAC,cAAc;YACtC,MAAMA,KAAK,QAAQ,CAAC,CAACS;gBACnB,MAAMC,YAAYR,SAAS,cAAc,CAACO;gBAC1C,IAAIC,WACFA,UAAU,MAAM;YAEpB,GAAG,IAAI,CAAC,WAAW;YACnB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;QACnB,EAAE,OAAOuC,OAAO;YAEdC,QAAQ,KAAK,CAAC,kCAAkCD;QAClD;IACF;IASA,MAAM,kBACJE,MAAc,EACdC,MAAc,EACdC,IAAY,EACZC,IAAY,EACZ;QACA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;QACjB,MAAM,IAAI,CAAC,YAAY;QACvB,MAAMtD,OAAO,MAAM,IAAI,CAAC,cAAc;QAEtC,MAAMA,KAAK,QAAQ,CAEjB,CAAC,EAAEmD,MAAM,EAAEC,MAAM,EAAEC,IAAI,EAAEC,IAAI,EAAE7C,WAAW,EAAE;YAE1C,MAAMM,qBAAqBb,SAAS,gBAAgB,CAClD;YAEFa,mBAAmB,OAAO,CAAC,CAACC,KAAOA,GAAG,MAAM;YAG5C,MAAMuC,gBAAgBrD,SAAS,aAAa,CAAC;YAC7CqD,cAAc,SAAS,GAAG;YAC1BrD,SAAS,IAAI,CAAC,WAAW,CAACqD;YAG1B,MAAMC,aAAatD,SAAS,aAAa,CAAC;YAC1CsD,WAAW,SAAS,GAAG;YACvBA,WAAW,KAAK,CAAC,IAAI,GAAG,GAAGL,OAAO,EAAE,CAAC;YACrCK,WAAW,KAAK,CAAC,GAAG,GAAG,GAAGJ,OAAO,EAAE,CAAC;YACpCG,cAAc,WAAW,CAACC;YAG1B,MAAMC,WAAWvD,SAAS,aAAa,CAAC;YACxCuD,SAAS,SAAS,GAAG;YACrBA,SAAS,KAAK,CAAC,IAAI,GAAG,GAAGJ,KAAK,EAAE,CAAC;YACjCI,SAAS,KAAK,CAAC,GAAG,GAAG,GAAGH,KAAK,EAAE,CAAC;YAChCC,cAAc,WAAW,CAACE;YAG1B,MAAMC,WAAWxD,SAAS,aAAa,CAAC;YACxCwD,SAAS,SAAS,GAAG;YAGrB,MAAMC,KAAKN,OAAOF;YAClB,MAAMS,KAAKN,OAAOF;YAClB,MAAMS,SAAS1C,KAAK,IAAI,CAACwC,KAAKA,KAAKC,KAAKA;YACxC,MAAME,QAAQ3C,AAAsB,MAAMA,KAAK,EAAC,GAAlCA,KAAK,KAAK,CAACyC,IAAID;YAE7BD,SAAS,KAAK,CAAC,KAAK,GAAG,GAAGG,OAAO,EAAE,CAAC;YACpCH,SAAS,KAAK,CAAC,IAAI,GAAG,GAAGP,OAAO,EAAE,CAAC;YACnCO,SAAS,KAAK,CAAC,GAAG,GAAG,GAAGN,OAAO,EAAE,CAAC;YAClCM,SAAS,KAAK,CAAC,SAAS,GAAG,CAAC,OAAO,EAAEI,MAAM,IAAI,CAAC;YAGhD,MAAMC,QAAQ7D,SAAS,aAAa,CAAC;YACrC6D,MAAM,SAAS,GAAG;YAClBL,SAAS,WAAW,CAACK;YAErBR,cAAc,WAAW,CAACG;YAG1B,MAAMhD,YAAYR,SAAS,cAAc,CAACO;YAC1C,IAAIC,WAAW;gBACb,MAAMQ,YAAYhB,SAAS,aAAa,CAAC;gBACzCgB,UAAU,SAAS,GAAG;gBACtBA,UAAU,WAAW,GAAG,CAAC,YAAY,EAAEC,KAAK,KAAK,CAACgC,QAAQ,EAAE,EAAEhC,KAAK,KAAK,CAACiC,QAAQ,MAAM,EAAEjC,KAAK,KAAK,CAACkC,MAAM,EAAE,EAAElC,KAAK,KAAK,CAACmC,MAAM,CAAC,CAAC;gBAEjI,MAAMlC,iBAAiBV,UAAU,aAAa,CAAC;gBAC/C,IAAIU,gBACFA,eAAe,MAAM;gBAGvBV,UAAU,WAAW,CAACQ;YACxB;YAGAG,WAAW;gBACTkC,cAAc,MAAM;YACtB,GAAG;QACL,GACA;YAAEJ;YAAQC;YAAQC;YAAMC;YAAM,aAAa,IAAI,CAAC,WAAW;QAAC;QAE9D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;IACnB;IAzwBA,YACUU,cAAmC,EACnCC,MAAc,CACtB;;;QAZF,uBAAQ,WAAR;QACA,uBAAQ,eAAR;QACA,uBAAQ,kBAAR;QACA,uBAAQ,eAAR;aAOUD,cAAc,GAAdA;aACAC,MAAM,GAANA;aAXF,OAAO,GAAG;aACV,WAAW,GAAG;aACd,cAAc,GAAG;aACjB,WAAW,GAAG;QAUpB,IAAI,CAAC,MAAM,GAAGA,OAAO,KAAK,CAAC;IAC7B;AAqwBF"}