@ui-tars/operator-browser
Version:
Native-browser operator for UI-TARS
687 lines (655 loc) • 26.6 kB
JavaScript
/**
* Copyright (c) 2025 Bytedance, Inc. and its affiliates.
* SPDX-License-Identifier: Apache-2.0
*/
;
var __webpack_require__ = {};
(()=>{
__webpack_require__.d = (exports1, definition)=>{
for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
enumerable: true,
get: definition[key]
});
};
})();
(()=>{
__webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
})();
(()=>{
__webpack_require__.r = (exports1)=>{
if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
value: 'Module'
});
Object.defineProperty(exports1, '__esModule', {
value: true
});
};
})();
var __webpack_exports__ = {};
__webpack_require__.r(__webpack_exports__);
__webpack_require__.d(__webpack_exports__, {
UIHelper: ()=>UIHelper
});
function _define_property(obj, key, value) {
if (key in obj) Object.defineProperty(obj, key, {
value: value,
enumerable: true,
configurable: true,
writable: true
});
else obj[key] = value;
return obj;
}
class UIHelper {
async injectStyles() {
const page = await this.getCurrentPage();
await page.evaluate((styleId)=>{
if (document.getElementById(styleId)) return;
const style = document.createElement('style');
style.id = styleId;
style.textContent = `
#gui-agent-helper-container {
position: fixed;
top: 20px;
right: 20px;
background: rgba(0, 0, 0, 0.85);
color: white;
padding: 15px 20px;
border-radius: 12px;
font-family: system-ui;
z-index: 999999;
max-width: 320px;
backdrop-filter: blur(8px);
box-shadow: 0 8px 32px rgba(0, 0, 0, 0.2);
border: 1px solid rgba(255, 255, 255, 0.1);
}
.gui-agent-title {
font-size: 14px;
font-weight: 600;
margin-bottom: 10px;
color: #00ff9d;
text-transform: uppercase;
letter-spacing: 0.5px;
}
.gui-agent-content {
font-size: 13px;
line-height: 1.5;
color: rgba(255, 255, 255, 0.9);
}
.gui-agent-coords {
margin-top: 8px;
font-size: 12px;
color: #00ff9d;
opacity: 0.8;
}
.gui-agent-thought {
margin-top: 10px;
padding-top: 10px;
border-top: 1px solid rgba(255, 255, 255, 0.15);
font-style: italic;
color: rgba(255, 255, 255, 0.7);
font-size: 12px;
}
.gui-agent-click-indicator {
position: fixed;
pointer-events: none;
width: 60px;
height: 60px;
border-radius: 50%;
border: 4px solid #00ff9d;
background: rgba(0, 255, 157, 0.3);
transform: translate(-50%, -50%);
animation: click-pulse 1.2s ease-out;
z-index: 2147483647;
}
.gui-agent-click-indicator::before {
content: '';
position: absolute;
top: 50%;
left: 50%;
width: 12px;
height: 12px;
background: #00ff9d;
border-radius: 50%;
transform: translate(-50%, -50%);
}
.gui-agent-drag-indicator {
position: fixed;
pointer-events: none;
z-index: 2147483647;
}
.gui-agent-drag-start {
width: 40px;
height: 40px;
border-radius: 50%;
border: 3px solid #ff6b00;
background: rgba(255, 107, 0, 0.4);
transform: translate(-50%, -50%);
position: absolute;
}
.gui-agent-drag-end {
width: 40px;
height: 40px;
border-radius: 50%;
border: 3px solid #00c3ff;
background: rgba(0, 195, 255, 0.4);
transform: translate(-50%, -50%);
position: absolute;
}
.gui-agent-drag-path {
position: absolute;
height: 6px;
background: linear-gradient(to right, #ff6b00, #00c3ff);
border-radius: 3px;
transform-origin: left center;
opacity: 0.7;
}
.gui-agent-drag-arrow {
position: absolute;
width: 0;
height: a0;
border-top: 12px solid transparent;
border-bottom: 12px solid transparent;
border-left: 16px solid #00c3ff;
transform-origin: left center;
right: -16px;
top: -9px;
}
.gui-agent-clickable-highlight {
outline: 3px solid rgba(0, 155, 255, 0.7) !important;
box-shadow: 0 0 0 3px rgba(0, 155, 255, 0.3) !important;
background-color: rgba(0, 155, 255, 0.05) !important;
transition: all 0.2s ease-in-out !important;
z-index: 999 !important;
position: relative !important;
}
.gui-agent-clickable-highlight:hover {
outline: 4px solid rgba(0, 155, 255, 0.9) !important;
background-color: rgba(0, 155, 255, 0.1) !important;
}
.gui-agent-clickable-highlight.gui-highlight-button {
outline: 3px solid rgba(255, 64, 129, 0.8) !important;
box-shadow: 0 0 0 3px rgba(255, 64, 129, 0.3) !important;
background-color: rgba(255, 64, 129, 0.05) !important;
}
.gui-agent-clickable-highlight.gui-highlight-button:hover {
outline: 4px solid rgba(255, 64, 129, 0.9) !important;
background-color: rgba(255, 64, 129, 0.1) !important;
}
.gui-agent-clickable-highlight.gui-highlight-link {
outline: 3px solid rgba(124, 77, 255, 0.8) !important;
box-shadow: 0 0 0 3px rgba(124, 77, 255, 0.3) !important;
background-color: rgba(124, 77, 255, 0.05) !important;
}
.gui-agent-clickable-highlight.gui-highlight-link:hover {
outline: 4px solid rgba(124, 77, 255, 0.9) !important;
background-color: rgba(124, 77, 255, 0.1) !important;
}
.gui-agent-clickable-highlight.gui-highlight-input {
outline: 3px solid rgba(0, 230, 118, 0.8) !important;
box-shadow: 0 0 0 3px rgba(0, 230, 118, 0.3) !important;
background-color: rgba(0, 230, 118, 0.05) !important;
}
.gui-agent-clickable-highlight.gui-highlight-input:hover {
outline: 4px solid rgba(0, 230, 118, 0.9) !important;
background-color: rgba(0, 230, 118, 0.1) !important;
}
.gui-agent-clickable-highlight.gui-highlight-other {
outline: 3px solid rgba(255, 171, 0, 0.8) !important;
box-shadow: 0 0 0 3px rgba(255, 171, 0, 0.3) !important;
background-color: rgba(255, 171, 0, 0.05) !important;
}
.gui-agent-clickable-highlight.gui-highlight-other:hover {
outline: 4px solid rgba(255, 171, 0, 0.9) !important;
background-color: rgba(255, 171, 0, 0.1) !important;
}
.gui-agent-legend {
position: fixed;
bottom: 20px;
left: 20px;
background: rgba(0, 0, 0, 0.85);
color: white;
padding: 12px 18px;
border-radius: 10px;
font-family: system-ui;
font-size: 12px;
z-index: 999999;
backdrop-filter: blur(8px);
box-shadow: 0 4px 16px rgba(0, 0, 0, 0.2);
border: 1px solid rgba(255, 255, 255, 0.1);
display: flex;
flex-direction: column;
gap: 8px;
}
.gui-agent-legend-title {
font-weight: 600;
margin-bottom: 4px;
font-size: 13px;
}
.gui-agent-legend-item {
display: flex;
align-items: center;
gap: 8px;
}
.gui-agent-legend-icon {
display: inline-block;
width: 14px;
height: 14px;
border-radius: 3px;
border: 2px solid rgba(255, 255, 255, 0.8);
}
.gui-legend-button {
background: rgba(255, 64, 129, 0.7);
}
.gui-legend-link {
background: rgba(124, 77, 255, 0.7);
}
.gui-legend-input {
background: rgba(0, 230, 118, 0.7);
}
.gui-legend-other {
background: rgba(255, 171, 0, 0.7);
}
@keyframes click-pulse {
0% {
transform: translate(-50%, -50%) scale(0.8);
opacity: 1;
}
100% {
transform: translate(-50%, -50%) scale(2.5);
opacity: 0;
}
}
`;
document.head.appendChild(style);
}, this.styleId);
}
async showActionInfo(prediction) {
this.logger.info('Showing action info ...');
await this.injectStyles();
const { action_type, action_inputs, thought } = prediction;
const page = await this.getCurrentPage();
await page.evaluate((params)=>{
const { containerId, action_type, action_inputs, thought } = params;
let container = document.getElementById(containerId);
if (!container) {
container = document.createElement('div');
container.id = containerId;
document.body.appendChild(container);
}
const actionMap = {
click: "\uD83D\uDDB1\uFE0F Single Click",
left_click: "\uD83D\uDDB1\uFE0F Single Click",
left_single: "\uD83D\uDDB1\uFE0F Single Click",
double_click: "\uD83D\uDDB1\uFE0F Double Click",
left_double: "\uD83D\uDDB1\uFE0F Double Click",
right_click: "\uD83D\uDDB1\uFE0F Right Click",
type: `\u{2328}\u{FE0F} Type: "${action_inputs.content}"`,
navigate: `\u{1F310} Navigate to: ${action_inputs.content}`,
hotkey: `\u{2328}\u{FE0F} Hotkey: ${action_inputs.key || action_inputs.hotkey}`,
scroll: `\u{1F4DC} Scroll ${action_inputs.direction}`,
wait: "\u23F3 Wait"
};
const actionText = actionMap[action_type] || action_type;
container.innerHTML = `
<div class="gui-agent-title">Next Action</div>
<div class="gui-agent-content">${actionText}</div>
${thought ? `<div class="gui-agent-thought">${thought}</div>` : ''}
`;
}, {
containerId: this.containerId,
action_type,
action_inputs,
thought
});
this.logger.info('Showing action info done.');
}
async showClickIndicator(x, y) {
this.logger.info('Showing click indicator...');
await this.injectStyles();
const page = await this.getCurrentPage();
await page.evaluate(({ x, y, containerId })=>{
const existingIndicators = document.querySelectorAll('.gui-agent-click-indicator');
existingIndicators.forEach((el)=>el.remove());
const indicator = document.createElement('div');
indicator.className = 'gui-agent-click-indicator';
indicator.style.left = `${x}px`;
indicator.style.top = `${y}px`;
document.body.appendChild(indicator);
const container = document.getElementById(containerId);
if (container) {
const coordsDiv = document.createElement('div');
coordsDiv.className = 'gui-agent-coords';
coordsDiv.textContent = `Click at: (${Math.round(x)}, ${Math.round(y)})`;
const existingCoords = container.querySelector('.gui-agent-coords');
if (existingCoords) existingCoords.remove();
container.appendChild(coordsDiv);
}
setTimeout(()=>{
indicator.remove();
}, 1200);
}, {
x,
y,
containerId: this.containerId
});
this.logger.info('Showing click indicator done.');
}
async showWaterFlow() {
this.logger.info('Showing water flow effect...');
await this.injectStyles();
const page = await this.getCurrentPage();
await page.evaluate((waterFlowId)=>{
if (document.getElementById(waterFlowId)) return;
const waterFlow = document.createElement('div');
waterFlow.id = waterFlowId;
waterFlow.style.cssText = `
position: fixed;
top: 0;
left: 0;
right: 0;
bottom: 0;
pointer-events: none;
z-index: 2147483647;
`;
const style = document.createElement('style');
style.textContent = `
#${waterFlowId}::before {
content: "";
position: fixed;
top: 0; right: 0; bottom: 0; left: 0;
pointer-events: none;
z-index: 9999;
background:
linear-gradient(to right, rgba(30, 144, 255, 0.4), transparent 50%) left,
linear-gradient(to left, rgba(30, 144, 255, 0.4), transparent 50%) right,
linear-gradient(to bottom, rgba(30, 144, 255, 0.4), transparent 50%) top,
linear-gradient(to top, rgba(30, 144, 255, 0.4), transparent 50%) bottom;
background-repeat: no-repeat;
background-size: 10% 100%, 10% 100%, 100% 10%, 100% 10%;
animation: waterflow 5s cubic-bezier(0.4, 0, 0.6, 1) infinite;
filter: blur(8px);
}
@keyframes waterflow {
0%, 100% {
background-image:
linear-gradient(to right, rgba(30, 144, 255, 0.4), transparent 50%),
linear-gradient(to left, rgba(30, 144, 255, 0.4), transparent 50%),
linear-gradient(to bottom, rgba(30, 144, 255, 0.4), transparent 50%),
linear-gradient(to top, rgba(30, 144, 255, 0.4), transparent 50%);
transform: scale(1);
}
25% {
background-image:
linear-gradient(to right, rgba(30, 144, 255, 0.39), transparent 52%),
linear-gradient(to left, rgba(30, 144, 255, 0.39), transparent 52%),
linear-gradient(to bottom, rgba(30, 144, 255, 0.39), transparent 52%),
linear-gradient(to top, rgba(30, 144, 255, 0.39), transparent 52%);
transform: scale(1.03);
}
50% {
background-image:
linear-gradient(to right, rgba(30, 144, 255, 0.38), transparent 55%),
linear-gradient(to left, rgba(30, 144, 255, 0.38), transparent 55%),
linear-gradient(to bottom, rgba(30, 144, 255, 0.38), transparent 55%),
linear-gradient(to top, rgba(30, 144, 255, 0.38), transparent 55%);
transform: scale(1.05);
}
75% {
background-image:
linear-gradient(to right, rgba(30, 144, 255, 0.39), transparent 52%),
linear-gradient(to left, rgba(30, 144, 255, 0.39), transparent 52%),
linear-gradient(to bottom, rgba(30, 144, 255, 0.39), transparent 52%),
linear-gradient(to top, rgba(30, 144, 255, 0.39), transparent 52%);
transform: scale(1.03);
}
}
`;
document.head.appendChild(style);
document.body.appendChild(waterFlow);
}, this.waterFlowId);
this.logger.info('Water flow effect shown.');
}
async hideWaterFlow() {
this.logger.info('Hiding water flow effect...');
const page = await this.getCurrentPage();
await page.evaluate((waterFlowId)=>{
const waterFlow = document.getElementById(waterFlowId);
if (waterFlow) waterFlow.remove();
}, this.waterFlowId);
this.logger.info('Water flow effect hidden.');
}
async highlightClickableElements() {
this.logger.info('Highlighting clickable elements...');
await this.injectStyles();
const page = await this.getCurrentPage();
await this.removeClickableHighlights();
await page.evaluate((highlightClass)=>{
const createLegend = ()=>{
const legend = document.createElement('div');
legend.className = 'gui-agent-legend';
legend.id = 'gui-agent-clickable-legend';
legend.innerHTML = `
<div class="gui-agent-legend-title">Clickable Elements</div>
<div class="gui-agent-legend-item">
<span class="gui-agent-legend-icon gui-legend-button"></span>
<span>Buttons</span>
</div>
<div class="gui-agent-legend-item">
<span class="gui-agent-legend-icon gui-legend-link"></span>
<span>Links</span>
</div>
<div class="gui-agent-legend-item">
<span class="gui-agent-legend-icon gui-legend-input"></span>
<span>Input Fields</span>
</div>
<div class="gui-agent-legend-item">
<span class="gui-agent-legend-icon gui-legend-other"></span>
<span>Other Clickables</span>
</div>
`;
document.body.appendChild(legend);
};
createLegend();
const buttonSelectors = [
'button',
'[role="button"]',
'.btn',
'.button',
'[type="button"]',
'[type="submit"]',
'[type="reset"]'
];
const linkSelectors = [
'a',
'[role="link"]',
'.nav-item'
];
const inputSelectors = [
'input',
'select',
'textarea',
'[role="checkbox"]',
'[role="radio"]',
'[role="textbox"]',
'[contenteditable="true"]'
];
const otherSelectors = [
'[role="tab"]',
'[role="menuitem"]',
'[role="option"]',
'[onclick]',
'[tabindex="0"]',
'.clickable',
'.selectable',
'summary',
'details',
'label'
];
const highlightElements = (selectors, typeClass)=>{
const selector = selectors.join(', ');
const elements = Array.from(document.querySelectorAll(selector));
const visibleElements = elements.filter((el)=>{
const rect = el.getBoundingClientRect();
const style = window.getComputedStyle(el);
const isVisible = rect.width > 0 && rect.height > 0 && 'none' !== style.display && 'hidden' !== style.visibility && '0' !== style.opacity;
let current = el;
let hasPointerEvents = true;
while(current && current !== document.body){
if ('none' === window.getComputedStyle(current).pointerEvents) {
hasPointerEvents = false;
break;
}
current = current.parentElement;
}
const isDisabled = el.hasAttribute('disabled') || 'true' === el.getAttribute('aria-disabled');
return isVisible && hasPointerEvents && !isDisabled;
});
visibleElements.forEach((el)=>{
el.classList.add(highlightClass);
el.classList.add(typeClass);
});
return visibleElements.length;
};
const buttonCount = highlightElements(buttonSelectors, 'gui-highlight-button');
const linkCount = highlightElements(linkSelectors, 'gui-highlight-link');
const inputCount = highlightElements(inputSelectors, 'gui-highlight-input');
const otherCount = highlightElements(otherSelectors, 'gui-highlight-other');
return {
buttons: buttonCount,
links: linkCount,
inputs: inputCount,
others: otherCount,
total: buttonCount + linkCount + inputCount + otherCount
};
}, this.highlightClass);
this.logger.info('Highlighting clickable elements done.');
}
async removeClickableHighlights() {
this.logger.info('Removing clickable highlights...');
try {
const page = await this.getCurrentPage();
await page.evaluate((highlightClass)=>{
const highlightedElements = document.querySelectorAll(`.${highlightClass}`);
highlightedElements.forEach((el)=>{
el.classList.remove(highlightClass);
el.classList.remove('gui-highlight-button');
el.classList.remove('gui-highlight-link');
el.classList.remove('gui-highlight-input');
el.classList.remove('gui-highlight-other');
});
const legend = document.getElementById('gui-agent-clickable-legend');
if (legend) legend.remove();
}, this.highlightClass);
} catch (error) {
console.error('Error removing clickable highlights:', error);
}
this.logger.info('Removing clickable highlights done.');
}
async cleanupTemporaryVisuals() {
try {
this.logger.info('cleanupTemporaryVisuals up...');
const page = await this.getCurrentPage();
await page.evaluate((containerId)=>{
const container = document.getElementById(containerId);
if (container) container.remove();
}, this.containerId);
this.logger.info('cleanupTemporaryVisuals up done!');
} catch (error) {
console.error('Error during UIHelper cleanup:', error);
}
}
async cleanup() {
try {
this.logger.info('Cleaning up...');
await this.removeClickableHighlights();
await this.hideWaterFlow();
const page = await this.getCurrentPage();
await page.evaluate((containerId)=>{
const container = document.getElementById(containerId);
if (container) container.remove();
}, this.containerId);
this.logger.info('Cleaning up done!');
} catch (error) {
console.error('Error during UIHelper cleanup:', error);
}
}
async showDragIndicator(startX, startY, endX, endY) {
this.logger.info('Showing drag indicator...');
await this.injectStyles();
const page = await this.getCurrentPage();
await page.evaluate(({ startX, startY, endX, endY, containerId })=>{
const existingIndicators = document.querySelectorAll('.gui-agent-drag-indicator');
existingIndicators.forEach((el)=>el.remove());
const dragIndicator = document.createElement('div');
dragIndicator.className = 'gui-agent-drag-indicator';
document.body.appendChild(dragIndicator);
const startPoint = document.createElement('div');
startPoint.className = 'gui-agent-drag-start';
startPoint.style.left = `${startX}px`;
startPoint.style.top = `${startY}px`;
dragIndicator.appendChild(startPoint);
const endPoint = document.createElement('div');
endPoint.className = 'gui-agent-drag-end';
endPoint.style.left = `${endX}px`;
endPoint.style.top = `${endY}px`;
dragIndicator.appendChild(endPoint);
const dragPath = document.createElement('div');
dragPath.className = 'gui-agent-drag-path';
const dx = endX - startX;
const dy = endY - startY;
const length = Math.sqrt(dx * dx + dy * dy);
const angle = 180 / Math.PI * Math.atan2(dy, dx);
dragPath.style.width = `${length}px`;
dragPath.style.left = `${startX}px`;
dragPath.style.top = `${startY}px`;
dragPath.style.transform = `rotate(${angle}deg)`;
const arrow = document.createElement('div');
arrow.className = 'gui-agent-drag-arrow';
dragPath.appendChild(arrow);
dragIndicator.appendChild(dragPath);
const container = document.getElementById(containerId);
if (container) {
const coordsDiv = document.createElement('div');
coordsDiv.className = 'gui-agent-coords';
coordsDiv.textContent = `Drag from: (${Math.round(startX)}, ${Math.round(startY)}) to (${Math.round(endX)}, ${Math.round(endY)})`;
const existingCoords = container.querySelector('.gui-agent-coords');
if (existingCoords) existingCoords.remove();
container.appendChild(coordsDiv);
}
setTimeout(()=>{
dragIndicator.remove();
}, 3000);
}, {
startX,
startY,
endX,
endY,
containerId: this.containerId
});
this.logger.info('Showing drag indicator done.');
}
constructor(getCurrentPage, logger){
_define_property(this, "getCurrentPage", void 0);
_define_property(this, "logger", void 0);
_define_property(this, "styleId", void 0);
_define_property(this, "containerId", void 0);
_define_property(this, "highlightClass", void 0);
_define_property(this, "waterFlowId", void 0);
this.getCurrentPage = getCurrentPage;
this.logger = logger;
this.styleId = 'gui-agent-helper-styles';
this.containerId = 'gui-agent-helper-container';
this.highlightClass = 'gui-agent-clickable-highlight';
this.waterFlowId = 'gui-agent-water-flow';
this.logger = logger.spawn('[UIHelper]');
}
}
exports.UIHelper = __webpack_exports__.UIHelper;
for(var __webpack_i__ in __webpack_exports__)if (-1 === [
"UIHelper"
].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
Object.defineProperty(exports, '__esModule', {
value: true
});
//# sourceMappingURL=ui-helper.js.map