@ui-tars/operator-nut-js
Version:
Operator Nut JS SDK for UI-TARS
247 lines (246 loc) • 16 kB
JavaScript
/**
* Copyright (c) 2025 Bytedance, Inc. and its affiliates.
* SPDX-License-Identifier: Apache-2.0
*/
import * as __WEBPACK_EXTERNAL_MODULE__ui_tars_sdk_core_23d0c2d3__ from "@ui-tars/sdk/core";
import * as __WEBPACK_EXTERNAL_MODULE_jimp__ from "jimp";
import * as __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__ from "@computer-use/nut-js";
import * as __WEBPACK_EXTERNAL_MODULE_big_js_ddb91b6b__ from "big.js";
function _define_property(obj, key, value) {
if (key in obj) Object.defineProperty(obj, key, {
value: value,
enumerable: true,
configurable: true,
writable: true
});
else obj[key] = value;
return obj;
}
const moveStraightTo = async (startX, startY)=>{
if (null === startX || null === startY) return;
await __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.mouse.move((0, __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.straightTo)(new __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Point(startX, startY)));
};
class NutJSOperator extends __WEBPACK_EXTERNAL_MODULE__ui_tars_sdk_core_23d0c2d3__.Operator {
async screenshot() {
const { logger } = (0, __WEBPACK_EXTERNAL_MODULE__ui_tars_sdk_core_23d0c2d3__.useContext)();
const grabImage = await __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.screen.grab();
const screenWithScale = await grabImage.toRGB();
const scaleFactor = screenWithScale.pixelDensity.scaleX;
logger.info('[NutjsOperator]', 'scaleX', screenWithScale.pixelDensity.scaleX, 'scaleY', screenWithScale.pixelDensity.scaleY);
const screenWithScaleImage = await __WEBPACK_EXTERNAL_MODULE_jimp__.Jimp.fromBitmap({
width: screenWithScale.width,
height: screenWithScale.height,
data: Buffer.from(screenWithScale.data)
});
const width = screenWithScale.width / screenWithScale.pixelDensity.scaleX;
const height = screenWithScale.height / screenWithScale.pixelDensity.scaleY;
const physicalScreenImage = await screenWithScaleImage.resize({
w: width,
h: height
}).getBuffer('image/png');
const output = {
base64: physicalScreenImage.toString('base64'),
scaleFactor
};
null == logger || logger.info(`[NutjsOperator] screenshot: ${width}x${height}, scaleFactor: ${scaleFactor}`);
return output;
}
async execute(params) {
const { logger } = (0, __WEBPACK_EXTERNAL_MODULE__ui_tars_sdk_core_23d0c2d3__.useContext)();
const { parsedPrediction, screenWidth, screenHeight, scaleFactor } = params;
const { action_type, action_inputs } = parsedPrediction;
const startBoxStr = (null == action_inputs ? void 0 : action_inputs.start_box) || '';
logger.info('[NutjsOperator] execute', scaleFactor);
const { x: startX, y: startY } = (0, __WEBPACK_EXTERNAL_MODULE__ui_tars_sdk_core_23d0c2d3__.parseBoxToScreenCoords)({
boxStr: startBoxStr,
screenWidth,
screenHeight
});
logger.info(`[NutjsOperator Position]: (${startX}, ${startY})`);
__WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.mouse.config.mouseSpeed = 3600;
const getHotkeys = (keyStr)=>{
if (keyStr) {
const platformCommandKey = 'darwin' === process.platform ? __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Key.LeftCmd : __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Key.LeftWin;
const platformCtrlKey = 'darwin' === process.platform ? __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Key.LeftCmd : __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Key.LeftControl;
const keyMap = {
return: __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Key.Enter,
enter: __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Key.Enter,
backspace: __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Key.Backspace,
delete: __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Key.Delete,
ctrl: platformCtrlKey,
shift: __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Key.LeftShift,
alt: __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Key.LeftAlt,
space: __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Key.Space,
'page down': __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Key.PageDown,
pagedown: __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Key.PageDown,
'page up': __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Key.PageUp,
pageup: __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Key.PageUp,
meta: platformCommandKey,
win: platformCommandKey,
command: platformCommandKey,
cmd: platformCommandKey,
comma: __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Key.Comma,
',': __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Key.Comma,
up: __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Key.Up,
down: __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Key.Down,
left: __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Key.Left,
right: __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Key.Right,
arrowup: __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Key.Up,
arrowdown: __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Key.Down,
arrowleft: __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Key.Left,
arrowright: __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Key.Right
};
const keys = keyStr.split(/[\s+]/).map((k)=>keyMap[k.toLowerCase()] || __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Key[k.toUpperCase()]);
logger.info('[NutjsOperator] hotkey: ', keys);
return keys;
}
logger.error('[NutjsOperator] hotkey error: ', `${keyStr} is not a valid key`);
return [];
};
switch(action_type){
case 'wait':
logger.info('[NutjsOperator] wait', action_inputs);
await (0, __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.sleep)(5000);
break;
case 'mouse_move':
case 'hover':
logger.info('[NutjsOperator] mouse_move');
await moveStraightTo(startX, startY);
break;
case 'click':
case 'left_click':
case 'left_single':
logger.info('[NutjsOperator] left_click');
await moveStraightTo(startX, startY);
await (0, __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.sleep)(100);
await __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.mouse.click(__WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Button.LEFT);
break;
case 'left_double':
case 'double_click':
logger.info(`[NutjsOperator] ${action_type}(${startX}, ${startY})`);
await moveStraightTo(startX, startY);
await (0, __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.sleep)(100);
await __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.mouse.doubleClick(__WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Button.LEFT);
break;
case 'right_click':
case 'right_single':
logger.info('[NutjsOperator] right_click');
await moveStraightTo(startX, startY);
await (0, __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.sleep)(100);
await __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.mouse.click(__WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Button.RIGHT);
break;
case 'middle_click':
logger.info('[NutjsOperator] middle_click');
await moveStraightTo(startX, startY);
await __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.mouse.click(__WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Button.MIDDLE);
break;
case 'left_click_drag':
case 'drag':
case 'select':
logger.info('[NutjsOperator] drag', action_inputs);
if (null == action_inputs ? void 0 : action_inputs.end_box) {
const { x: endX, y: endY } = (0, __WEBPACK_EXTERNAL_MODULE__ui_tars_sdk_core_23d0c2d3__.parseBoxToScreenCoords)({
boxStr: action_inputs.end_box,
screenWidth,
screenHeight
});
if (startX && startY && endX && endY) {
const diffX = (0, __WEBPACK_EXTERNAL_MODULE_big_js_ddb91b6b__["default"])(endX).minus(startX).toNumber();
const diffY = (0, __WEBPACK_EXTERNAL_MODULE_big_js_ddb91b6b__["default"])(endY).minus(startY).toNumber();
await __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.mouse.drag((0, __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.straightTo)((0, __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.centerOf)(new __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Region(startX, startY, diffX, diffY))));
}
}
break;
case 'type':
{
var _action_inputs_content;
const content = null === (_action_inputs_content = action_inputs.content) || void 0 === _action_inputs_content ? void 0 : _action_inputs_content.trim();
logger.info('[NutjsOperator] type', content);
if (content) {
const stripContent = content.replace(/\\n$/, '').replace(/\n$/, '');
__WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.keyboard.config.autoDelayMs = 0;
if ('win32' === process.platform) {
const originalClipboard = await __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.clipboard.getContent();
await __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.clipboard.setContent(stripContent);
await __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.keyboard.pressKey(__WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Key.LeftControl, __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Key.V);
await (0, __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.sleep)(50);
await __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.keyboard.releaseKey(__WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Key.LeftControl, __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Key.V);
await (0, __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.sleep)(50);
await __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.clipboard.setContent(originalClipboard);
} else await __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.keyboard.type(stripContent);
if (content.endsWith('\n') || content.endsWith('\\n')) {
await __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.keyboard.pressKey(__WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Key.Enter);
await __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.keyboard.releaseKey(__WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.Key.Enter);
}
__WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.keyboard.config.autoDelayMs = 500;
}
break;
}
case 'hotkey':
{
const keyStr = (null == action_inputs ? void 0 : action_inputs.key) || (null == action_inputs ? void 0 : action_inputs.hotkey);
const keys = getHotkeys(keyStr);
if (keys.length > 0) {
await __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.keyboard.pressKey(...keys);
await __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.keyboard.releaseKey(...keys);
}
break;
}
case 'press':
{
const keyStr = (null == action_inputs ? void 0 : action_inputs.key) || (null == action_inputs ? void 0 : action_inputs.hotkey);
const keys = getHotkeys(keyStr);
if (keys.length > 0) await __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.keyboard.pressKey(...keys);
break;
}
case 'release':
{
const keyStr = (null == action_inputs ? void 0 : action_inputs.key) || (null == action_inputs ? void 0 : action_inputs.hotkey);
const keys = getHotkeys(keyStr);
if (keys.length > 0) await __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.keyboard.releaseKey(...keys);
break;
}
case 'scroll':
{
const { direction } = action_inputs;
if (null !== startX && null !== startY) await moveStraightTo(startX, startY);
switch(null == direction ? void 0 : direction.toLowerCase()){
case 'up':
await __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.mouse.scrollUp(500);
break;
case 'down':
await __WEBPACK_EXTERNAL_MODULE__computer_use_nut_js_52062ecb__.mouse.scrollDown(500);
break;
default:
console.warn(`[NutjsOperator] Unsupported scroll direction: ${direction}`);
}
break;
}
case 'error_env':
case 'call_user':
case 'finished':
case 'user_stop':
return {
status: __WEBPACK_EXTERNAL_MODULE__ui_tars_sdk_core_23d0c2d3__.StatusEnum.END
};
default:
logger.warn(`Unsupported action: ${action_type}`);
}
}
}
_define_property(NutJSOperator, "MANUAL", {
ACTION_SPACES: [
"click(start_box='[x1, y1, x2, y2]')",
"left_double(start_box='[x1, y1, x2, y2]')",
"right_single(start_box='[x1, y1, x2, y2]')",
"drag(start_box='[x1, y1, x2, y2]', end_box='[x3, y3, x4, y4]')",
"hotkey(key='')",
"type(content='') #If you want to submit your input, use \"\\n\" at the end of `content`.",
"scroll(start_box='[x1, y1, x2, y2]', direction='down or up or right or left')",
"wait() #Sleep for 5s and take a screenshot to check for any changes.",
"finished()",
"call_user() # Submit the task and call the user when the task is unsolvable, or when you need the user's help."
]
});
export { NutJSOperator };
//# sourceMappingURL=index.mjs.map