@ui-tars/sdk
Version:
A powerful cross-platform(ANY device/platform) toolkit for building GUI automation agents for UI-TARS
322 lines (321 loc) • 15.7 kB
JavaScript
/**
* Copyright (c) 2025 Bytedance, Inc. and its affiliates.
* SPDX-License-Identifier: Apache-2.0
*/
;
var __webpack_require__ = {};
(()=>{
__webpack_require__.n = (module)=>{
var getter = module && module.__esModule ? ()=>module['default'] : ()=>module;
__webpack_require__.d(getter, {
a: getter
});
return getter;
};
})();
(()=>{
__webpack_require__.d = (exports1, definition)=>{
for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
enumerable: true,
get: definition[key]
});
};
})();
(()=>{
__webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
})();
(()=>{
__webpack_require__.r = function(exports1) {
if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
value: 'Module'
});
Object.defineProperty(exports1, '__esModule', {
value: true
});
};
})();
var __webpack_exports__ = {};
__webpack_require__.r(__webpack_exports__);
__webpack_require__.d(__webpack_exports__, {
GUIAgent: ()=>GUIAgent
});
const types_namespaceObject = require("@ui-tars/shared/types");
const constants_namespaceObject = require("@ui-tars/shared/constants");
const utils_namespaceObject = require("@ui-tars/shared/utils");
const external_async_retry_namespaceObject = require("async-retry");
var external_async_retry_default = /*#__PURE__*/ __webpack_require__.n(external_async_retry_namespaceObject);
const external_jimp_namespaceObject = require("jimp");
const useContext_js_namespaceObject = require("./context/useContext.js");
const external_Model_js_namespaceObject = require("./Model.js");
const index_js_namespaceObject = require("./base/index.js");
const external_utils_js_namespaceObject = require("./utils.js");
const external_constants_js_namespaceObject = require("./constants.js");
function _define_property(obj, key, value) {
if (key in obj) Object.defineProperty(obj, key, {
value: value,
enumerable: true,
configurable: true,
writable: true
});
else obj[key] = value;
return obj;
}
class GUIAgent extends index_js_namespaceObject.BaseGUIAgent {
async run(instruction) {
const { operator, model, logger } = this;
const { signal, onData, onError, retry = {}, maxLoopCount = constants_namespaceObject.MAX_LOOP_COUNT } = this.config;
const currentTime = Date.now();
const data = {
version: types_namespaceObject.ShareVersion.V1,
systemPrompt: this.systemPrompt,
instruction,
modelName: this.model.modelName,
status: types_namespaceObject.StatusEnum.INIT,
logTime: currentTime,
conversations: [
{
from: 'human',
value: instruction,
timing: {
start: currentTime,
end: currentTime,
cost: 0
}
}
]
};
(0, useContext_js_namespaceObject.setContext)(Object.assign(this.config, {
logger: this.logger,
systemPrompt: this.systemPrompt,
factors: this.model.factors,
model: this.model
}));
let loopCnt = 0;
let snapshotErrCnt = 0;
data.status = types_namespaceObject.StatusEnum.RUNNING;
await (null == onData ? void 0 : onData({
data: {
...data,
conversations: []
}
}));
try {
while(true){
var _retry_screenshot, _retry_screenshot1, _retry_model, _retry_model1;
console.log('[run_data_status]', data.status);
if (data.status !== types_namespaceObject.StatusEnum.RUNNING || (null == signal ? void 0 : signal.aborted)) {
(null == signal ? void 0 : signal.aborted) && (data.status = types_namespaceObject.StatusEnum.END);
await (null == onData ? void 0 : onData({
data: {
...data,
conversations: []
}
}));
break;
}
if (loopCnt >= maxLoopCount || snapshotErrCnt >= external_constants_js_namespaceObject.MAX_SNAPSHOT_ERR_CNT) {
Object.assign(data, {
status: types_namespaceObject.StatusEnum.MAX_LOOP,
errMsg: loopCnt >= maxLoopCount ? 'Exceeds the maximum number of loops' : 'Too many screenshot failures'
});
await (null == onData ? void 0 : onData({
data: {
...data,
conversations: []
}
}));
break;
}
loopCnt += 1;
const start = Date.now();
const snapshot = await external_async_retry_default()(()=>operator.screenshot(), {
retries: (null == retry ? void 0 : null === (_retry_screenshot = retry.screenshot) || void 0 === _retry_screenshot ? void 0 : _retry_screenshot.maxRetries) ?? 0,
onRetry: null == retry ? void 0 : null === (_retry_screenshot1 = retry.screenshot) || void 0 === _retry_screenshot1 ? void 0 : _retry_screenshot1.onRetry
});
const { width, height, mime } = await external_jimp_namespaceObject.Jimp.fromBuffer(Buffer.from((0, external_utils_js_namespaceObject.replaceBase64Prefix)(snapshot.base64), 'base64')).catch((e)=>{
logger.error('[GUIAgent] screenshot error', e);
return {
width: null,
height: null,
mime: ''
};
});
const isValidImage = !!((null == snapshot ? void 0 : snapshot.base64) && width && height);
if (!isValidImage) {
loopCnt -= 1;
snapshotErrCnt += 1;
await (0, utils_namespaceObject.sleep)(1000);
continue;
}
let end = Date.now();
if (isValidImage) {
data.conversations.push({
from: 'human',
value: constants_namespaceObject.IMAGE_PLACEHOLDER,
screenshotBase64: snapshot.base64,
screenshotContext: {
size: {
width,
height
},
mime,
scaleFactor: snapshot.scaleFactor
},
timing: {
start,
end,
cost: end - start
}
});
await (null == onData ? void 0 : onData({
data: {
...data,
conversations: data.conversations.slice(-1)
}
}));
}
const modelFormat = (0, external_utils_js_namespaceObject.toVlmModelFormat)({
conversations: data.conversations,
systemPrompt: data.systemPrompt
});
const vlmParams = {
...(0, external_utils_js_namespaceObject.processVlmParams)(modelFormat.conversations, modelFormat.images),
screenContext: {
width,
height
},
mime,
scaleFactor: snapshot.scaleFactor
};
const { prediction, parsedPredictions } = await external_async_retry_default()(async (bail)=>{
try {
const result = await model.invoke(vlmParams);
return result;
} catch (error) {
var _error_message;
if (error instanceof Error && ((null == error ? void 0 : error.name) === 'APIUserAbortError' || (null == error ? void 0 : null === (_error_message = error.message) || void 0 === _error_message ? void 0 : _error_message.includes('aborted')))) {
bail(error);
return {
prediction: '',
parsedPredictions: []
};
}
throw error;
}
}, {
retries: (null == retry ? void 0 : null === (_retry_model = retry.model) || void 0 === _retry_model ? void 0 : _retry_model.maxRetries) ?? 0,
onRetry: null == retry ? void 0 : null === (_retry_model1 = retry.model) || void 0 === _retry_model1 ? void 0 : _retry_model1.onRetry
});
logger.info('[GUIAgent Response]:', prediction);
logger.info('GUIAgent Parsed Predictions:', JSON.stringify(parsedPredictions));
if (!prediction) {
logger.error('[GUIAgent Response Empty]:', prediction);
continue;
}
const predictionSummary = (0, external_utils_js_namespaceObject.getSummary)(prediction);
end = Date.now();
data.conversations.push({
from: 'gpt',
value: predictionSummary,
timing: {
start,
end,
cost: end - start
},
screenshotContext: {
size: {
width,
height
},
scaleFactor: snapshot.scaleFactor
},
predictionParsed: parsedPredictions
});
await (null == onData ? void 0 : onData({
data: {
...data,
conversations: data.conversations.slice(-1)
}
}));
for (const parsedPrediction of parsedPredictions){
const actionType = parsedPrediction.action_type;
logger.info('GUIAgent Action:', actionType);
if ([
external_constants_js_namespaceObject.INTERNAL_ACTION_SPACES_ENUM.CALL_USER,
external_constants_js_namespaceObject.INTERNAL_ACTION_SPACES_ENUM.ERROR_ENV,
external_constants_js_namespaceObject.INTERNAL_ACTION_SPACES_ENUM.FINISHED
].includes(actionType)) {
data.status = types_namespaceObject.StatusEnum.END;
break;
}
if (actionType === external_constants_js_namespaceObject.INTERNAL_ACTION_SPACES_ENUM.MAX_LOOP) {
data.status = types_namespaceObject.StatusEnum.MAX_LOOP;
break;
}
if (!(null == signal ? void 0 : signal.aborted)) {
var _retry_execute, _retry_execute1;
logger.info('GUIAgent Action Inputs:', parsedPrediction.action_inputs, parsedPrediction.action_type);
const executeOutput = await external_async_retry_default()(()=>operator.execute({
prediction,
parsedPrediction,
screenWidth: width,
screenHeight: height,
scaleFactor: snapshot.scaleFactor,
factors: this.model.factors
}), {
retries: (null == retry ? void 0 : null === (_retry_execute = retry.execute) || void 0 === _retry_execute ? void 0 : _retry_execute.maxRetries) ?? 0,
onRetry: null == retry ? void 0 : null === (_retry_execute1 = retry.execute) || void 0 === _retry_execute1 ? void 0 : _retry_execute1.onRetry
}).catch((e)=>{
logger.error('GUIAgent execute error', e);
});
if (executeOutput && (null == executeOutput ? void 0 : executeOutput.status)) data.status = executeOutput.status;
}
}
}
} catch (error) {
var _error_message;
if (error instanceof Error && ('AbortError' === error.name || (null === (_error_message = error.message) || void 0 === _error_message ? void 0 : _error_message.includes('aborted')))) {
logger.info('Request was aborted');
return;
}
logger.error('[GUIAgent] run error', error);
null == onError || onError({
data,
error: {
code: -1,
error: 'GUIAgent Service Error',
stack: `${error}`
}
});
throw error;
} finally{
const prevStatus = data.status;
data.status = types_namespaceObject.StatusEnum.END;
if (data.status !== prevStatus) await (null == onData ? void 0 : onData({
data: {
...data,
conversations: []
}
}));
logger.info('[GUIAgent] finally: status', data.status);
}
}
buildSystemPrompt() {
var _this_operator_constructor_MANUAL, _this_operator_constructor;
const actionSpaces = null === (_this_operator_constructor = this.operator.constructor) || void 0 === _this_operator_constructor ? void 0 : null === (_this_operator_constructor_MANUAL = _this_operator_constructor.MANUAL) || void 0 === _this_operator_constructor_MANUAL ? void 0 : _this_operator_constructor_MANUAL.ACTION_SPACES;
return null == actionSpaces || 0 === actionSpaces.length ? external_constants_js_namespaceObject.SYSTEM_PROMPT : external_constants_js_namespaceObject.SYSTEM_PROMPT_TEMPLATE.replace('{{action_spaces_holder}}', actionSpaces.join('\n'));
}
constructor(config){
super(config), _define_property(this, "operator", void 0), _define_property(this, "model", void 0), _define_property(this, "logger", void 0), _define_property(this, "systemPrompt", void 0);
this.operator = config.operator;
this.model = config.model instanceof external_Model_js_namespaceObject.UITarsModel ? config.model : new external_Model_js_namespaceObject.UITarsModel(config.model);
this.logger = config.logger || console;
this.systemPrompt = config.systemPrompt || this.buildSystemPrompt();
}
}
var __webpack_export_target__ = exports;
for(var __webpack_i__ in __webpack_exports__)__webpack_export_target__[__webpack_i__] = __webpack_exports__[__webpack_i__];
if (__webpack_exports__.__esModule) Object.defineProperty(__webpack_export_target__, '__esModule', {
value: true
});
//# sourceMappingURL=GUIAgent.js.map