@ui-tars/sdk
Version:
A powerful cross-platform(ANY device/platform) toolkit for building GUI automation agents for UI-TARS
153 lines (152 loc) • 6.45 kB
JavaScript
/**
* Copyright (c) 2025 Bytedance, Inc. and its affiliates.
* SPDX-License-Identifier: Apache-2.0
*/
import { Jimp } from "jimp";
import { IMAGE_PLACEHOLDER, MAX_IMAGE_LENGTH } from "@ui-tars/shared/constants";
import { DEFAULT_FACTORS } from "./constants.mjs";
const parseBoxToScreenCoords = ({ boxStr, screenWidth, screenHeight, factors = DEFAULT_FACTORS })=>{
if (!boxStr) return {
x: null,
y: null
};
const coords = boxStr.replace('[', '').replace(']', '').split(',').map((num)=>parseFloat(num.trim()));
const [x1, y1, x2 = x1, y2 = y1] = coords;
const [widthFactor, heightFactor] = factors;
return {
x: Math.round((x1 + x2) / 2 * screenWidth * widthFactor) / widthFactor,
y: Math.round((y1 + y2) / 2 * screenHeight * heightFactor) / heightFactor
};
};
const processVlmParams = (conversations, images, maxImageLength = MAX_IMAGE_LENGTH)=>{
if (images.length > maxImageLength) {
const excessCount = images.length - maxImageLength;
images = images.slice(excessCount);
let imageCountToRemove = excessCount;
conversations = conversations.filter((convo)=>{
if (imageCountToRemove > 0 && convo.value === IMAGE_PLACEHOLDER) {
imageCountToRemove--;
return false;
}
return true;
});
}
return {
images,
conversations
};
};
const toVlmModelFormat = ({ historyMessages, conversations, systemPrompt })=>{
const USER_INSTRUCTION_MARKER = '## User Instruction';
const history = formatHistoryMessages(historyMessages);
return {
conversations: conversations.map((conv, idx)=>{
if (0 === idx && 'human' === conv.from) {
let newValue = '';
if (systemPrompt.includes(USER_INSTRUCTION_MARKER)) {
const insertIndex = systemPrompt.lastIndexOf(USER_INSTRUCTION_MARKER);
const slicedPrefix = systemPrompt.slice(0, insertIndex);
const slicedSuffix = systemPrompt.slice(insertIndex);
newValue = slicedPrefix + (slicedPrefix.endsWith('\n') ? '' : '\n') + history + '\n' + slicedSuffix + (slicedSuffix.endsWith('\n') ? '' : '\n') + conv.value;
} else newValue = `${systemPrompt}\n${history}\n${USER_INSTRUCTION_MARKER}\n${conv.value}`;
return {
from: conv.from,
value: newValue
};
}
return {
from: conv.from,
value: conv.value
};
}),
images: conversations.filter((conv)=>conv.value === IMAGE_PLACEHOLDER && !!conv.screenshotBase64).map((conv)=>conv.screenshotBase64)
};
};
const getSummary = (prediction)=>prediction.replace(/Reflection:[\s\S]*?(?=Action_Summary:|Action:|$)/g, '').trim();
const convertToOpenAIMessages = ({ conversations, images })=>{
const messages = [];
let imageIndex = 0;
conversations.forEach((conv)=>{
if (conv.value === IMAGE_PLACEHOLDER) {
if (imageIndex < images.length) {
messages.push({
role: 'user',
content: [
{
type: 'image_url',
image_url: {
url: `data:image/png;base64,${images[imageIndex]}`
}
}
]
});
imageIndex++;
}
} else messages.push({
role: 'human' === conv.from ? 'user' : 'assistant',
content: conv.value
});
});
return messages;
};
function replaceBase64Prefix(base64) {
return base64.replace(/^data:image\/\w+;base64,/, '');
}
async function preprocessResizeImage(image_base64, maxPixels) {
try {
const imageBuffer = Buffer.from(image_base64, 'base64');
const image = await Jimp.read(imageBuffer);
const { width, height } = image.bitmap;
const currentPixels = width * height;
if (currentPixels > maxPixels) {
const resizeFactor = Math.sqrt(maxPixels / currentPixels);
const newWidth = Math.floor(width * resizeFactor);
const newHeight = Math.floor(height * resizeFactor);
const resized = await image.resize({
w: newWidth,
h: newHeight
}).getBuffer('image/png', {
quality: 60
});
return resized.toString('base64');
}
const base64 = await image.getBase64('image/png', {
quality: 60
});
return replaceBase64Prefix(base64);
} catch (error) {
console.error('preprocessResizeImage error:', error);
throw error;
}
}
function formatHistoryMessages(messages) {
const lastMessages = messages.slice(-30);
const lines = lastMessages.map((msg)=>{
const role = 'human' === msg.from ? 'human' : 'assistant';
return `${role}: ${msg.value}`;
});
return '## History Messages\n' + lines.join('\n') + '\n';
}
const convertToResponseApiInput = (messages)=>messages.map((message)=>{
if (Array.isArray(null == message ? void 0 : message.content) && (null == message ? void 0 : message.content.length) > 0) {
const content = message.content.map((item)=>{
var _item_image_url;
if ('image_url' === item.type && (null == (_item_image_url = item.image_url) ? void 0 : _item_image_url.url)) return {
type: 'input_image',
image_url: item.image_url.url
};
return item;
});
return {
role: message.role,
content
};
}
return message;
});
const isMessageImage = (c)=>'role' in c && 'user' === c.role && Array.isArray(c.content) && c.content.some((item)=>{
var _item_image_url;
return 'image_url' === item.type && (null == (_item_image_url = item.image_url) ? void 0 : _item_image_url.url) || 'input_image' === item.type && item.image_url;
});
export { convertToOpenAIMessages, convertToResponseApiInput, getSummary, isMessageImage, parseBoxToScreenCoords, preprocessResizeImage, processVlmParams, replaceBase64Prefix, toVlmModelFormat };
//# sourceMappingURL=utils.mjs.map