@gui-agent/operator-aio
Version:
AIO (All-in-One) operator for GUI Agent
225 lines (224 loc) • 6.62 kB
JavaScript
/**
* Copyright (c) 2025 Bytedance, Inc. and its affiliates.
* SPDX-License-Identifier: Apache-2.0
*/
import { ConsoleLogger } from "@agent-infra/logger";
function _define_property(obj, key, value) {
if (key in obj) Object.defineProperty(obj, key, {
value: value,
enumerable: true,
configurable: true,
writable: true
});
else obj[key] = value;
return obj;
}
const logger = new ConsoleLogger('AIOComputer');
class AIOComputer {
async request(action) {
const url = `${this.baseURL}/v1/browser/actions`;
try {
logger.info('[AIOComputer] Executing action:', action.action_type, action);
const response = await fetch(url, {
method: 'POST',
headers: this.headers,
body: JSON.stringify(action),
signal: AbortSignal.timeout(this.timeout)
});
if (!response.ok) throw new Error(`HTTP ${response.status}: ${response.statusText}`);
const result = await response.json();
logger.info('[AIOComputer] Action result:', result);
return {
success: true,
data: result
};
} catch (error) {
logger.error('[AIOComputer] Action failed:', error);
return {
success: false,
message: error instanceof Error ? error.message : 'Unknown error'
};
}
}
async screenshot(delay = 1000) {
const url = `${this.baseURL}/v1/browser/screenshot`;
try {
logger.info('[AIOComputer] Taking screenshot' + (delay ? ` with ${delay}ms delay` : ''));
if (delay > 0) await new Promise((resolve)=>setTimeout(resolve, delay));
const response = await fetch(url, {
method: 'GET',
headers: this.headers,
signal: AbortSignal.timeout(this.timeout)
});
if (!response.ok) throw new Error(`HTTP ${response.status}: ${response.statusText}`);
const contentType = response.headers.get('content-type');
if (contentType && contentType.startsWith('image/')) {
const arrayBuffer = await response.arrayBuffer();
const base64 = Buffer.from(arrayBuffer).toString('base64');
logger.info('[AIOComputer] Screenshot taken successfully');
return {
success: true,
data: {
base64,
scaleFactor: 1,
contentType
}
};
}
{
const result = await response.json();
logger.info('[AIOComputer] Screenshot result:', result);
return {
success: true,
data: result
};
}
} catch (error) {
logger.error('[AIOComputer] Screenshot failed:', error);
return {
success: false,
message: error instanceof Error ? error.message : 'Unknown error'
};
}
}
async moveTo(x, y) {
const action = {
action_type: 'MOVE_TO',
x,
y
};
return this.request(action);
}
async click(x, y, button, numClicks) {
const action = {
action_type: 'CLICK',
...void 0 !== x && {
x
},
...void 0 !== y && {
y
},
...button && {
button
},
...numClicks && {
num_clicks: numClicks
}
};
return this.request(action);
}
async mouseDown(button) {
const action = {
action_type: 'MOUSE_DOWN',
...button && {
button
}
};
return this.request(action);
}
async mouseUp(button) {
const action = {
action_type: 'MOUSE_UP',
...button && {
button
}
};
return this.request(action);
}
async rightClick(x, y) {
const action = {
action_type: 'RIGHT_CLICK',
...void 0 !== x && {
x
},
...void 0 !== y && {
y
}
};
return this.request(action);
}
async doubleClick(x, y) {
const action = {
action_type: 'DOUBLE_CLICK',
...void 0 !== x && {
x
},
...void 0 !== y && {
y
}
};
return this.request(action);
}
async dragTo(x, y) {
const action = {
action_type: 'DRAG_TO',
x,
y
};
return this.request(action);
}
async scroll(dx, dy) {
const action = {
action_type: 'SCROLL',
...void 0 !== dx && {
dx
},
...void 0 !== dy && {
dy
}
};
return this.request(action);
}
async type(text) {
const action = {
action_type: 'TYPING',
text
};
return this.request(action);
}
async press(key) {
const action = {
action_type: 'PRESS',
key
};
return this.request(action);
}
async keyDown(key) {
const action = {
action_type: 'KEY_DOWN',
key
};
return this.request(action);
}
async keyUp(key) {
const action = {
action_type: 'KEY_UP',
key
};
return this.request(action);
}
async hotkey(keys) {
const lowercaseKeys = keys.map((key)=>key.toLowerCase());
const action = {
action_type: 'HOTKEY',
keys: lowercaseKeys
};
return this.request(action);
}
async execute(action) {
return this.request(action);
}
constructor(options){
_define_property(this, "baseURL", void 0);
_define_property(this, "timeout", void 0);
_define_property(this, "headers", void 0);
this.baseURL = options.baseURL.replace(/\/$/, '');
this.timeout = options.timeout || 30000;
this.headers = {
'Content-Type': 'application/json',
...options.headers
};
}
}
export { AIOComputer };
//# sourceMappingURL=AIOComputer.mjs.map