askui
Version:
Reliable, automated end-to-end-testing that depends on what is shown on your screen instead of the technology you are running on
474 lines (473 loc) • 22.5 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.AgentOsClient = void 0;
const grpc = __importStar(require("@grpc/grpc-js"));
const protoLoader = __importStar(require("@grpc/proto-loader"));
const crypto_1 = require("crypto");
const path_1 = __importDefault(require("path"));
const ui_control_commands_1 = require("../../core/ui-control-commands");
const logger_1 = require("../../lib/logger");
const sharp_1 = require("../../utils/base_64_image/sharp");
const ui_controller_client_connection_state_1 = require("../ui-controller-client-connection-state");
const misc_1 = require("../misc");
const agent_os_error_1 = require("./agent-os-error");
const agent_os_not_connected_error_1 = require("./agent-os-not-connected-error");
const agent_os_action_not_supported_error_1 = require("./agent-os-action-not-supported-error");
/**
* Client for the AskUI AgentOS (AskUI Remote Device Controller). Talks gRPC to the
* `Askui.API.TDKv1.ControllerAPI` service, mirroring the connection approach of the
* AskUI Python SDK (askui/vision-agent): open an insecure channel to a locally running
* AgentOS, start a session, start execution, and set the active display.
*
* The AgentOS listens on `localhost:23000` when running standalone and on
* `localhost:26000` when managed by the AskUI OS service (`AskuiCoreService`).
*/
class AgentOsClient {
constructor(url) {
this.url = url;
this.connectionState = ui_controller_client_connection_state_1.UiControllerClientConnectionState.NOT_CONNECTED;
this.displayId = 1;
}
static buildControllerApiConstructor() {
const packageDefinition = protoLoader.loadSync(path_1.default.join(__dirname, 'proto', 'Controller_V1.proto'), {
defaults: true,
enums: String,
keepCase: true,
// 64-bit integers (e.g. the session GUID parts) must round-trip without
// precision loss, so they are represented as strings.
longs: String,
oneofs: true,
});
const grpcObject = grpc.loadPackageDefinition(packageDefinition);
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return grpcObject.Askui.API.TDKv1.ControllerAPI;
}
static normalizeAddress(url) {
const address = url.replace(/^(https?|wss?):\/\//, '');
if (address.includes(':')) {
return address;
}
return `${address}:${AgentOsClient.DEFAULT_PORT}`;
}
static openChannel(address) {
return __awaiter(this, void 0, void 0, function* () {
const ControllerApi = AgentOsClient.buildControllerApiConstructor();
const client = new ControllerApi(address, grpc.credentials.createInsecure(), {
'grpc.max_receive_message_length': Math.pow(2, 30),
'grpc.max_send_message_length': Math.pow(2, 30),
});
return new Promise((resolve, reject) => {
client.waitForReady(new Date(Date.now() + AgentOsClient.CONNECT_TIMEOUT_IN_MS), (error) => {
if (error) {
client.close();
reject(error);
return;
}
resolve(client);
});
});
});
}
requireClient() {
if (this.client === undefined
|| this.connectionState !== ui_controller_client_connection_state_1.UiControllerClientConnectionState.CONNECTED) {
throw new agent_os_not_connected_error_1.AgentOsNotConnectedError();
}
return this.client;
}
static invokeOn(client, method, request, timeoutInMs = AgentOsClient.REQUEST_TIMEOUT_IN_MS) {
logger_1.logger.debug(`AgentOS request: ${method}`);
return new Promise((resolve, reject) => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
client[method](request, { deadline: new Date(Date.now() + timeoutInMs) }, (error, response) => {
if (error) {
reject(new agent_os_error_1.AgentOsError(`The AgentOS request "${method}" failed. Cause: ${error.message}`));
return;
}
resolve(response);
});
});
}
invoke(method, request, timeoutInMs = AgentOsClient.REQUEST_TIMEOUT_IN_MS) {
const { client } = this;
if (client === undefined) {
throw new agent_os_not_connected_error_1.AgentOsNotConnectedError();
}
return AgentOsClient.invokeOn(client, method, request, timeoutInMs);
}
connect() {
return __awaiter(this, void 0, void 0, function* () {
var _a;
this.connectionState = ui_controller_client_connection_state_1.UiControllerClientConnectionState.CONNECTING;
const address = AgentOsClient.normalizeAddress(this.url);
try {
this.client = yield AgentOsClient.openChannel(address);
this.address = address;
yield this.startSession();
this.connectionState = ui_controller_client_connection_state_1.UiControllerClientConnectionState.CONNECTED;
logger_1.logger.debug(`Connected to AgentOS at ${address}`);
return this.connectionState;
}
catch (error) {
this.connectionState = ui_controller_client_connection_state_1.UiControllerClientConnectionState.ERROR;
(_a = this.client) === null || _a === void 0 ? void 0 : _a.close();
this.client = undefined;
this.address = undefined;
throw new agent_os_error_1.AgentOsError(`Connection to the AskUI AgentOS at "${address}" cannot be established. `
+ 'Make sure the AskUI AgentOS is installed and running. '
+ `Install it from ${AgentOsClient.INSTALLATION_DOCS_URL}. Cause: ${error}`);
}
});
}
startSession() {
return __awaiter(this, void 0, void 0, function* () {
const sessionGuid = `{${(0, crypto_1.randomUUID)().toUpperCase()}}`;
const response = yield this.invoke('StartSession', {
immediateExecution: true,
sessionGUID: sessionGuid,
});
this.sessionInfo = response.sessionInfo;
yield this.invoke('StartExecution', { sessionInfo: this.sessionInfo });
yield this.invoke('SetActiveDisplay', { displayID: this.displayId });
});
}
disconnect() {
const { client, sessionInfo } = this;
if (client === undefined) {
return;
}
this.client = undefined;
this.sessionInfo = undefined;
this.connectionState = ui_controller_client_connection_state_1.UiControllerClientConnectionState.NOT_CONNECTED;
// End the session gracefully in the background, then close the channel.
// Keeping this fire-and-forget preserves the synchronous disconnect() API
// while still releasing the single AgentOS session on the server.
(() => __awaiter(this, void 0, void 0, function* () {
try {
yield AgentOsClient.invokeOn(client, 'StopExecution', { sessionInfo });
yield AgentOsClient.invokeOn(client, 'EndSession', { sessionInfo });
}
catch (error) {
logger_1.logger.debug(`Ending the AgentOS session failed: ${error}`);
}
finally {
client.close();
}
}))();
}
setActiveDisplay(displayId) {
return __awaiter(this, void 0, void 0, function* () {
this.requireClient();
yield this.invoke('SetActiveDisplay', { displayID: displayId });
this.displayId = displayId;
});
}
runAction(actionRequest, timeoutInMs) {
return __awaiter(this, void 0, void 0, function* () {
var _a, _b;
const response = yield this.invoke('RunRecordedAction', {
actionClassID: actionRequest.actionClassID,
actionParameters: actionRequest.actionParameters,
sessionInfo: this.sessionInfo,
}, timeoutInMs);
yield (0, misc_1.delay)(response.requiredMilliseconds);
/* eslint-disable no-await-in-loop */
for (let retry = 0; retry < AgentOsClient.ACTION_POLL_MAX_RETRIES; retry += 1) {
const pollResponse = yield this.invoke('Poll', {
pollEventID: 'PollEventID_ActionFinished',
sessionInfo: this.sessionInfo,
});
if (((_b = (_a = pollResponse.pollEventParameters) === null || _a === void 0 ? void 0 : _a.actionFinished) === null || _b === void 0 ? void 0 : _b.actionID) === response.actionID) {
return;
}
yield (0, misc_1.delay)(AgentOsClient.ACTION_POLL_INTERVAL_IN_MS);
}
/* eslint-enable no-await-in-loop */
throw new agent_os_error_1.AgentOsError(`The AgentOS did not finish the action "${actionRequest.actionClassID}" in time.`);
});
}
requestControl(controlCommand) {
return __awaiter(this, void 0, void 0, function* () {
this.requireClient();
/* eslint-disable no-await-in-loop, no-restricted-syntax */
for (const action of controlCommand.actions) {
for (const actionRequest of AgentOsClient.mapAction(action)) {
yield this.runAction(actionRequest);
}
}
/* eslint-enable no-await-in-loop, no-restricted-syntax */
});
}
static mouseButtonPressAndRelease(button, count) {
return [{
actionClassID: 'ActionClassID_MouseButton_PressAndRelease',
actionParameters: {
mouseButtonPressAndRelease: { count, mouseButton: button },
},
}];
}
static parseKeySequence(text) {
const modifierKeyNames = [];
let remaining = text;
let matched = true;
while (matched) {
matched = false;
/* eslint-disable-next-line no-restricted-syntax */
for (const modifier of AgentOsClient.MODIFIER_KEYS) {
if (remaining.length > modifier.length + 1
&& remaining.startsWith(`${modifier}+`)) {
modifierKeyNames.push(modifier);
remaining = remaining.substring(modifier.length + 1);
matched = true;
break;
}
}
}
return { keyName: remaining, modifierKeyNames };
}
static typeText(text) {
return [{
actionClassID: 'ActionClassID_KeyboardType_UnicodeText',
actionParameters: {
keyboardTypeUnicodeText: {
text: Buffer.from(text, 'utf16le'),
typingSpeed: AgentOsClient.TYPING_SPEED_IN_CHARACTERS_PER_SECOND,
typingSpeedValue: 'TypingSpeedValue_CharactersPerSecond',
},
},
}];
}
static mouseScroll(deltaX, deltaY) {
const actionRequests = [];
if (deltaX !== 0) {
actionRequests.push({
actionClassID: 'ActionClassID_MouseWheelScroll',
actionParameters: {
mouseWheelScroll: {
delta: deltaX,
deltaType: 'MouseWheelDelta_Raw',
direction: 'MouseWheelScrollDirection_Horizontal',
milliseconds: AgentOsClient.MOUSE_SCROLL_DURATION_IN_MS,
},
},
});
}
if (deltaY !== 0) {
actionRequests.push({
actionClassID: 'ActionClassID_MouseWheelScroll',
actionParameters: {
mouseWheelScroll: {
delta: deltaY,
deltaType: 'MouseWheelDelta_Raw',
direction: 'MouseWheelScrollDirection_Vertical',
milliseconds: AgentOsClient.MOUSE_SCROLL_DURATION_IN_MS,
},
},
});
}
return actionRequests;
}
static keyPressOrRelease(action, press) {
const key = typeof action.parameters['key'] === 'string' ? action.parameters['key'] : action.text;
const modifiers = Array.isArray(action.parameters['modifiers'])
? action.parameters['modifiers']
: [];
if (press) {
return [{
actionClassID: 'ActionClassID_KeyboardKey_Press',
actionParameters: {
keyboardKeyPress: { keyName: key, modifierKeyNames: modifiers },
},
}];
}
return [{
actionClassID: 'ActionClassID_KeyboardKey_Release',
actionParameters: {
keyboardKeyRelease: { keyName: key, modifierKeyNames: modifiers },
},
}];
}
static mapAction(action) {
switch (action.inputEvent) {
case ui_control_commands_1.InputEvent.NO_COMMAND:
return [];
case ui_control_commands_1.InputEvent.MOUSE_MOVE:
return [{
actionClassID: 'ActionClassID_MouseMove',
actionParameters: {
mouseMove: {
milliseconds: AgentOsClient.MOUSE_MOVE_DURATION_IN_MS,
position: { x: action.position.x, y: action.position.y },
},
},
}];
case ui_control_commands_1.InputEvent.MOUSE_MOVE_RELATIVELY:
return [{
actionClassID: 'ActionClassID_MouseMove_Delta',
actionParameters: {
mouseMoveDelta: {
delta: { x: action.position.x, y: action.position.y },
milliseconds: AgentOsClient.MOUSE_MOVE_DURATION_IN_MS,
},
},
}];
case ui_control_commands_1.InputEvent.MOUSE_CLICK_LEFT:
return AgentOsClient.mouseButtonPressAndRelease('MouseButton_Left', 1);
case ui_control_commands_1.InputEvent.MOUSE_CLICK_RIGHT:
return AgentOsClient.mouseButtonPressAndRelease('MouseButton_Right', 1);
case ui_control_commands_1.InputEvent.MOUSE_CLICK_MIDDLE:
return AgentOsClient.mouseButtonPressAndRelease('MouseButton_Middle', 1);
case ui_control_commands_1.InputEvent.MOUSE_CLICK_DOUBLE_LEFT:
return AgentOsClient.mouseButtonPressAndRelease('MouseButton_Left', 2);
case ui_control_commands_1.InputEvent.MOUSE_CLICK_DOUBLE_RIGHT:
return AgentOsClient.mouseButtonPressAndRelease('MouseButton_Right', 2);
case ui_control_commands_1.InputEvent.MOUSE_CLICK_DOUBLE_MIDDLE:
return AgentOsClient.mouseButtonPressAndRelease('MouseButton_Middle', 2);
case ui_control_commands_1.InputEvent.MOUSE_DOWN:
return [{
actionClassID: 'ActionClassID_MouseButton_Press',
actionParameters: {
mouseButtonPress: { mouseButton: 'MouseButton_Left' },
},
}];
case ui_control_commands_1.InputEvent.MOUSE_UP:
return [{
actionClassID: 'ActionClassID_MouseButton_Release',
actionParameters: {
mouseButtonRelease: { mouseButton: 'MouseButton_Left' },
},
}];
case ui_control_commands_1.InputEvent.MOUSE_SCROLL:
return AgentOsClient.mouseScroll(action.position.x, action.position.y);
case ui_control_commands_1.InputEvent.TYPE:
case ui_control_commands_1.InputEvent.TYPE_TEXT:
return AgentOsClient.typeText(action.text);
case ui_control_commands_1.InputEvent.PRESS_KEY_SEQUENCE: {
const { keyName, modifierKeyNames } = AgentOsClient.parseKeySequence(action.text);
return [{
actionClassID: 'ActionClassID_KeyboardKey_PressAndRelease',
actionParameters: {
keyboardKeyPressAndRelease: { keyName, modifierKeyNames },
},
}];
}
case ui_control_commands_1.InputEvent.KEY_PRESS:
return AgentOsClient.keyPressOrRelease(action, true);
case ui_control_commands_1.InputEvent.KEY_RELEASE:
return AgentOsClient.keyPressOrRelease(action, false);
case ui_control_commands_1.InputEvent.EXECUTE_COMMAND:
return [{
actionClassID: 'ActionClassID_RunCommand',
actionParameters: {
runcommand: {
command: action.text,
timeoutInMilliseconds: AgentOsClient.REQUEST_TIMEOUT_IN_MS,
},
},
}];
case ui_control_commands_1.InputEvent.PRESS_ANDROID_KEY_SEQUENCE:
case ui_control_commands_1.InputEvent.PRESS_ANDROID_SINGLE_KEY:
throw new agent_os_action_not_supported_error_1.AgentOsActionNotSupportedError(action.inputEvent);
default:
throw new agent_os_error_1.AgentOsError(`Unknown input event "${action.inputEvent}".`);
}
}
static bitmapToRgba(bitmap) {
const { width, height, lineWidth, bytesPerPixel, data, } = bitmap;
const rgba = Buffer.alloc(width * height * 4);
for (let row = 0; row < height; row += 1) {
for (let column = 0; column < width; column += 1) {
const source = row * lineWidth + column * bytesPerPixel;
const target = (row * width + column) * 4;
// The AgentOS returns the bitmap in BGRA channel order.
rgba[target] = data[source + 2];
rgba[target + 1] = data[source + 1];
rgba[target + 2] = data[source];
rgba[target + 3] = 255;
}
}
return rgba;
}
requestScreenshot() {
return __awaiter(this, void 0, void 0, function* () {
this.requireClient();
const response = yield this.invoke('CaptureScreen', {
captureParameters: { displayID: this.displayId },
sessionInfo: this.sessionInfo,
});
const { width, height } = response.bitmap;
const sharpFactory = yield (0, sharp_1.getSharpFactory)();
const pngBuffer = yield sharpFactory(AgentOsClient.bitmapToRgba(response.bitmap), {
raw: { channels: 4, height, width },
})
.png()
.toBuffer();
return `data:image/png;base64,${pngBuffer.toString('base64')}`;
});
}
getStartingArguments() {
return __awaiter(this, void 0, void 0, function* () {
var _a;
this.requireClient();
const [host = 'localhost', port = AgentOsClient.DEFAULT_PORT] = ((_a = this.address) !== null && _a !== void 0 ? _a : '').split(':');
return {
displayNum: this.displayId,
host,
port,
runtime: 'desktop',
};
});
}
}
exports.AgentOsClient = AgentOsClient;
AgentOsClient.INSTALLATION_DOCS_URL = 'https://docs.askui.com/06-agent-os/installation/service';
AgentOsClient.DEFAULT_PORT = '26000';
AgentOsClient.REQUEST_TIMEOUT_IN_MS = 30000;
AgentOsClient.CONNECT_TIMEOUT_IN_MS = 10000;
AgentOsClient.ACTION_POLL_MAX_RETRIES = 10;
AgentOsClient.ACTION_POLL_INTERVAL_IN_MS = 50;
AgentOsClient.TYPING_SPEED_IN_CHARACTERS_PER_SECOND = 50;
AgentOsClient.MOUSE_MOVE_DURATION_IN_MS = 500;
AgentOsClient.MOUSE_SCROLL_DURATION_IN_MS = 50;
AgentOsClient.MODIFIER_KEYS = [
'command',
'alt',
'control',
'shift',
'right_shift',
];