askui
Version:
Reliable, automated end-to-end-testing that depends on what is shown on your screen instead of the technology you are running on
250 lines (249 loc) • 12.3 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.AndroidAdbClient = void 0;
const child_process_1 = require("child_process");
const ui_control_commands_1 = require("../../core/ui-control-commands");
const logger_1 = require("../../lib/logger");
const ui_controller_client_connection_state_1 = require("../ui-controller-client-connection-state");
const android_error_1 = require("./android-error");
const android_not_connected_error_1 = require("./android-not-connected-error");
const no_android_device_error_1 = require("./no-android-device-error");
/**
* Drives an Android device directly via `adb`, implementing {@link DeviceClient}
* so it is interchangeable with the desktop AgentOS gRPC client. Replaces the
* legacy UI Controller's WebSocket/adbkit path: every operation is a plain
* `adb` invocation, so no separate controller process or binary is required.
*/
class AndroidAdbClient {
constructor(args = {}) {
var _a, _b, _c;
this.args = args;
this.connectionState = ui_controller_client_connection_state_1.UiControllerClientConnectionState.NOT_CONNECTED;
this.screenWidth = 0;
this.screenHeight = 0;
this.mousePosition = { x: 0, y: 0 };
this.adbPath = (_a = args.adbPath) !== null && _a !== void 0 ? _a : 'adb';
this.keyboard = (_b = args.keyboard) !== null && _b !== void 0 ? _b : 'input-text';
this.actionDelayInMs = (_c = args.actionDelayInMs) !== null && _c !== void 0 ? _c : 0;
this.deviceId = args.id;
}
exec(args) {
return new Promise((resolve, reject) => {
(0, child_process_1.execFile)(this.adbPath, args, { encoding: 'buffer', maxBuffer: AndroidAdbClient.EXEC_MAX_BUFFER }, (error, stdout, stderr) => {
if (error) {
const details = (stderr === null || stderr === void 0 ? void 0 : stderr.toString().trim()) || error.message;
reject(new android_error_1.AndroidError(`adb ${args.join(' ')} failed: ${details}`));
return;
}
resolve(stdout);
});
});
}
shell(command) {
return __awaiter(this, void 0, void 0, function* () {
if (this.deviceId === undefined) {
throw new android_not_connected_error_1.AndroidNotConnectedError();
}
const output = yield this.exec(['-s', this.deviceId, 'shell', command]);
return output.toString().trim();
});
}
listDevices() {
return __awaiter(this, void 0, void 0, function* () {
const output = (yield this.exec(['devices'])).toString();
return output
.split('\n')
.slice(1)
.map((line) => line.trim())
.filter((line) => line.endsWith('\tdevice'))
.map((line) => line.split('\t')[0]);
});
}
resolveDeviceId() {
return __awaiter(this, void 0, void 0, function* () {
const devices = yield this.listDevices();
if (this.args.id !== undefined) {
if (!devices.includes(this.args.id)) {
throw new no_android_device_error_1.NoAndroidDeviceError(this.args.id);
}
return this.args.id;
}
if (devices.length === 0) {
throw new no_android_device_error_1.NoAndroidDeviceError();
}
return devices[0];
});
}
updateScreenSize() {
return __awaiter(this, void 0, void 0, function* () {
const output = yield this.shell('wm size');
// Prefer the override size ("Override size: WxH") if present, else physical.
const matches = [...output.matchAll(/(?:Physical|Override) size:\s*(\d+)x(\d+)/g)];
const last = matches[matches.length - 1];
if (last) {
this.screenWidth = Number(last[1]);
this.screenHeight = Number(last[2]);
}
});
}
connect() {
return __awaiter(this, void 0, void 0, function* () {
this.connectionState = ui_controller_client_connection_state_1.UiControllerClientConnectionState.CONNECTING;
try {
this.deviceId = yield this.resolveDeviceId();
yield this.updateScreenSize();
this.connectionState = ui_controller_client_connection_state_1.UiControllerClientConnectionState.CONNECTED;
logger_1.logger.debug(`Connected to Android device ${this.deviceId} (${this.screenWidth}x${this.screenHeight})`);
return this.connectionState;
}
catch (error) {
this.connectionState = ui_controller_client_connection_state_1.UiControllerClientConnectionState.ERROR;
this.deviceId = this.args.id;
if (error instanceof android_error_1.AndroidError) {
throw error;
}
throw new android_error_1.AndroidError(`Connection to an Android device via adb failed. Make sure "${this.adbPath}" `
+ `is installed and a device is connected. Cause: ${error}`);
}
});
}
disconnect() {
this.deviceId = this.args.id;
this.connectionState = ui_controller_client_connection_state_1.UiControllerClientConnectionState.NOT_CONNECTED;
}
// eslint-disable-next-line class-methods-use-this
setActiveDisplay() {
return __awaiter(this, void 0, void 0, function* () {
// Android targets a single device/display selected at connect time.
});
}
requireConnected() {
if (this.deviceId === undefined
|| this.connectionState !== ui_controller_client_connection_state_1.UiControllerClientConnectionState.CONNECTED) {
throw new android_not_connected_error_1.AndroidNotConnectedError();
}
}
delay() {
if (this.actionDelayInMs <= 0) {
return Promise.resolve();
}
return new Promise((resolve) => {
setTimeout(resolve, this.actionDelayInMs);
});
}
requestControl(controlCommand) {
return __awaiter(this, void 0, void 0, function* () {
this.requireConnected();
/* eslint-disable no-await-in-loop, no-restricted-syntax */
for (const action of controlCommand.actions) {
yield this.runAction(action);
yield this.delay();
}
/* eslint-enable no-await-in-loop, no-restricted-syntax */
});
}
runAction(action) {
return __awaiter(this, void 0, void 0, function* () {
switch (action.inputEvent) {
case ui_control_commands_1.InputEvent.NO_COMMAND:
break;
case ui_control_commands_1.InputEvent.MOUSE_MOVE:
this.mousePosition = { x: action.position.x, y: action.position.y };
break;
case ui_control_commands_1.InputEvent.MOUSE_MOVE_RELATIVELY:
this.mousePosition = {
x: this.mousePosition.x + action.position.x,
y: this.mousePosition.y + action.position.y,
};
break;
case ui_control_commands_1.InputEvent.MOUSE_DOWN:
yield this.shell(`input motionevent DOWN ${this.mousePosition.x} ${this.mousePosition.y}`);
break;
case ui_control_commands_1.InputEvent.MOUSE_UP:
yield this.shell(`input motionevent UP ${this.mousePosition.x} ${this.mousePosition.y}`);
break;
case ui_control_commands_1.InputEvent.MOUSE_CLICK_LEFT:
case ui_control_commands_1.InputEvent.MOUSE_CLICK_RIGHT:
case ui_control_commands_1.InputEvent.MOUSE_CLICK_MIDDLE:
yield this.shell(`input tap ${this.mousePosition.x} ${this.mousePosition.y}`);
break;
case ui_control_commands_1.InputEvent.MOUSE_CLICK_DOUBLE_LEFT:
case ui_control_commands_1.InputEvent.MOUSE_CLICK_DOUBLE_RIGHT:
case ui_control_commands_1.InputEvent.MOUSE_CLICK_DOUBLE_MIDDLE:
yield this.shell(`input tap ${this.mousePosition.x} ${this.mousePosition.y}`);
yield this.shell(`input tap ${this.mousePosition.x} ${this.mousePosition.y}`);
break;
case ui_control_commands_1.InputEvent.MOUSE_SCROLL:
// Negated to match the desktop scroll orientation.
yield this.shell(`input roll ${-action.position.x} ${-action.position.y}`);
break;
case ui_control_commands_1.InputEvent.TYPE:
yield this.typeText(action.text);
break;
case ui_control_commands_1.InputEvent.TYPE_TEXT:
yield this.shell(`input tap ${action.position.x} ${action.position.y}`);
yield this.typeText(action.text);
break;
case ui_control_commands_1.InputEvent.PRESS_ANDROID_SINGLE_KEY:
yield this.shell(`input keyevent ${action.text.toUpperCase()}`);
break;
case ui_control_commands_1.InputEvent.PRESS_ANDROID_KEY_SEQUENCE: {
// Press the keys one after another. `+` and whitespace both separate keys.
const keys = action.text.toUpperCase().split(/[\s+]+/).filter(Boolean);
yield this.shell(`input keyevent ${keys.join(' ')}`);
break;
}
case ui_control_commands_1.InputEvent.EXECUTE_COMMAND:
yield this.shell(action.text);
break;
default:
throw new android_error_1.AndroidError(`The action "${action.inputEvent}" is not supported on Android.`);
}
});
}
typeText(text) {
return __awaiter(this, void 0, void 0, function* () {
if (this.keyboard === 'adb-keyboard') {
// ADBKeyboard IME broadcast (base64-encoded UTF-8) supports full Unicode.
const encoded = Buffer.from(text, 'utf-8').toString('base64');
yield this.shell(`am broadcast -a ADB_INPUT_B64 --es msg ${encoded}`);
return;
}
// `input text` treats spaces specially and does not support Unicode.
const escaped = text
.replace(/(["`$\\])/g, '\\$1')
.replace(/ /g, '%s');
yield this.shell(`input text "${escaped}"`);
});
}
requestScreenshot() {
return __awaiter(this, void 0, void 0, function* () {
this.requireConnected();
const png = yield this.exec(['-s', this.deviceId, 'exec-out', 'screencap', '-p']);
return `data:image/png;base64,${png.toString('base64')}`;
});
}
getStartingArguments() {
return __awaiter(this, void 0, void 0, function* () {
this.requireConnected();
return {
deviceId: this.deviceId,
displayNum: 0,
height: this.screenHeight,
runtime: 'android',
width: this.screenWidth,
};
});
}
}
exports.AndroidAdbClient = AndroidAdbClient;
AndroidAdbClient.EXEC_MAX_BUFFER = 256 * 1024 * 1024;