askui
Version:
Reliable, automated end-to-end-testing that depends on what is shown on your screen instead of the technology you are running on
906 lines (905 loc) • 38.8 kB
JavaScript
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.UiControlClient = void 0;
const yup_1 = require("yup");
const custom_element_1 = require("../core/model/custom-element");
const dsl_1 = require("./dsl");
const annotation_writer_1 = require("../core/annotation/annotation-writer");
const logger_1 = require("../lib/logger");
const ui_control_client_dependency_builder_1 = require("./ui-control-client-dependency-builder");
const ai_element_collection_1 = require("../core/ai-element/ai-element-collection");
const retry_strategies_1 = require("./retry-strategies");
const anthropic_1 = require("../core/models/anthropic");
const askui_api_tools_1 = require("../core/models/anthropic/tools/askui-api-tools");
class UiControlClient extends dsl_1.ApiCommands {
constructor(workspaceId, executionRuntime, stepReporter, aiElementArgs, agent) {
super();
this.workspaceId = workspaceId;
this.executionRuntime = executionRuntime;
this.stepReporter = stepReporter;
this.aiElementArgs = aiElementArgs;
this.agent = agent;
this.secretText = undefined;
}
static build() {
return __awaiter(this, arguments, void 0, function* (clientArgs = {}) {
const builder = ui_control_client_dependency_builder_1.UiControlClientDependencyBuilder;
const clientArgsWithDefaults = yield builder.getClientArgsWithDefaults(clientArgs);
const { workspaceId, executionRuntime, stepReporter, } = yield builder.build(clientArgsWithDefaults);
const agent = new anthropic_1.AskUIAgent(executionRuntime);
return new UiControlClient(workspaceId, executionRuntime, stepReporter, clientArgsWithDefaults.aiElementArgs, agent);
});
}
/**
* Connects to the askui UI Controller.
*/
connect() {
return __awaiter(this, void 0, void 0, function* () {
const connectionState = yield this.executionRuntime.connect();
yield this.agent.initializeOsAgentHandler();
yield this.agent.configureAsDesktopAgent();
return connectionState;
});
}
/**
* Disconnects from the askui UI Controller.
*/
disconnect() {
this.executionRuntime.disconnect();
}
/**
* Disconnects from the askui UI Controller.
*
* @deprecated Use {@link disconnect} instead.
*/
close() {
this.disconnect();
}
startVideoRecording() {
return __awaiter(this, void 0, void 0, function* () {
yield this.executionRuntime.startVideoRecording();
});
}
stopVideoRecording() {
return __awaiter(this, void 0, void 0, function* () {
yield this.executionRuntime.stopVideoRecording();
});
}
readVideoRecording() {
return __awaiter(this, void 0, void 0, function* () {
return this.executionRuntime.readVideoRecording();
});
}
shouldAnnotateAfterCommandExecution(error) {
return (this.stepReporter.config.withDetectedElements === 'onFailure' && error !== undefined)
|| (this.stepReporter.config.withDetectedElements === 'always');
}
beforeNoneInferenceCallCommandExecution(instruction) {
return __awaiter(this, void 0, void 0, function* () {
this.stepReporter.resetStep(instruction);
let annotation;
if (this.stepReporter.config.withDetectedElements === 'begin'
|| this.stepReporter.config.withDetectedElements === 'always') {
annotation = yield this.executionRuntime.annotateImage();
}
const createdAt = new Date();
yield this.stepReporter.onStepBegin({
createdAt,
detectedElements: annotation === null || annotation === void 0 ? void 0 : annotation.detected_elements,
screenshot: annotation === null || annotation === void 0 ? void 0 : annotation.image,
});
});
}
afterCommandExecution(instruction, error) {
return __awaiter(this, void 0, void 0, function* () {
var _a;
const createdAt = new Date();
let annotation;
let screenshot;
if (this.shouldAnnotateAfterCommandExecution(error)) {
annotation = yield this.executionRuntime.annotateImage(undefined, instruction.customElements);
}
if (annotation !== undefined || this.stepReporter.config.withScreenshots === 'always') {
screenshot = (_a = annotation === null || annotation === void 0 ? void 0 : annotation.image) !== null && _a !== void 0 ? _a : yield this.executionRuntime.getScreenshot();
}
yield this.stepReporter.onStepEnd({
createdAt,
detectedElements: annotation === null || annotation === void 0 ? void 0 : annotation.detected_elements,
screenshot,
}, error);
});
}
annotate() {
return __awaiter(this, arguments, void 0, function* (annotationRequest = {}) {
const annotation = yield this.executionRuntime.annotateImage(annotationRequest.imagePath, annotationRequest.customElements, annotationRequest.elements);
annotation_writer_1.AnnotationWriter.write(annotation.toHtml(), annotationRequest.outputPath, annotationRequest.fileNamePrefix);
return annotation;
});
}
annotateInteractively() {
return __awaiter(this, void 0, void 0, function* () {
try {
yield this.executionRuntime.annotateInteractively();
}
catch (err) {
logger_1.logger.error(err);
}
});
}
// eslint-disable-next-line class-methods-use-this
escapeSeparatorString(instruction) {
return instruction.split(dsl_1.Separators.STRING).join('"');
}
buildInstruction(instructionString_1) {
return __awaiter(this, arguments, void 0, function* (instructionString, customElementJson = []) {
return {
customElements: yield custom_element_1.CustomElement.fromJsonListWithImagePathOrImage(customElementJson),
secretText: this.getAndResetSecretText(),
value: instructionString,
valueHumanReadable: this.escapeSeparatorString(instructionString),
};
});
}
getAIElementsByNames(names) {
return __awaiter(this, void 0, void 0, function* () {
if (names.length === 0) {
return [];
}
// eslint-disable-next-line max-len
const workspaceAIElementCollection = yield ai_element_collection_1.AIElementCollection.collectAIElements(this.workspaceId, this.aiElementArgs);
return workspaceAIElementCollection.getByNames(names);
});
}
fluentCommandExecutor(instructionString_1, modelComposition_1) {
return __awaiter(this, arguments, void 0, function* (instructionString, modelComposition, context = { customElementsJson: [], aiElementNames: [] }) {
const aiElements = yield this.getAIElementsByNames(context.aiElementNames);
const instruction = yield this.buildInstruction(instructionString, [
...context.customElementsJson,
...aiElements,
]);
logger_1.logger.debug(instruction);
try {
this.stepReporter.resetStep(instruction);
yield this.executionRuntime.executeInstruction(instruction, modelComposition);
yield this.afterCommandExecution(instruction);
return yield Promise.resolve();
}
catch (error) {
yield this.afterCommandExecution(instruction, error instanceof Error ? error : new Error(String(error)));
return Promise.reject(error);
}
});
}
getterExecutor(instruction_1) {
return __awaiter(this, arguments, void 0, function* (instruction, context = { customElementsJson: [], aiElementNames: [] }) {
const aiElements = yield this.getAIElementsByNames(context.aiElementNames);
const customElements = yield custom_element_1.CustomElement.fromJsonListWithImagePathOrImage(context.customElementsJson);
const stringWithoutSeparators = this.escapeSeparatorString(instruction);
logger_1.logger.debug(stringWithoutSeparators);
return this.executionRuntime.getDetectedElements(instruction, [
...customElements,
...aiElements,
]);
});
}
/**
* Takes a prompt that contains a question you want to be answered
* or the data you want to have extracted from your screen.
*
* The optional 'config' can be used to specifiy the JSON schema the
* returned object shall have (https://json-schema.org).
*
* See the following examples on how to use it:
*
* let isWidgetsNew =
* await aui.ask(
* "Does the sidebar element 'Widgets' have a 'NEW' tag?",
* {
* json_schema: {
* "type": "boolean"
* }
* });
*
* Output of console.log(isWidgetsNew): true
*
* let newClients =
* await aui.ask(
* "How many new clients?",
* {
* json_schema: {
* "type": "number"
* }
* });
*
* Output of console.log(newClients): 9123
*
* let userNames =
* await aui.ask(
* "Return a list with the users names.",
* {
* json_schema: {
* "type": "array",
* "items": {
* "type": "string"
* }
* }
* });
*
* Output of console.log(userNames):
* [
* 'Yiorgos Avraamu',
* 'Avram Tsarios',
* 'Quintin Ed',
* 'Enéas Kwadwo',
* 'Agapetus Tadeáš'
* ]
*
* let users =
* await aui.ask(
* "Extract the users from the table.",
* {
* json_schema: {
* "type": "array",
* "items": {
* "type": "object",
* "properties": {
* "name": {
* "type": "string"
* },
* "usage": {
* "type": "number"
* }
* },
* "additionalProperties": false,
* "required": ["name", "usage"]
* },
* },
* });
*
* Output of console.log(users):
* [
* { name: 'Yiorgos Avraamu', usage: 50 },
* { name: 'Avram Tarasios', usage: 10 },
* { name: 'Quintin Ed', usage: 74 },
* { name: 'Eneás Kwadwo', usage: 98 },
* { name: 'Agapetus Tadeáš', usage: 22 }
* ]
*
* @param {string} prompt - The question you want to be answered or
* the data you want to have extracted.
* @param {Object} config - object that specifies the return json: {json_schema: {...}}.
* @returns {any} - The answer as JSON specified in the config object.
*/
// eslint-disable-next-line @typescript-eslint/no-explicit-any
ask(prompt, config) {
return __awaiter(this, void 0, void 0, function* () {
return this.executionRuntime.predictVQA(prompt, config);
});
}
getAndResetSecretText() {
const { secretText } = this;
this.secretText = undefined;
return secretText;
}
/**
* Types a text inside the filtered element.
*
* By default, the `text` is included in the logs and sent over to the askui Inference server to
* predict in which context the typing has to occur. You can exclude the `text` from the logs
* and the request to the askui Inference server setting `options.isSecret` to `true`.
* This should not change the quality of the prediction of the askui Inference server. In this
* case, `options.secretMask` is included in logs and sent over instead of the `text`.
*
* @param {string} text - A text to type.
* @param {Object} [options]
* @param {boolean} [options.isSecret = false] - If set to `true`, `text` is neither included in
* logs of askui nor sent over to askui Inference for prediction.
* @param {string} [options.secretMask = '****'] - If `options.isSecret` is set to `true`, this
* is included in logs and sent over to askui Inference for prediction instead of the `text`.
*
* @return {FluentFilters}
*/
typeIn(text, { isSecret = false, secretMask = '****' } = {}) {
if (text.length === 0) {
throw new yup_1.ValidationError('Empty string is not allowed. Typing of an empty string was rejected.');
}
if (isSecret) {
this.secretText = text;
return super.typeIn(secretMask);
}
return super.typeIn(text);
}
/**
* Types a text at the current position.
*
* By default, the `text` is included in the logs and sent over to the askui Inference server to
* predict in which context the typing has to occur. You can exclude the `text` from the logs
* and the request to the askui Inference server setting `options.isSecret` to `true`.
* This should not change the quality of the prediction of the askui Inference server. In this
* case, `options.secretMask` is included in logs and sent over instead of the `text`.
*
* @param {string} text - A text to type.
* @param {Object} options
* @param {boolean} [options.isSecret = false] - If set to `true`, `text` is neither included in
* logs of askui nor sent over to askui Inference for prediction.
* @param {string} [options.secretMask = '****'] - If `options.isSecret` is set to `true`, this
* is included in logs and sent over to askui Inference for prediction instead of the `text`.
*
* @return {Exec}
*/
type(text, { isSecret = false, secretMask = '****' } = {}) {
if (text.length === 0) {
throw new yup_1.ValidationError('Empty string is not allowed. Typing of an empty string was rejected.');
}
if (isSecret) {
this.secretText = text;
return super.type(secretMask);
}
return super.type(text);
}
/**
* Waits for `<delayInMs>` ms, e.g., 1000 ms. The exact delay may be a little longer
* than `<delayInMs>` but never shorter than that.
*
* @param {number} delayInMs - The delay in ms to wait for.
*
* @return {Executable}
*/
// eslint-disable-next-line class-methods-use-this
waitFor(delayInMs) {
return {
exec: () => __awaiter(this, void 0, void 0, function* () {
const stepTitle = `Wait for ${delayInMs} ms`;
const instruction = yield this.buildInstruction(stepTitle, []);
yield this.beforeNoneInferenceCallCommandExecution(instruction);
yield new Promise((resolve) => { setTimeout(resolve, delayInMs); });
yield this.afterCommandExecution(instruction);
return Promise.resolve();
}),
};
}
/**
* Press a key multiple times. At least two times.
*
* @param {PC_AND_MODIFIER_KEY} key
*
* @param {number} times
*/
pressKeyNTimes(key_1) {
return __awaiter(this, arguments, void 0, function* (key, times = 2) {
/* eslint-disable no-await-in-loop */
for (let i = 0; i < times; i += 1) {
yield this.pressKey(key).exec();
}
});
}
/**
* Press an array of keys one after another.
*
* For example press the following keys: right, left, enter.
*
* pressKeys(['right', 'left', 'enter'])
*
* @param {PC_AND_MODIFIER_KEY[]} keys
*/
pressKeys(keys) {
return __awaiter(this, void 0, void 0, function* () {
/* eslint-disable no-await-in-loop */
for (let i = 0; i < keys.length; i += 1) {
yield this.pressKey(keys[i]).exec();
}
});
}
/**
* Searches for text elements and clicks them
* one after another when found.
*
* @param {string[]} texts - An array of texts to be searched.
*/
clickTexts(texts) {
return __awaiter(this, void 0, void 0, function* () {
/* eslint-disable no-await-in-loop */
for (let i = 0; i < texts.length; i += 1) {
yield this.click().text(texts[i]).exec();
}
});
}
/**
* Searches for an element of type textfield with a specific placeholder text.
* If found, clicks it.
*
* @param {string} placeholder - The textfields placeholder text.
*/
clickTextfield(placeholder) {
return __awaiter(this, void 0, void 0, function* () {
yield this.click().textfield().contains().text()
.withText(placeholder)
.exec();
});
}
/**
* Searches for an element of type textfield with a specific
* label nearest to it. If found, clicks it.
*
* @param {string} label - The textfields label.
*/
clickTextfieldNearestTo(label) {
return __awaiter(this, void 0, void 0, function* () {
yield this.click().textfield().nearestTo().text(label)
.exec();
});
}
/**
* Wait until an AskUICommand does not fail.
*
* Use it to wait for an element to appear like this:
*
* await waitUntil(
* aui.expect().text('Github').exists()
* );
*
* @param {Executable} AskUICommand - For example: aui.moveMouse(0, 0)
* @param {number} maxTry - Number of maximum retries
* @param {number} waitTime - Time in milliseconds
*/
waitUntil(AskUICommand_1) {
return __awaiter(this, arguments, void 0, function* (AskUICommand, maxTry = 5, waitTime = 2000) {
const userDefinedStrategy = this.executionRuntime.retryStrategy;
try {
this.executionRuntime.retryStrategy = new retry_strategies_1.NoRetryStrategy();
yield AskUICommand.exec();
this.executionRuntime.retryStrategy = userDefinedStrategy;
}
catch (error) {
if (maxTry === 0) {
throw error;
}
yield this.waitFor(waitTime).exec();
yield this.waitUntil(AskUICommand, maxTry - 1, waitTime);
}
});
}
// eslint-disable-next-line class-methods-use-this
evaluateRelation(command, relation, text) {
switch (relation) {
case 'leftOf':
return command.leftOf().text(text);
case 'above':
return command.above().text(text);
case 'rightOf':
return command.rightOf().text(text);
case 'below':
return command.below().text(text);
case 'contains':
return command.contains().text(text);
case 'nearestTo':
return command.nearestTo().text(text);
default:
throw new yup_1.ValidationError(`'relation' has to be 'nearestTo', 'leftOf', 'above', 'rightOf', 'below' or 'contains' but was '${relation}'`);
}
}
/**
* Click a button with a specific label.
* Optional relation identifies the button in relation to another element.
*
* **Examples:**
* ```typescript
* await aui.clickButton({})
* await aui.clickButton({label: 'Checkout here'})
* await aui.clickButton({relation: {type: 'leftOf', text: 'Choose a ticket'}})
* await aui.clickButton({label: 'Click', {relation: {type: 'leftOf', text: 'Choose a ticket'}})
* ```
*
* @param {Object} params - Object containing properties.
* @property {string} [params.label] - The text label of the button. Defaults to an empty string.
* @property {Object} [params.relation] - Object describing the relationship between
* the clicked button and another element.
* @property {RelationsForConvenienceMethods} params.relation.type - The type of relation.
* @property {string} params.relation.text - The text element the relation is based on.
*/
clickButton(params) {
return __awaiter(this, void 0, void 0, function* () {
let command = this.click().button();
if (params.label) {
command = command.withText(params.label);
}
if (params.relation) {
command = this.evaluateRelation(command, params.relation.type, params.relation.text);
}
yield command.exec();
});
}
/**
* Click a checkbox with a specific label.
* You can also specify where the label is placed relationally.
*
* **Examples:**
* ```typescript
* await aui.clickCheckbox({label: 'Toggle'})
* await aui.clickCheckbox({label: 'Toggle', relation: {type: 'leftOf'}})
* ```
*
* @param {Object} params - Object containing required `label` property and
* optional `relation` property.
* @property {string} params.label - The label for the checkbox.
* @property {Object} [params.relation] - Object describing the relationship between
* the clicked checkbox and another element.
* @property {RelationsForConvenienceMethods} params.relation.type - The type of relation.
*/
clickCheckbox(params) {
return __awaiter(this, void 0, void 0, function* () {
let command = this.click().checkbox();
if (!params.relation) {
command = command.nearestTo().text(params.label);
}
else {
command = this.evaluateRelation(command, params.relation.type, params.label);
}
yield command.exec();
});
}
/**
* Click a switch with a specific label.
* You can also specify where the label is placed relationally.
*
* **Examples:**
* ```typescript
* await aui.clickSwitch({label: 'Toggle'})
* await aui.clickSwitch({label: 'Toggle', relation: {type: 'leftOf'}})
* ```
*
* @param {Object} params - Object containing required `label` property and
* optional `relation` property.
* @property {string} params.label - The label for the checkbox.
* @property {Object} [params.relation] - Object describing the relationship between
* the clicked checkbox and another element.
* @property {RelationsForConvenienceMethods} params.relation.type - The type of relation.
*/
clickSwitch(params) {
return __awaiter(this, void 0, void 0, function* () {
let command = this.click().switch();
if (!params.relation) {
command = command.nearestTo().text(params.label);
}
else {
command = this.evaluateRelation(command, params.relation.type, params.label);
}
yield command.exec();
});
}
/**
* Types a given text into a textfield.
* Use a relation to specify how to find
* the textfield in relation to a specific label.
*
* **Examples:**
* ```typescript
* // Finds the textfield nearest to the label 'Email'
* await aui.typeIntoTextfield({textToWrite: 'Hello World', relation: {label: 'Email'}});
*
* // Finds the textfield above/below a label 'Password'
* await aui.typeIntoTextfield(
* {textToWrite: 'Hello World', relation: {type: 'above', label: 'Password'}}
* );
* await aui.typeIntoTextfield(
* {textToWrite: 'Hello World', relation: {type: 'below', label: 'Password'}}
* );
*
* // If there is no label but a placeholder, the label is contained in the textfield
* await aui.typeIntoTextfield(
* {textToWrite: 'Hello World', relation: {type: 'contains', label: 'Enter email'}}
* );
* ```
*
* @param {Object} params - Object containing required `textToWrite` property and
* optional `relation` property.
* @property {string} params.textToWrite - The text to be typed into the textfield.
* @property {Object} params.relation - Object describing the relationship between the
* textfield being interacted with and another element.
* @property {RelationsForConvenienceMethods} params.relation.type - The type of
* relation, optional.
* @property {string} params.relation.label - The label associated with the related
* element, optional.
*/
typeIntoTextfield(params) {
return __awaiter(this, void 0, void 0, function* () {
let command = this.typeIn(params.textToWrite).textfield();
if (!params.relation.type) {
command = command.nearestTo().text(params.relation.label);
}
else {
command = this.evaluateRelation(command, params.relation.type, params.relation.label);
}
yield command.exec();
});
}
/**
* Click on a specific text.
* You can also use a RegEx or match the text exactly by specifying the specific flag.
* Use a relation to find the text in relation to a specific text.
*
* **Examples:**
* ```typescript
* // Click text that matches exactly
* await aui.clickText({text: 'askui', matching: 'similar'})
*
* // Click text that contains 'pie' or 'cake' or 'Pie' or 'Cake'
* await aui.clickText({text: '.*([Pp]ie|[Cc]ake).*', matching: 'regex'})
*
* // Click the text 'TERMINAL' that is left of the text 'Ports'
* await aui.clickText({
* text: 'TERMINAL',
* matching: "exact",
* relation: { type: 'leftOf', text: 'PORTS' }
* })
* ```
*
* @param {Object} params - Object containing required `text` property and optional properties
* for regular expression matching and relation.
* @property {string} params.text - The text to be clicked.
* @property {string} params.matching - Whether the text is matched using similarity,
* exact match or a regular expression.
* @property {Object} [params.relation] - Object describing the relationship between the
* clicked text and another element.
* @property {RelationsForConvenienceMethods} params.relation.type - The type of relation.
* @property {string} params.relation.text - The label or text associated with the
* related element or state.
*/
clickText(params) {
return __awaiter(this, void 0, void 0, function* () {
let command = this.click().text();
command = this.evaluateMatchingProperty(command, { value: params.text, matching: params.matching });
if (params.relation) {
command = this.evaluateRelation(command, params.relation.type, params.relation.text);
}
yield command.exec();
});
}
// eslint-disable-next-line class-methods-use-this
evaluateMatchingProperty(command, text) {
var _a;
switch ((_a = text.matching) !== null && _a !== void 0 ? _a : 'similar') {
case 'exact':
return command.withExactText(text.value);
case 'regex':
return command.withTextRegex(text.value);
case 'similar':
return command.withText(text.value);
default:
throw new yup_1.ValidationError(`'text.matching' property has to be 'similar', 'exact' or 'regex' but was '${text.matching}'`);
}
}
/**
* Check if one or multiple elements are detected.
*
* **Examples:**
* ```typescript
* await aui.expectAllExist([
* {
* type: 'text',
* text: {
* value: 'Switch to Dark',
* matching: 'similar'
* }
* },
* ]);
*
* // Check for existence of multiple elements
* await aui.expectAllExist([
* {
* type: 'textfield',
* relation: {
* type: 'rightOf',
* text: 'Email:'
* }
* },
* {
* type: 'element',
* text: {
* value: 'Switch to Dark'
* }
* },
* ]);
*
* // Validate existence
* const exists = await aui.expectAllExist([...]);
* exists.allExist // true when every element exists
*
* // Check which elements do not exist
* // with the elements property
* const nonExistentElements = exists.elements.filter((e) => e.exists===false)
* ```
*
* @param {ElementExistsQuery[]} query - Objects containing the required property
* 'type' and the optional properties
* 'text' and 'relation'.
* @property {string} query.type - The type of the element: 'otherElement' | 'switch' |
* 'element' | 'container' | 'checkbox' | 'element' |
* 'button' | 'table' | 'text' | 'icon' | 'image' | 'textfield'
* @property {Object} [query.text] - Object containing value and matching strategy.
* @property {string} query.text.value - The text to match for.
* @property {string} [query.text.matching] - Whether the text is matched using similarity,
* exact match or a regular expression.
* @property {Object} [query.relation] - Object describing the relationship between the
* clicked text and another element.
* @property {RelationsForConvenienceMethods} query.relation.type - The type of relation.
* @property {string} query.relation.text - The label or text associated with the
* related element or state.
* @returns {ExpectAllExistResult.allExist} - If every element exists.
* @returns {ExpectAllExistResult.elements} - ExpectExistenceElement[].
*/
expectAllExist(query) {
return __awaiter(this, void 0, void 0, function* () {
const elements = yield query.reduce((accumulatorPromise, subquery) => __awaiter(this, void 0, void 0, function* () {
const acc = yield accumulatorPromise;
const command = this.get()[subquery.type]();
let finalCommand = subquery.text !== undefined
? this.evaluateMatchingProperty(command, subquery.text)
: command;
if (subquery.relation) {
finalCommand = this.evaluateRelation(finalCommand, subquery.relation.type, subquery.relation.text);
}
return [
...acc,
Object.assign(Object.assign({}, subquery), { exists: (yield finalCommand.exec()).length > 0 }),
];
}), Promise.resolve([]));
return {
elements,
allExist: elements.every((el) => el.exists),
};
});
}
/**
* Holds down a key on the keyboard.
*
* **Examples:**
* ```typescript
* await aui.keyDown('a').exec();
* ```
*
* @param {PC_AND_MODIFIER_KEY} key - The key to hold down.
*/
keyDown(key) {
return {
exec: () => __awaiter(this, void 0, void 0, function* () {
const stepTitle = `Hold down key ${key}`;
const instruction = yield this.buildInstruction(stepTitle, []);
try {
yield this.beforeNoneInferenceCallCommandExecution(instruction);
yield this.agent.getOsAgentHandler().desktopKeyHoldDown(key, []);
yield this.afterCommandExecution(instruction);
}
catch (error) {
yield this.afterCommandExecution(instruction, error instanceof Error ? error : new Error(String(error)));
return Promise.reject(error);
}
return Promise.resolve();
}),
};
}
/**
* Releases a key up that was previously held down.
*
* **Examples:**
* ```typescript
* await aui.keyUp('a').exec();
* ```
*
* @param {PC_AND_MODIFIER_KEY} key - The key to release up.
*/
keyUp(key) {
return {
exec: () => __awaiter(this, void 0, void 0, function* () {
const stepTitle = `Release key ${key}`;
const instruction = yield this.buildInstruction(stepTitle, []);
try {
yield this.beforeNoneInferenceCallCommandExecution(instruction);
yield this.agent.getOsAgentHandler().desktopKeyRelease(key, []);
yield this.afterCommandExecution(instruction);
}
catch (error) {
yield this.afterCommandExecution(instruction, error instanceof Error ? error : new Error(String(error)));
return Promise.reject(error);
}
return Promise.resolve();
}),
};
}
act(goal, imageOrOptions, options) {
return __awaiter(this, void 0, void 0, function* () {
if (typeof imageOrOptions === 'string') {
return this.agent.act(goal, imageOrOptions, options);
}
const fullTitle = `Act: ${goal}`;
const stepTitle = fullTitle.length > 50 ? `${fullTitle.substring(0, 47)}...` : fullTitle;
const instruction = yield this.buildInstruction(stepTitle, []);
try {
yield this.beforeNoneInferenceCallCommandExecution(instruction);
const result = yield this.agent.act(goal, undefined, imageOrOptions);
yield this.afterCommandExecution(instruction);
return result;
}
catch (error) {
yield this.afterCommandExecution(instruction, error instanceof Error ? error : new Error(String(error)));
return Promise.reject(error);
}
});
}
/**
* Adds tools to the agent that allow it to interact with AI elements.
*
* @returns {Promise<void>} - A promise that resolves when the tools are added to the agent.
*/
addAIElementsToolsToAgent() {
return __awaiter(this, void 0, void 0, function* () {
const aiElementLocator = (aiElementName) => this.get().aiElement(aiElementName).exec();
const askUIGetAskUIElementTool = new askui_api_tools_1.AskUIGetAskUIElementTool(this.agent.getOsAgentHandler(), aiElementLocator, 'aiElement');
this.agent.addTool(askUIGetAskUIElementTool);
const listAIElementNamesFunction = () => (ai_element_collection_1.AIElementCollection.collectAIElements(this.workspaceId, this.aiElementArgs)).then((aiElementCollection) => aiElementCollection.getNames());
const askUIListAIElementTool = new askui_api_tools_1.AskUIListAIElementTool(listAIElementNamesFunction);
this.agent.addTool(askUIListAIElementTool);
});
}
/**
* Retrieves the starting arguments used when the controller server was initialized.
*
* Useful for debugging, logging, or verifying the current server configuration.
*
* @property {string} displayNum - Display number controlled by the controller
* @property {boolean} minimize - Whether controller starts minimized
* @property {string} runtime - Runtime type ("desktop" or "android")
* @property {number} port - Communication port
* @property {number} actionWaitTime - Action wait time
* @property {string} host - Host address
* @property {string} logFile - Log file path
* @property {boolean} hideOverlay - Whether overlay is hidden
* @property {boolean} debugDraw - Whether debug drawing is enabled
* @property {string} deviceId - Android device ID
* @property {string} configFile - Configuration file path
* @property {string} logLevel - Logging level
*
* @example
* ```typescript
* const startingArguments = await aui.getControllerStartingArguments();
* console.log(startingArguments);
* // Output example:
* // {
* // displayNum: 0,
* // minimize: true,
* // runtime: 'desktop',
* // port: 5000,
* // actionWaitTime: 1000,
* // host: '127.0.0.1',
* // logFile: '/tmp/askui/askui-server.log',
* // hideOverlay: false,
* // debugDraw: false,
* // deviceId: 'emulator-5554',
* // configFile: '/tmp/askui/askui-config.json',
* // logLevel: 'info',
* // }
* ```
*
* @example Retrieving Android device ID:
* ```typescript
* const startingArguments = await aui.getControllerStartingArguments();
* console.log(startingArguments.deviceId);
* // Output example: "emulator-5554"
* ```
*/
getControllerStartingArguments() {
return __awaiter(this, void 0, void 0, function* () {
return this.executionRuntime.getStartingArguments();
});
}
}
exports.UiControlClient = UiControlClient;