askui
Version:
Reliable, automated end-to-end-testing that depends on what is shown on your screen instead of the technology you are running on
82 lines (81 loc) • 4.13 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.AskUIListAIElementTool = exports.AskUIGetAskUIElementTool = void 0;
const base_1 = require("./base");
class AskUIGetAskUIElementTool extends base_1.BaseAgentTool {
constructor(osAgentHandler, locatorFunction, elementType) {
super();
this.osAgentHandler = osAgentHandler;
this.locatorFunction = locatorFunction;
this.elementType = elementType;
}
execute(params) {
return __awaiter(this, void 0, void 0, function* () {
const { elementName } = params;
const detectedElements = yield this.locatorFunction(elementName);
const scaledElementsBoundingBoxes = detectedElements.map((element) => {
const xMid = (element.bndbox.xmin + element.bndbox.xmax) / 2;
const yMid = (element.bndbox.ymin + element.bndbox.ymax) / 2;
const [x, y] = this.osAgentHandler.scaleCoordinates('computer', xMid, yMid);
return {
x, y,
};
});
return {
output: `Found ${scaledElementsBoundingBoxes.length} elements of type ${this.elementType}. center coordinates: ${JSON.stringify(scaledElementsBoundingBoxes)}`,
};
});
}
toParams() {
return {
description: `Locates and retrieves the bounding box coordinates of AskUI ${this.elementType} elements on the screen. ` +
`This tool is essential for UI automation as it provides the exact pixel coordinates needed to interact with UI elements. ` +
`The coordinates returned can be used for clicking, hovering, or other mouse interactions. ` +
`Use this tool when you need to find and interact with specific ${this.elementType} UI elements by their semantic names.`,
input_schema: {
properties: {
elementName: {
type: 'string',
description: `The semantic name or identifier of the ${this.elementType} element to locate on the screen. `
},
},
required: ['elementName'],
type: 'object',
},
name: `get_askui_${this.elementType}_element_tool`,
};
}
}
exports.AskUIGetAskUIElementTool = AskUIGetAskUIElementTool;
class AskUIListAIElementTool extends base_1.BaseAgentTool {
constructor(listFunction) {
super();
this.listFunction = listFunction;
}
execute() {
return __awaiter(this, void 0, void 0, function* () {
const elementNames = yield this.listFunction();
return {
output: `Found ${elementNames.length} element names that can be used to retrieve bounding boxes. Names: ${JSON.stringify(elementNames)}`,
};
});
}
toParams() {
return {
description: 'Retrieves a comprehensive list of all valid AskUI AI element names that can be used for element location and interaction. ' +
'The returned names can be used as input for the get_askui_aiElement_element_tool to locate specific ai elements. ',
input_schema: { type: 'object', properties: {}, required: [] },
name: 'list_ai_element_names_tool',
};
}
}
exports.AskUIListAIElementTool = AskUIListAIElementTool;