askui
Version:
Reliable, automated end-to-end-testing that depends on what is shown on your screen instead of the technology you are running on
227 lines (226 loc) • 11 kB
JavaScript
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
import { ControlCommandCode } from '../core/ui-control-commands';
import { CustomElement } from '../core/model/custom-element';
import { RepeatError } from './repeat-error';
import { delay } from './misc';
import { ControlCommandError } from './control-command-error';
import { Annotation } from '../core/annotation/annotation';
import { logger } from '../lib/logger';
import { Base64Image } from '../utils/base_64_image/base-64-image';
export class ExecutionRuntime {
constructor(uiControllerClient, inferenceClient, stepReporter, retryStrategy) {
this.uiControllerClient = uiControllerClient;
this.inferenceClient = inferenceClient;
this.stepReporter = stepReporter;
this.retryStrategy = retryStrategy;
this.EXEC_REPETITION_COUNT = 25;
}
connect() {
return __awaiter(this, void 0, void 0, function* () {
return this.uiControllerClient.connect();
});
}
disconnect() {
this.uiControllerClient.disconnect();
}
startVideoRecording() {
return __awaiter(this, void 0, void 0, function* () {
yield this.uiControllerClient.startVideoRecording();
});
}
stopVideoRecording() {
return __awaiter(this, void 0, void 0, function* () {
yield this.uiControllerClient.stopVideoRecording();
});
}
readVideoRecording() {
return __awaiter(this, void 0, void 0, function* () {
const response = yield this.uiControllerClient.readVideoRecording();
return response.data.video;
});
}
requestControl(controlCommand) {
return __awaiter(this, void 0, void 0, function* () {
yield this.uiControllerClient.requestControl(controlCommand);
});
}
executeInstruction(instruction, modelComposition) {
return __awaiter(this, void 0, void 0, function* () {
var _a, _b, _c;
const controlCommand = yield this.predictCommandWithRetry(instruction, modelComposition);
if (controlCommand.code === ControlCommandCode.OK) {
return this.requestControl(controlCommand);
}
if (controlCommand.tryToRepeat) {
yield this.requestControl(controlCommand);
return this.executeCommandRepeatedly(instruction, modelComposition);
}
throw new ControlCommandError((_c = (_b = (_a = controlCommand.actions) === null || _a === void 0 ? void 0 : _a[0]) === null || _b === void 0 ? void 0 : _b.text) !== null && _c !== void 0 ? _c : '');
});
}
executeCommandRepeatedly(instruction, modelComposition) {
return __awaiter(this, void 0, void 0, function* () {
var _a, _b, _c;
/* eslint-disable no-await-in-loop */
for (let repeatCount = this.EXEC_REPETITION_COUNT; repeatCount >= 0; repeatCount -= 1) {
if (repeatCount === 0) {
throw new RepeatError(`Max. number (${this.EXEC_REPETITION_COUNT}) of repetitions of command executions `
+ 'from a single test step reached');
}
logger.debug('Repeat command execution....');
const controlCommand = yield this.predictCommandWithRetry(instruction, modelComposition);
if (controlCommand.code === ControlCommandCode.OK) {
break;
}
if (controlCommand.tryToRepeat) {
yield this.requestControl(controlCommand);
}
else {
throw new ControlCommandError((_c = (_b = (_a = controlCommand.actions) === null || _a === void 0 ? void 0 : _a[0]) === null || _b === void 0 ? void 0 : _b.text) !== null && _c !== void 0 ? _c : '');
}
}
/* eslint-enable no-await-in-loop */
});
}
/**
* Command prediction may fail, e.g., due to application still loading
* --> retry with linear back-off
*/
/* eslint-disable-next-line consistent-return */
predictCommandWithRetry(instruction, modelComposition) {
return __awaiter(this, void 0, void 0, function* () {
var _a, _b, _c;
let command = yield this.predictCommand(instruction, modelComposition);
/* eslint-disable no-await-in-loop */
for (let k = 0; k < this.retryStrategy.retryCount; k += 1) {
if (command.code === ControlCommandCode.OK) {
return command;
}
const msUntilRetry = this.retryStrategy.getDelay(k + 1);
logger.debug(`Wait ${msUntilRetry} and retry predicting command...`);
yield delay(msUntilRetry);
command = yield this.predictCommand(instruction, modelComposition, new ControlCommandError((_c = (_b = (_a = command.actions) === null || _a === void 0 ? void 0 : _a[0]) === null || _b === void 0 ? void 0 : _b.text) !== null && _c !== void 0 ? _c : ''));
}
/* eslint-enable no-await-in-loop */
return command;
});
}
isImageRequiredByConfig() {
return this.stepReporter.config.withScreenshots === 'begin'
|| this.stepReporter.config.withScreenshots === 'always';
}
isImageRequired(instruction) {
return __awaiter(this, void 0, void 0, function* () {
if (this.isImageRequiredByConfig())
return Promise.resolve(true);
return this.inferenceClient.isImageRequired(instruction);
});
}
isAnnotationRequired() {
return this.stepReporter.config.withDetectedElements === 'begin'
|| this.stepReporter.config.withDetectedElements === 'always';
}
getScreenshot() {
return __awaiter(this, void 0, void 0, function* () {
const requestScreenshotResponse = yield this.uiControllerClient.requestScreenshot();
return requestScreenshotResponse.data.image;
});
}
buildSnapshot(instruction) {
return __awaiter(this, void 0, void 0, function* () {
const createdAt = new Date();
const screenshot = (yield this.isImageRequired(instruction))
? yield this.getScreenshot() : undefined;
const annotation = this.isAnnotationRequired() ? yield this.annotateImage() : undefined;
return {
createdAt,
detectedElements: annotation === null || annotation === void 0 ? void 0 : annotation.detected_elements,
screenshot,
};
});
}
predictCommand(instruction, modelComposition, retryError) {
return __awaiter(this, void 0, void 0, function* () {
const snapshot = yield this.buildSnapshot(instruction.value);
if (retryError !== undefined)
this.stepReporter.onStepRetry(snapshot, retryError);
else
this.stepReporter.onStepBegin(snapshot);
const controlCommand = yield this.inferenceClient.predictControlCommand(instruction.value, modelComposition, instruction.customElements, snapshot.screenshot);
if (instruction.secretText !== undefined) {
controlCommand.setTextToBeTyped(instruction.secretText);
}
return controlCommand;
});
}
annotateInteractively() {
return __awaiter(this, void 0, void 0, function* () {
const annotationResponse = yield this.annotateImage();
yield this.uiControllerClient.annotateInteractively(annotationResponse.detected_elements, annotationResponse.image);
});
}
takeScreenshotIfImageisNotProvided(imagePath) {
return __awaiter(this, void 0, void 0, function* () {
let base64Image = '';
if (imagePath !== undefined) {
base64Image = (yield Base64Image.fromPath(imagePath)).toString();
}
if (imagePath === undefined) {
const screenshotResponse = yield this.uiControllerClient.requestScreenshot();
base64Image = screenshotResponse.data.image;
}
return base64Image;
});
}
getStartingArguments() {
return __awaiter(this, void 0, void 0, function* () {
const startingArgumentsResponse = yield this.uiControllerClient.getStartingArguments();
return startingArgumentsResponse.data.arguments;
});
}
getDetectedElements(instruction, customElementJson) {
return __awaiter(this, void 0, void 0, function* () {
let customElements = [];
const base64Image = yield this.takeScreenshotIfImageisNotProvided();
if (customElementJson !== undefined) {
customElements = yield CustomElement.fromJsonListWithImagePathOrImage(customElementJson);
}
return this.inferenceClient.getDetectedElements(instruction, base64Image, customElements);
});
}
annotateImage(imagePath, customElementJson, elements) {
return __awaiter(this, void 0, void 0, function* () {
let customElements = [];
const base64Image = yield this.takeScreenshotIfImageisNotProvided(imagePath);
if (elements !== undefined) {
if (elements.length === 0) {
logger.warn("Parameter 'elements' is an empty list.");
}
return new Annotation(base64Image, elements);
}
if (customElementJson !== undefined) {
customElements = yield CustomElement.fromJsonListWithImagePathOrImage(customElementJson);
}
return this.inferenceClient.predictImageAnnotation(base64Image, customElements);
});
}
predictVQA(prompt, config) {
return __awaiter(this, void 0, void 0, function* () {
const base64Image = yield this.takeScreenshotIfImageisNotProvided();
return this.inferenceClient.predictVQAAnswer(prompt, base64Image, config);
});
}
predictActResponse(params) {
return __awaiter(this, void 0, void 0, function* () {
return this.inferenceClient.predictActResponse(params);
});
}
}