askui
Version:
Reliable, automated end-to-end-testing that depends on what is shown on your screen instead of the technology you are running on
477 lines (476 loc) • 19.4 kB
TypeScript
import { Exec, Executable, FluentFilters, ApiCommands, PC_AND_MODIFIER_KEY, CommandExecutorContext } from './dsl';
import { UiControllerClientConnectionState } from './ui-controller-client-connection-state';
import { Annotation } from '../core/annotation/annotation';
import { AnnotationRequest } from '../core/model/annotation-result/annotation-interface';
import { DetectedElement } from '../core/model/annotation-result/detected-element';
import { ClientArgs } from './ui-controller-client-interface';
import { ModelCompositionBranch } from './model-composition-branch';
export type RelationsForConvenienceMethods = 'nearestTo' | 'leftOf' | 'above' | 'rightOf' | 'below' | 'contains';
export type TextMatchingOption = 'similar' | 'exact' | 'regex';
export type ElementExistsQueryType = 'otherElement' | 'switch' | 'element' | 'container' | 'checkbox' | 'element' | 'button' | 'table' | 'text' | 'icon' | 'image' | 'textfield';
export interface ElementExistsQueryText {
value: string;
matching?: TextMatchingOption;
}
export interface ElementExistsQueryRelation {
type: RelationsForConvenienceMethods;
text: string;
}
export interface ElementExistsQuery {
type: keyof Pick<FluentFilters, ElementExistsQueryType>;
text?: ElementExistsQueryText;
relation?: ElementExistsQueryRelation;
}
export interface ExpectExistenceElement extends ElementExistsQuery {
exists: boolean;
}
export interface ExpectAllExistResult {
allExist: boolean;
elements: ExpectExistenceElement[];
}
export declare class UiControlClient extends ApiCommands {
private workspaceId;
private executionRuntime;
private stepReporter;
private aiElementArgs;
private constructor();
static build(clientArgs?: ClientArgs): Promise<UiControlClient>;
/**
* Connects to the askui UI Controller.
*/
connect(): Promise<UiControllerClientConnectionState>;
/**
* Disconnects from the askui UI Controller.
*/
disconnect(): void;
/**
* Disconnects from the askui UI Controller.
*
* @deprecated Use {@link disconnect} instead.
*/
close(): void;
startVideoRecording(): Promise<void>;
stopVideoRecording(): Promise<void>;
readVideoRecording(): Promise<string>;
private shouldAnnotateAfterCommandExecution;
private afterCommandExecution;
annotate(annotationRequest?: AnnotationRequest): Promise<Annotation>;
annotateInteractively(): Promise<void>;
private escapeSeparatorString;
private buildInstruction;
private getAIElementsByNames;
fluentCommandExecutor(instructionString: string, modelComposition: ModelCompositionBranch[], context?: CommandExecutorContext): Promise<void>;
getterExecutor(instruction: string, context?: CommandExecutorContext): Promise<DetectedElement[]>;
/**
* Takes a prompt that contains a question you want to be answered
* or the data you want to have extracted from your screen.
*
* The optional 'config' can be used to specifiy the JSON schema the
* returned object shall have (https://json-schema.org).
*
* See the following examples on how to use it:
*
* let isWidgetsNew =
* await aui.ask(
* "Does the sidebar element 'Widgets' have a 'NEW' tag?",
* {
* json_schema: {
* "type": "boolean"
* }
* });
*
* Output of console.log(isWidgetsNew): true
*
* let newClients =
* await aui.ask(
* "How many new clients?",
* {
* json_schema: {
* "type": "number"
* }
* });
*
* Output of console.log(newClients): 9123
*
* let userNames =
* await aui.ask(
* "Return a list with the users names.",
* {
* json_schema: {
* "type": "array",
* "items": {
* "type": "string"
* }
* }
* });
*
* Output of console.log(userNames):
* [
* 'Yiorgos Avraamu',
* 'Avram Tsarios',
* 'Quintin Ed',
* 'Enéas Kwadwo',
* 'Agapetus Tadeáš'
* ]
*
* let users =
* await aui.ask(
* "Extract the users from the table.",
* {
* json_schema: {
* "type": "array",
* "items": {
* "type": "object",
* "properties": {
* "name": {
* "type": "string"
* },
* "usage": {
* "type": "number"
* }
* },
* "additionalProperties": false,
* "required": ["name", "usage"]
* },
* },
* });
*
* Output of console.log(users):
* [
* { name: 'Yiorgos Avraamu', usage: 50 },
* { name: 'Avram Tarasios', usage: 10 },
* { name: 'Quintin Ed', usage: 74 },
* { name: 'Eneás Kwadwo', usage: 98 },
* { name: 'Agapetus Tadeáš', usage: 22 }
* ]
*
* @param {string} prompt - The question you want to be answered or
* the data you want to have extracted.
* @param {Object} config - object that specifies the return json: {json_schema: {...}}.
* @returns {any} - The answer as JSON specified in the config object.
*/
ask(prompt: string, config?: object): Promise<any>;
private secretText;
private getAndResetSecretText;
/**
* Types a text inside the filtered element.
*
* By default, the `text` is included in the logs and sent over to the askui Inference server to
* predict in which context the typing has to occur. You can exclude the `text` from the logs
* and the request to the askui Inference server setting `options.isSecret` to `true`.
* This should not change the quality of the prediction of the askui Inference server. In this
* case, `options.secretMask` is included in logs and sent over instead of the `text`.
*
* @param {string} text - A text to type.
* @param {Object} [options]
* @param {boolean} [options.isSecret = false] - If set to `true`, `text` is neither included in
* logs of askui nor sent over to askui Inference for prediction.
* @param {string} [options.secretMask = '****'] - If `options.isSecret` is set to `true`, this
* is included in logs and sent over to askui Inference for prediction instead of the `text`.
*
* @return {FluentFilters}
*/
typeIn(text: string, { isSecret, secretMask }?: {
isSecret?: boolean;
secretMask?: string;
}): FluentFilters;
/**
* Types a text at the current position.
*
* By default, the `text` is included in the logs and sent over to the askui Inference server to
* predict in which context the typing has to occur. You can exclude the `text` from the logs
* and the request to the askui Inference server setting `options.isSecret` to `true`.
* This should not change the quality of the prediction of the askui Inference server. In this
* case, `options.secretMask` is included in logs and sent over instead of the `text`.
*
* @param {string} text - A text to type.
* @param {Object} options
* @param {boolean} [options.isSecret = false] - If set to `true`, `text` is neither included in
* logs of askui nor sent over to askui Inference for prediction.
* @param {string} [options.secretMask = '****'] - If `options.isSecret` is set to `true`, this
* is included in logs and sent over to askui Inference for prediction instead of the `text`.
*
* @return {Exec}
*/
type(text: string, { isSecret, secretMask }?: {
isSecret?: boolean;
secretMask?: string;
}): Exec;
/**
* Waits for `<delayInMs>` ms, e.g., 1000 ms. The exact delay may be a little longer
* than `<delayInMs>` but never shorter than that.
*
* @param {number} delayInMs - The delay in ms to wait for.
*
* @return {Executable}
*/
waitFor(delayInMs: number): Executable;
/**
* Press a key multiple times. At least two times.
*
* @param {PC_AND_MODIFIER_KEY} key
*
* @param {number} times
*/
pressKeyNTimes(key: PC_AND_MODIFIER_KEY, times?: number): Promise<void>;
/**
* Press an array of keys one after another.
*
* For example press the following keys: right, left, enter.
*
* pressKeys(['right', 'left', 'enter'])
*
* @param {PC_AND_MODIFIER_KEY[]} keys
*/
pressKeys(keys: PC_AND_MODIFIER_KEY[]): Promise<void>;
/**
* Searches for text elements and clicks them
* one after another when found.
*
* @param {string[]} texts - An array of texts to be searched.
*/
clickTexts(texts: string[]): Promise<void>;
/**
* Searches for an element of type textfield with a specific placeholder text.
* If found, clicks it.
*
* @param {string} placeholder - The textfields placeholder text.
*/
clickTextfield(placeholder: string): Promise<void>;
/**
* Searches for an element of type textfield with a specific
* label nearest to it. If found, clicks it.
*
* @param {string} label - The textfields label.
*/
clickTextfieldNearestTo(label: string): Promise<void>;
/**
* Wait until an AskUICommand does not fail.
*
* Use it to wait for an element to appear like this:
*
* await waitUntil(
* aui.expect().text('Github').exists()
* );
*
* @param {Executable} AskUICommand - For example: aui.moveMouse(0, 0)
* @param {number} maxTry - Number of maximum retries
* @param {number} waitTime - Time in milliseconds
*/
waitUntil(AskUICommand: Executable, maxTry?: number, waitTime?: number): Promise<void>;
private evaluateRelation;
/**
* Click a button with a specific label.
* Optional relation identifies the button in relation to another element.
*
* **Examples:**
* ```typescript
* await aui.clickButton({})
* await aui.clickButton({label: 'Checkout here'})
* await aui.clickButton({relation: {type: 'leftOf', text: 'Choose a ticket'}})
* await aui.clickButton({label: 'Click', {relation: {type: 'leftOf', text: 'Choose a ticket'}})
* ```
*
* @param {Object} params - Object containing properties.
* @property {string} [params.label] - The text label of the button. Defaults to an empty string.
* @property {Object} [params.relation] - Object describing the relationship between
* the clicked button and another element.
* @property {RelationsForConvenienceMethods} params.relation.type - The type of relation.
* @property {string} params.relation.text - The text element the relation is based on.
*/
clickButton(params: {
label?: string;
relation?: {
type: RelationsForConvenienceMethods;
text: string;
};
}): Promise<void>;
/**
* Click a checkbox with a specific label.
* You can also specify where the label is placed relationally.
*
* **Examples:**
* ```typescript
* await aui.clickCheckbox({label: 'Toggle'})
* await aui.clickCheckbox({label: 'Toggle', relation: {type: 'leftOf'}})
* ```
*
* @param {Object} params - Object containing required `label` property and
* optional `relation` property.
* @property {string} params.label - The label for the checkbox.
* @property {Object} [params.relation] - Object describing the relationship between
* the clicked checkbox and another element.
* @property {RelationsForConvenienceMethods} params.relation.type - The type of relation.
*/
clickCheckbox(params: {
label: string;
relation?: {
type: RelationsForConvenienceMethods;
};
}): Promise<void>;
/**
* Click a switch with a specific label.
* You can also specify where the label is placed relationally.
*
* **Examples:**
* ```typescript
* await aui.clickSwitch({label: 'Toggle'})
* await aui.clickSwitch({label: 'Toggle', relation: {type: 'leftOf'}})
* ```
*
* @param {Object} params - Object containing required `label` property and
* optional `relation` property.
* @property {string} params.label - The label for the checkbox.
* @property {Object} [params.relation] - Object describing the relationship between
* the clicked checkbox and another element.
* @property {RelationsForConvenienceMethods} params.relation.type - The type of relation.
*/
clickSwitch(params: {
label: string;
relation?: {
type: RelationsForConvenienceMethods;
};
}): Promise<void>;
/**
* Types a given text into a textfield.
* Use a relation to specify how to find
* the textfield in relation to a specific label.
*
* **Examples:**
* ```typescript
* // Finds the textfield nearest to the label 'Email'
* await aui.typeIntoTextfield({textToWrite: 'Hello World', relation: {label: 'Email'}});
*
* // Finds the textfield above/below a label 'Password'
* await aui.typeIntoTextfield(
* {textToWrite: 'Hello World', relation: {type: 'above', label: 'Password'}}
* );
* await aui.typeIntoTextfield(
* {textToWrite: 'Hello World', relation: {type: 'below', label: 'Password'}}
* );
*
* // If there is no label but a placeholder, the label is contained in the textfield
* await aui.typeIntoTextfield(
* {textToWrite: 'Hello World', relation: {type: 'contains', label: 'Enter email'}}
* );
* ```
*
* @param {Object} params - Object containing required `textToWrite` property and
* optional `relation` property.
* @property {string} params.textToWrite - The text to be typed into the textfield.
* @property {Object} params.relation - Object describing the relationship between the
* textfield being interacted with and another element.
* @property {RelationsForConvenienceMethods} params.relation.type - The type of
* relation, optional.
* @property {string} params.relation.label - The label associated with the related
* element, optional.
*/
typeIntoTextfield(params: {
textToWrite: string;
relation: {
type?: RelationsForConvenienceMethods;
label: string;
};
}): Promise<void>;
/**
* Click on a specific text.
* You can also use a RegEx or match the text exactly by specifying the specific flag.
* Use a relation to find the text in relation to a specific text.
*
* **Examples:**
* ```typescript
* // Click text that matches exactly
* await aui.clickText({text: 'askui', matching: 'similar'})
*
* // Click text that contains 'pie' or 'cake' or 'Pie' or 'Cake'
* await aui.clickText({text: '.*([Pp]ie|[Cc]ake).*', matching: 'regex'})
*
* // Click the text 'TERMINAL' that is left of the text 'Ports'
* await aui.clickText({
* text: 'TERMINAL',
* matching: "exact",
* relation: { type: 'leftOf', text: 'PORTS' }
* })
* ```
*
* @param {Object} params - Object containing required `text` property and optional properties
* for regular expression matching and relation.
* @property {string} params.text - The text to be clicked.
* @property {string} params.matching - Whether the text is matched using similarity,
* exact match or a regular expression.
* @property {Object} [params.relation] - Object describing the relationship between the
* clicked text and another element.
* @property {RelationsForConvenienceMethods} params.relation.type - The type of relation.
* @property {string} params.relation.text - The label or text associated with the
* related element or state.
*/
clickText(params: {
text: string;
matching: TextMatchingOption;
relation?: {
type: RelationsForConvenienceMethods;
text: string;
};
}): Promise<void>;
private evaluateMatchingProperty;
/**
* Check if one or multiple elements are detected.
*
* **Examples:**
* ```typescript
* await aui.expectAllExist([
* {
* type: 'text',
* text: {
* value: 'Switch to Dark',
* matching: 'similar'
* }
* },
* ]);
*
* // Check for existence of multiple elements
* await aui.expectAllExist([
* {
* type: 'textfield',
* relation: {
* type: 'rightOf',
* text: 'Email:'
* }
* },
* {
* type: 'element',
* text: {
* value: 'Switch to Dark'
* }
* },
* ]);
*
* // Validate existence
* const exists = await aui.expectAllExist([...]);
* exists.allExist // true when every element exists
*
* // Check which elements do not exist
* // with the elements property
* const nonExistentElements = exists.elements.filter((e) => e.exists===false)
* ```
*
* @param {ElementExistsQuery[]} query - Objects containing the required property
* 'type' and the optional properties
* 'text' and 'relation'.
* @property {string} query.type - The type of the element: 'otherElement' | 'switch' |
* 'element' | 'container' | 'checkbox' | 'element' |
* 'button' | 'table' | 'text' | 'icon' | 'image' | 'textfield'
* @property {Object} [query.text] - Object containing value and matching strategy.
* @property {string} query.text.value - The text to match for.
* @property {string} [query.text.matching] - Whether the text is matched using similarity,
* exact match or a regular expression.
* @property {Object} [query.relation] - Object describing the relationship between the
* clicked text and another element.
* @property {RelationsForConvenienceMethods} query.relation.type - The type of relation.
* @property {string} query.relation.text - The label or text associated with the
* related element or state.
* @returns {ExpectAllExistResult.allExist} - If every element exists.
* @returns {ExpectAllExistResult.elements} - ExpectExistenceElement[].
*/
expectAllExist(query: ElementExistsQuery[]): Promise<ExpectAllExistResult>;
}