askui
Version:
Reliable, automated end-to-end-testing that depends on what is shown on your screen instead of the technology you are running on
1,271 lines • 140 kB
JavaScript
/* eslint-disable @typescript-eslint/no-use-before-define */
/* eslint-disable @typescript-eslint/naming-convention */
/* eslint-disable max-classes-per-file */
/* eslint-disable max-len */
// Autogenerated from typescript.template file
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
function isStackTraceCodeline(line) {
return /[ \t]+at .+/.test(line);
}
function splitStackTrace(stacktrace) {
const errorStackTraceLines = stacktrace.split('\n');
const errorIndexOfFirstCodeLine = errorStackTraceLines.findIndex(isStackTraceCodeline);
const errorStacktraceHead = errorStackTraceLines.slice(0, errorIndexOfFirstCodeLine);
const errorStacktraceCodeLines = errorStackTraceLines.slice(errorIndexOfFirstCodeLine);
return { head: errorStacktraceHead, codelines: errorStacktraceCodeLines };
}
function rewriteStackTraceForError(error, newStackTrace) {
var _a;
const errorStacktraceSplit = splitStackTrace((_a = error.stack) !== null && _a !== void 0 ? _a : '');
const newStacktraceSplit = splitStackTrace(newStackTrace);
// eslint-disable-next-line no-param-reassign
error.stack = [
...errorStacktraceSplit.head,
...newStacktraceSplit.codelines,
' ',
...errorStacktraceSplit.head,
...errorStacktraceSplit.codelines,
].join('\n');
return error;
}
export var Separators;
(function (Separators) {
Separators["STRING"] = "<|string|>";
})(Separators || (Separators = {}));
export const PC_KEY_VALUES = ['backspace', 'delete', 'enter', 'tab', 'escape', 'up', 'down', 'right', 'left', 'home', 'end', 'pageup', 'pagedown', 'f1', 'f2', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9', 'f10', 'f11', 'f12', 'space', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '!', '"', '#', '$', '%', '&', "'", '(', ')', '*', '+', ',', '-', '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_', '`', '{', '|', '}', '~ '];
export const ANDROID_KEY_VALUES = ['backspace', 'delete', 'enter', 'tab', 'escape', 'up', 'down', 'right', 'left', 'home', 'end', 'pageup', 'pagedown', 'f1', 'f2', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9', 'f10', 'f11', 'f12', 'space', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '!', '"', '#', '$', '%', '&', "'", '(', ')', '*', '+', ',', '-', '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_', '`', '{', '|', '}', '~ '];
export const MODIFIER_KEY_VALUES = ['command', 'alt', 'control', 'shift', 'right_shift'];
class FluentBase {
constructor(prev) {
this.prev = prev;
this._textStr = '';
this._params = new Map();
}
static addParams(paramsList, params) {
params.forEach((value, key) => {
if (!paramsList.has(key)) {
paramsList.set(key, []);
}
paramsList.set(key, [value, ...paramsList.get(key)]);
});
return paramsList;
}
fluentCommandStringBuilder(modelComposition = [], skipCache = false, retryStrategy, currentInstruction = '', paramsList = new Map()) {
const newCurrentInstruction = `${this.textStr} ${currentInstruction}`;
const newParamsList = FluentBase.addParams(paramsList, this._params);
if (this instanceof FluentCommand) {
const fluentCommand = this;
const customElements = newParamsList.has('customElement') ? newParamsList.get('customElement') : [];
const aiElementNames = newParamsList.has('aiElementName') ? newParamsList.get('aiElementName') : [];
return fluentCommand.fluentCommandExecutor(newCurrentInstruction.trim(), modelComposition, {
customElementsJson: customElements,
aiElementNames,
}, skipCache, retryStrategy);
}
if (!this.prev) {
throw new Error('Prev element not defined');
}
return this.prev.fluentCommandStringBuilder(modelComposition, skipCache, retryStrategy, newCurrentInstruction, newParamsList);
}
getterStringBuilder(currentInstruction = '', paramsList = new Map()) {
const newCurrentInstruction = `${this.textStr} ${currentInstruction}`;
const newParamsList = FluentBase.addParams(paramsList, this._params);
if (this instanceof Getter) {
const getter = this;
const customElements = newParamsList.has('customElement') ? newParamsList.get('customElement') : [];
const aiElementNames = newParamsList.has('aiElementName') ? newParamsList.get('aiElementName') : [];
return getter.getterExecutor(newCurrentInstruction.trim(), {
customElementsJson: customElements,
aiElementNames,
});
}
if (!this.prev) {
throw new Error('Prev element not defined');
}
return this.prev.getterStringBuilder(newCurrentInstruction, newParamsList);
}
get textStr() { return this._textStr; }
get params() { return this._params; }
}
export class Exec extends FluentBase {
exec(execOptions) {
const originStacktrace = { stack: '' };
Error.captureStackTrace(originStacktrace, this.exec);
return this.fluentCommandStringBuilder(execOptions === null || execOptions === void 0 ? void 0 : execOptions.modelComposition, execOptions === null || execOptions === void 0 ? void 0 : execOptions.skipCache, execOptions === null || execOptions === void 0 ? void 0 : execOptions.retryStrategy).catch((err) => Promise.reject(rewriteStackTraceForError(err, originStacktrace.stack)));
}
}
// Filters
export class FluentFilters extends FluentBase {
/**
* Filters for a UI element 'other element'.
*
* @return {FluentFiltersOrRelations}
*/
otherElement() {
this._textStr = '';
this._textStr += 'other';
this._textStr += ' element';
return new FluentFiltersOrRelations(this);
}
/**
* Filters for a UI element 'switch'.
*
* @return {FluentFiltersOrRelations}
*/
switch() {
this._textStr = '';
this._textStr += 'switch';
return new FluentFiltersOrRelations(this);
}
/**
* Filters for a UI element 'container'.
*
* @return {FluentFiltersOrRelations}
*/
container() {
this._textStr = '';
this._textStr += 'container';
return new FluentFiltersOrRelations(this);
}
/**
* Filters for a UI element 'checkbox'.
*
* @return {FluentFiltersOrRelations}
*/
checkbox() {
this._textStr = '';
this._textStr += 'checkbox';
return new FluentFiltersOrRelations(this);
}
/**
* Filters for any UI element on the screen.
*
* **Examples:**
* ```typescript
* await aui.moveMouseTo().element().exec()
* ```
*
* @return {FluentFiltersOrRelations}
*/
element() {
this._textStr = '';
this._textStr += 'element';
return new FluentFiltersOrRelations(this);
}
/**
* Filters special elements defined over a specifically trained custom element descriptor.
*
* Custom element descriptors are trained on your elements that were not detected with our
* default models. Please contact us for further details on how to do this. We are working on
* a solution to provide this in our User Portal.
*
* In the example below circle refers to a circle shaped icon with specific properties.
*
* **Examples:**
* ```typescript
* await aui.moveMouseTo().element().special("circle").exec()
* ```
*
* @param {string} text - A text to be matched.
*
* @return {FluentFiltersOrRelations}
*/
special(text) {
this._textStr = '';
this._textStr += 'special';
this._textStr += ` ${Separators.STRING}${text}${Separators.STRING}`;
return new FluentFiltersOrRelations(this);
}
/**
* Filters for a UI element 'button'.
*
* **Examples:**
* ```typescript
* await aui.click().button().contains().text().withText('Google Search').exec()
* ```
*
* 
*
* @return {FluentFiltersOrRelations}
*/
button() {
this._textStr = '';
this._textStr += 'button';
return new FluentFiltersOrRelations(this);
}
/**
* Filters for a UI element 'table'.
*
* @return {FluentFiltersOrRelations}
*/
table() {
this._textStr = '';
this._textStr += 'table';
return new FluentFiltersOrRelations(this);
}
/**
* Filters for an UI element 'text'.
*
* Takes an optional parameter to filter for a specific text.
* See the examples below.
*
* See also the filters `withTextRegex()` and `withExactText()`
*
* **Examples:**
* ```typescript
* await aui.click().text().exec();
* await aui.click().text('Username').exec();
*
* // Matching with an exact text
* await aui.click().text().withExactText('Username').exec();
*
* // Matching with a regex
* await aui.click().text().withTextRegex('\b[Ss]\w+').exec();
* ```
*
* @param {string} [text] - A text to be matched.
*
* @return {FluentFiltersOrRelations}
*/
text(text) {
this._textStr = '';
this._textStr += 'text';
if (text !== undefined) {
this._textStr += ` ${Separators.STRING}${text}${Separators.STRING}`;
}
return new FluentFiltersOrRelations(this);
}
/**
* Filters for an UI element 'wordlevel'.
*
* Takes an optional parameter to filter for a specific text.
* See the examples below.
*
* **Examples:**
* ```typescript
* await aui.fluentCommandExecutor('Click on button right of wordlevel 'User_PRG in:'');
* ```
*
* @param {string} [wordlevel] - A text to be matched.
*
* @return {FluentFiltersOrRelations}
*/
wordlevel(wordlevel) {
this._textStr = '';
this._textStr += 'wordlevel';
if (wordlevel !== undefined) {
this._textStr += ` ${Separators.STRING}${wordlevel}${Separators.STRING}`;
}
return new FluentFiltersOrRelations(this);
}
/**
* Filters for a UI element 'icon'.
*
* You can combine it with the element-description 'withText()' to look for a specific icon.
*
* **Examples:**
* ```typescript
* icon().withText('plus')
* ```
*
* 
*
* **Note:** This is an alpha feature. The prediction of the icon name is sometimes unstable. Use custom elements as an alternative.
*
* @return {FluentFiltersOrRelations}
*/
icon() {
this._textStr = '';
this._textStr += 'icon';
return new FluentFiltersOrRelations(this);
}
/**
* Filters for a 'custom element', that is a UI element that is defined by providing an image and other parameters such as degree of rotation. It allows filtering for a UI element based on an image instead of using text or element descriptions like `button().withText('Submit')` in `await aui.click().button().withText('Submit').exec()`.
*
* See the tutorial - [Custom Element](https://docs.askui.com/docs/general/Element%20Selection/custom-elements) for more details.
*
* **Example**
* ```typescript
* await aui
* .click()
* .customElement({
* customImage: './logo.png', // required
* name: 'myLogo', // optional
* threshold: 0.5, // optional, defaults to 0.5
* stopThreshold: 0.9, // optional, defaults to 0.9
* rotationDegreePerStep: 0, // optional, defaults to 0
* imageCompareFormat: 'grayscale', // optional, defaults to 'grayscale'
* // mask:{x:0, y:0}[] // optional, a polygon to match only a certain area of the custom element
* })
* .exec();
* ```
*
* **Arguments**
*
* - **customImage** (*`string`, required*):
* - A cropped image in the form of a base64 string or file path.
* - **name** (*`string`, optional*):
* - A unique name that can be used for filtering for the custom element. If not given, any text inside the custom image will be detected via OCR.
* - **threshold** (*`number`, optional*):
* - A threshold for how much a UI element needs to be similar to the custom element as defined by the image. Takes values between `0.0` (== all elements are recognized as the custom element which is probably not what you want) and `1.0` (== elements need to look exactly like the `customImage` which is unlikely to be achieved as even minor differences count). Defaults to `0.5`.
* - **stopThreshold** (*`number`, optional*):
* - A threshold for when to stop searching for UI elements similar to the custom element. As soon as UI elements have been found that are at least as similar as the `stopThreshold`, the search is going to stop. After that elements are filtered using the `threshold`. Because of that the `stopThreshold` should be greater than or equal to `threshold`. It is primarily to be used as a speed improvement (by lowering the value). Takes values between `0.0` and `1.0`. Defaults to `0.9`.
* - **rotationDegreePerStep** (*`number`, optional*):
* - Step size in rotation degree. Rotates the custom image by this step size until 360° is exceeded. The range is from `0` to `360`. Defaults to `0`.
* - **imageCompareFormat** (*`'RGB' | 'grayscale' | 'edges'`, optional*):
* - The color compare style. `'edges'` compares only edges, `'greyscale'` compares the brightness of each pixel whereas `'RGB'` compares all three colors (red, green, blue). Defaults to `'grayscale'`.
*
*
* @param {CustomElementJson} customElement - The custom element to filter for.
*
* @return {FluentFiltersOrRelations}
*/
customElement(customElement) {
this._textStr = '';
this._textStr += 'custom';
this._textStr += ' element';
this._params.set('customElement', customElement);
return new FluentFiltersOrRelations(this);
}
/**
* Detects an AI Element created with the [snipping workflow](https://docs.askui.com/docs/general/Element%20Selection/aielement#snipping-workflow).
*
* **Examples:**
*
* ```typescript
* await aui.click().aiElement('askui-logo').exec();
* ```
*
* @param {string} aiElementName - Name of the AI Element.
*
* @return {FluentFiltersOrRelations}
*/
aiElement(aiElementName) {
this._textStr = '';
this._textStr += 'ai';
this._textStr += ' element';
this._textStr += ' with';
this._textStr += ' name';
this._textStr += ` ${Separators.STRING}${aiElementName}${Separators.STRING}`;
this._params.set('aiElementName', aiElementName);
return new FluentFiltersOrRelations(this);
}
/**
* Filters for a UI element 'image'.
*
* **Examples:**
* ```typescript
* // Works if there is only one image visible on the screen
* await aui.click().image().exec();
*
* // Works if you have an image with
* // a text below
* await aui.click().image().above().text().withText('Automating WebGL').exec();
* ```
*
* 
*
*
*
* @return {FluentFiltersOrRelations}
*/
image() {
this._textStr = '';
this._textStr += 'image';
return new FluentFiltersOrRelations(this);
}
/**
* Filters for a UI element 'textfield'.
*
* **Examples:**
* ```typescript
* // Works if there is only one textfield visible on the screen
* await aui.typeIn('Oh yeah').textfield().exec();
*
* // Works if you have a labeled textfield
* // Label is above the textfield
* await aui.typeIn('Oh yeah').textfield().below().text().withText('E-Mail Address').exec();
* ```
*
* 
*
*
*
* @return {FluentFiltersOrRelations}
*/
textfield() {
this._textStr = '';
this._textStr += 'textfield';
return new FluentFiltersOrRelations(this);
}
/**
* Filters for similar -- meaning >70% similar -- text.
*
* Takes an optional parameter to specify the similarity. Usually you need the optional parameter for long texts you want to match precisely.
*
* _We use [RapidFuzz](https://maxbachmann.github.io/RapidFuzz/Usage/fuzz.html#ratio) which calculates the similarity like this:_
*
* `1 - (distance / (lengthString1 + lengthString2))`
*
* **Examples:**
* ```typescript
* 'text' === withText('text') => true
* 'test' === withText('text') => true
* 'Test' === withText('text') => true
* 'Text' === withText('text') => true
* 'TEXT' === withText('text') => true
* 'texst' === withText('text') => true
* 'texts' === withText('text') => true
*
* // usually false
* 'atebxtc' === withText('text') => false
* 'other' === withText('text') => false
*
* // optional parameter: similarity_score
* '978-0-201-00650-6' == withText("978-0-201-00", 90) => false with 82.76 < 90 similarity
* '978-0-201-00650-6' == withText("978-0-201-00650", 90) => true with 93.75 > 90 similarity
* ```
* 
*
* @param {string} text - A text to be matched.
* @param {number} [similarityScore=70] - Similarity score minimum value, it should be between `0` and `100`.
*
* @return {FluentFiltersOrRelations}
*/
withText(text, similarityScore = 70) {
this._textStr = '';
this._textStr += 'with';
this._textStr += ' text';
this._textStr += ` ${Separators.STRING}${text}${Separators.STRING}`;
if (similarityScore !== undefined) {
this._textStr += ` that matches to ${similarityScore} %`;
}
return new FluentFiltersOrRelations(this);
}
/**
* Filters for texts, which match the regex pattern.
*
* **Examples:**
*
* ```typescript
* 'The rain in Spain' === withTextRegex('\b[Ss]\w+') => true
* 'The rain in Portugal' === withTextRegex('\b[Ss]\w+') => false
* 'The rain in switzerland' === withTextRegex('\b[Ss]\w+') => true
*
* await aui.get().text().withTextRegex('\b[Ss]\w+').exec()
* ```
*
* 
*
*
*
* @param {string} regex_pattern - A regex pattern
*
* @return {FluentFiltersOrRelations}
*/
withTextRegex(regex_pattern) {
this._textStr = '';
this._textStr += 'match';
this._textStr += ' regex';
this._textStr += ' pattern';
this._textStr += ` ${Separators.STRING}${regex_pattern}${Separators.STRING}`;
return new FluentFiltersOrRelations(this);
}
/**
* Filters for equal text.
*
* **Note:** This should be only used in cases where the similarity
* comparison of {@link FluentFilters.withText()} allows not for
* specific enough filtering (too many elements).
*
* **Examples:**
* ```typescript
* 'text' === withExactText('text') => true
* 'test' === withExactText('text') => false
* 'other' === withExactText('text') => false
*
* await aui.moveMouseTo().text().withExactText('Password').exec()
* ```
*
* 
*
*
*
* @param {string} text - A text to be matched.
*
* @return {FluentFiltersOrRelations}
*/
withExactText(text) {
this._textStr = '';
this._textStr += 'equals';
this._textStr += ' text';
this._textStr += ` ${Separators.STRING}${text}${Separators.STRING}`;
return new FluentFiltersOrRelations(this);
}
/**
* Filters for text containing the text provided as an argument.
*
* **Examples:**
* ```typescript
* 'This is a text' === containsText('text') => true
* 'This is a text' === containsText('other text') => false
* 'This is a text' === containsText('other') => false
* ```
* 
*
* @param {string} text - A text to be matched.
*
* @return {FluentFiltersOrRelations}
*/
containsText(text) {
this._textStr = '';
this._textStr += 'contain';
this._textStr += ' text';
this._textStr += ` ${Separators.STRING}${text}${Separators.STRING}`;
return new FluentFiltersOrRelations(this);
}
/**
* Filters for PTA locating the text provided as an argument.
*
* @param {string} text - A text to be located.
*
* @return {FluentFiltersOrRelations}
*/
pta(text) {
this._textStr = '';
this._textStr += 'pta';
this._textStr += ` ${Separators.STRING}${text}${Separators.STRING}`;
return new FluentFiltersOrRelations(this);
}
}
// Relations
export class FluentFiltersOrRelations extends FluentFilters {
/**
* Logic or operator
*
* **Examples:**
* ```text
* scene 1
* -------------- ---------------
* | button | | icon |
* -------------- ---------------
*
* scene 2
* -------------- ---------------
* | button | | text |
* -------------- ---------------
*
* ```
* In case, that your reference element can have multiple values, in the following example, the element right of the button can be either icon or text.
* You can use **the `or()` relation**, so your instruction is valid for both scenes
* ```typescript
* const button = await aui.get().button().rightOf().icon().or().text().exec();
* console.log(button);
* ```
* Returns the same button for both cases
* ```text
* console output: [
* DetectedElement {
* name: 'BUTTON',
* text: 'button',
* bndbox: BoundingBox {
* xmin: 900,
* ymin: 910,
* xmax: 920,
* ymax: 930
* }
* }
* ]
* ```
*
* @return {FluentFilters}
*/
or() {
this._textStr = '';
this._textStr += 'or';
return new FluentFilters(this);
}
/**
* Logic and operator
*
* **Examples:**
* ```text
* example scene:
* --------------- ----------------
* | icon user | | icon search |
* --------------- ----------------
* ```
* ```typescript
* const icons = await aui.get().icon().exec();
* console.log(icons);
* ```
* Using only the element-description icon, the get will return both icons
* ```text
* console output: [
* DetectedElement {
* name: 'ICON',
* text: 'user',
* bndbox: BoundingBox {
* xmin: 1000,
* ymin: 1010,
* xmax: 1020,
* ymax: 1030
* }
* },
* DetectedElement {
* name: 'ICON',
* text: 'search',
* bndbox: BoundingBox {
* xmin: 900,
* ymin: 910,
* xmax: 920,
* ymax: 930
* }
* }
* ]
* ```
* You can combine element-descriptions with **the `and()` relation** and specify exactly which icon you want.
* ```typescript
* const icons = await aui.get().icon().and().withText('user').exec()
* console.log(icons)
* ```
* The get returns only the user icon although both elements are icons.
* ```text
* console output: [
* DetectedElement {
* name: 'ICON',
* text: 'user',
* bndbox: BoundingBox {
* xmin: 900,
* ymin: 910,
* xmax: 920,
* ymax: 930
* }
* }
* ]
* ```
*
* @return {FluentFilters}
*/
and() {
this._textStr = '';
this._textStr += 'and';
return new FluentFilters(this);
}
/**
* Filters for an element inside another element.
*
* **Examples:**
* ```typescript
* --------------------
* | outerEl |
* | -------------- |
* | | innerEl | |
* | -------------- |
* | |
* --------------------
*
* // Returns innerEl because innerEl is inside outerEl
* ...innerEl().in().outerEl()
* // Returns nothing because innerEl is not inside outerEl
* ...outerEl().in().innerEl()
* ```
* 
*
* @return {FluentFilters}
*/
in() {
this._textStr = '';
this._textStr += 'in';
return new FluentFilters(this);
}
/**
* Filters for an element right of another element.
*
* Takes an optional parameter `index` to select the nth element (defaults to `0`).
*
* Takes an optional parameter `intersection_area` to specify which elements right of the other element are filtered for based on their vertical position (y-coordinates of bounding box):
* - `"element_center_line"` - considered right of the other element if element's bounding box intersects with a horizontal line passing through the center of the other element
* - `"element_edge_area"` - considered right of the other element if element's bounding box intersects with an area between the top and the bottom edge of the other element
* - `"display_edge_area"` - considered right of the other element no matter where it is placed vertically on the screen (y-axis)
*
* **Examples:**
* ```typescript
* ---------- --------- ---------
* | button | | text0 | | text3 |
* ---------- --------- --------- ---------
* | text1 | ---------
* --------- | text2 |
* ---------
*
* // General explanation for element_center_line
* // This will find text0 and text3
* ...text().rightOf(..., 'element_center_line').button()
*
* // General explanation for element_edge_area
* // This will find text0, text1 and text3
* ...text().rightOf(..., 'element_edge_area').button()
*
* // General explanation and display_edge_area
* // This will find text0, text1, text2 and text3
* ...text().rightOf(..., 'display_edge_area').button()
*
* // More examples:
* // Returns text0 because it is the first element rightOf button
* ...text().rightOf().button()
* ...text().rightOf(0).button()
* ...text().rightOf(0, 'element_edge_area').button()
*
* // Returns text3 because it is the second text touched by the
* // horizontal line from the center of button
* // Notice: text1 is not touched!
* ...text().rightOf(1, 'element_center_line').button()
*
* // Returns text3 because it is the third text touched by the
* // vertical area rightOf the y-axis of button
* // Notice: text2 is not touched!
* ...text().rightOf(2, 'element_edge_area').button()
*
* // Returns text2 because it is the third element rightOf button
* ...text().rightOf(2, 'display_edge_area').button()
*
* // Returns no element because button is rightOf the texts
* ...button().rightOf().text()
* ```
* 
*
* @param {number} [index=0] - Index of element to filter for going into the direction specified. Defaults to `0` which is the first element (zero-indexed) found in that direction.
* @param {INTERSECTION_AREA} [intersection_area="element_edge_area"] - Intersecting with either `"element_center_line"`, `"element_edge_area"` or `"display_edge_area"`. Defaults to `"element_edge_area"`.
*
* @return {FluentFilters}
*/
rightOf(index = 0, intersection_area = 'element_edge_area') {
this._textStr = '';
if (index !== undefined) {
this._textStr += `index ${index}`;
}
this._textStr += ' right';
this._textStr += ' of';
if (intersection_area !== undefined) {
this._textStr += ` intersection_area ${intersection_area}`;
}
return new FluentFilters(this);
}
/**
* Filters for an element left of another element.
*
* Takes an optional parameter `index` to select the nth element (defaults to `0`).
*
* Takes an optional parameter `intersection_area` to specify which elements left of the other element are filtered for based on their vertical position (y-coordinates of bounding box):
* - `"element_center_line"` - considered left of the other element if element's bounding box intersects with a horizontal line passing through the center of the other element
* - `"element_edge_area"` - considered left of the other element if element's bounding box intersects with an area between the top and the bottom edge of the other element
* - `"display_edge_area"` - considered left of the other element no matter where it is placed vertically on the screen (y-axis)
*
* **Examples:**
* ```typescript
* --------- --------- ----------
* | text3 | | text0 | | button |
* --------- --------- --------- ----------
* --------- | text1 |
* | text2 | ---------
* ---------
*
* // General explanation for element_center_line
* // This will find text0 and text3
* ...text().leftOf(..., 'element_center_line').button()
*
* // General explanation for element_edge_area
* // This will find text0, text1 and text3
* ...text().leftOf(..., 'element_edge_area').button()
*
* // General explanation and display_edge_area
* // This will find text0, text1, text2 and text3
* ...text().leftOf(..., 'display_edge_area').button()
*
* // More examples:
* // Returns text0 because it is the first element leftOf button
* ...text().leftOf().button()
* ...text().leftOf(0).button()
* ...text().leftOf(0, 'element_edge_area').button()
*
* // Returns text3 because it is the second text touched by the
* // horizontal line from the center of button
* // Notice: text1 is not touched!
* ...text().leftOf(1, 'element_center_line').button()
*
* // Returns text3 because it is the third text touched by the
* // vertical area leftOf the y-axis of button
* // Notice: text2 is not touched!
* ...text().leftOf(2, 'element_edge_area').button()
*
* // Returns text2 because it is the third element leftOf button
* ...text().leftOf(2, 'display_edge_area').button()
*
* // Returns no element because button is rightOf the texts
* ...button().leftOf().text()
* ```
* 
*
* @param {number} [index=0] - Index of element to filter for going into the direction specified. Defaults to `0` which is the first element (zero-indexed) found in that direction.
* @param {INTERSECTION_AREA} [intersection_area="element_edge_area"] - Intersecting with either `"element_center_line"`, `"element_edge_area"` or `"display_edge_area"`. Defaults to `"element_edge_area"`.
*
* @return {FluentFilters}
*/
leftOf(index = 0, intersection_area = 'element_edge_area') {
this._textStr = '';
if (index !== undefined) {
this._textStr += `index ${index}`;
}
this._textStr += ' left';
this._textStr += ' of';
if (intersection_area !== undefined) {
this._textStr += ` intersection_area ${intersection_area}`;
}
return new FluentFilters(this);
}
/**
* Filters for an element below another element.
*
* Takes an optional parameter `index` to select the nth element (defaults to `0`).
*
* Takes an optional parameter `intersection_area` to specify which elements below of the other element are filtered for based on their horizontal position (y-coordinates of bounding box):
* - `"element_center_line"` - considered below of the other element if element's bounding box intersects with a vertical line passing through the center of the other element
* - `"element_edge_area"` - considered below of the other element if element's bounding box intersects with an area between the left and the right edge of the other element
* - `"display_edge_area"` - considered below of the other element no matter where it is placed horizontally on the screen (y-axis)
*
* **Examples:**
* ```typescript
* ------------
* | text |
* ------------
* ------------
* | button0 |
* ------------
* -----------
* | button1 |
* -----------
* -----------
* | button2 |
* -----------
* ------------
* | button3 |
* ------------
*
* // General explanation for element_center_line
* // This will find button0 and button3
* ...button().below(..., 'element_center_line').text()
*
* // General explanation for element_edge_area
* // This will find button0, button1 and button3
* ...button().below(..., 'element_edge_area').text()
*
* // General explanation and display_edge_area
* // This will find button0, button1, button2 and button3
* ...button().below(..., 'display_edge_area').text()
*
* // More examples:
* // Returns button0 because button0 is the first button below text
* ...button().below().text()
* ...button().below(0).text()
* ...button().below(0, 'element_edge_area').text()
*
* // Returns button3 because it is the second button touched by the
* // vertical line from the center of text
* // Notice: button1 is not touched
* ...button().below(1, 'element_center_line').text()
*
* // Returns button3 because it is the third button touched by the
* // vertical area below the x-axis of text
* // Notice: button2 is not touched!
* ...button().below(2, 'element_edge_area').text()
*
* // Returns button2 because it is the third element below text
* ...button().below(2, 'display_edge_area').text()
*
* // Returns no element because text is above the buttons
* ...text().below().button()
* ```
* 
*
* @param {number} [index=0] - Index of element to filter for going into the direction specified. Defaults to `0` which is the first element (zero-indexed) found in that direction.
* @param {INTERSECTION_AREA} [intersection_area="element_edge_area"] - Intersecting with either `"element_center_line"`, `"element_edge_area"` or `"display_edge_area"`. Defaults to `"element_edge_area"`.
*
* @return {FluentFilters}
*/
below(index = 0, intersection_area = 'element_edge_area') {
this._textStr = '';
if (index !== undefined) {
this._textStr += `index ${index}`;
}
this._textStr += ' below';
if (intersection_area !== undefined) {
this._textStr += ` intersection_area ${intersection_area}`;
}
return new FluentFilters(this);
}
/**
* Filters for an element above another element.
*
* Takes an optional parameter `index` to select the nth element (defaults to `0`).
*
* Takes an optional parameter `intersection_area` to specify which elements above of the other element are filtered for based on their horizontal position (y-coordinates of bounding box):
* - `"element_center_line"` - considered above of the other element if element's bounding box intersects with a vertical line passing through the center of the other element
* - `"element_edge_area"` - considered above of the other element if element's bounding box intersects with an area between the left and the right edge of the other element
* - `"display_edge_area"` - considered above of the other element no matter where it is placed horizontally on the screen (y-axis)
*
* **Examples:**
* ```typescript
* ------------
* | text3 |
* ------------
* ------------
* | text2 |
* ------------
* ------------
* | text1 |
* ------------
* ------------
* | text0 |
* ------------
* ------------
* | button |
* ------------
*
* // General explanation for element_center_line
* // This will find text0 and text3
* ...text().above(..., 'element_center_line').button()
*
* // General explanation for element_edge_area
* // This will find text0, text1 and text3
* ...text().above(..., 'element_edge_area').button()
*
* // General explanation and display_edge_area
* // This will find text0, text1, text2 and text3
* ...text().above(..., 'display_edge_area').button()
*
* // More examples:
* // Returns text0 because it is the first element above button
* ...text().above().button()
* ...text().above(0).button()
* ...text().above(0, 'element_edge_area').button()
*
* // Returns text3 because it is the second text touched by the
* // vertical line from the center of button
* // Notice: text1 is not touched!
* ...text().above(1, 'element_center_line').button()
*
* // Returns text3 because it is the third text touched by the
* // vertical area above the x-axis of button
* // Notice: text2 is not touched!
* ...text().above(2, 'element_edge_area').button()
*
* // Returns text2 because it is the third element above button
* ...text().above(2, 'display_edge_area').button()
*
* // Returns no element because button is below the texts
* ...button().above().text()
* ```
* 
*
* @param {number} [index=0] - Index of element to filter for going into the direction specified. Defaults to `0` which is the first element (zero-indexed) found in that direction.
* @param {INTERSECTION_AREA} [intersection_area="element_edge_area"] - Intersecting with either `"element_center_line"`, `"element_edge_area"` or `"display_edge_area"`. Defaults to `"element_edge_area"`.
*
* @return {FluentFilters}
*/
above(index = 0, intersection_area = 'element_edge_area') {
this._textStr = '';
if (index !== undefined) {
this._textStr += `index ${index}`;
}
this._textStr += ' above';
if (intersection_area !== undefined) {
this._textStr += ` intersection_area ${intersection_area}`;
}
return new FluentFilters(this);
}
/**
* Filters for an element nearest to another element.
*
* **Examples:**
* ```typescript
* --------------
* | button 1 |
* --------------
* --------------
* | text |
* --------------
*
*
*
* --------------
* | button 2 |
* --------------
*
* // Returns button 1 because button 1 is nearer to the text than button 2
* ...button().nearestTo().text()
* ```
* 
*
* @return {FluentFilters}
*/
nearestTo() {
this._textStr = '';
this._textStr += 'nearest';
this._textStr += ' to';
return new FluentFilters(this);
}
/**
* Filters for an element containing another element.
*
* **Example:**
* ```typescript
* --------------------
* | outerEl |
* | -------------- |
* | | innerEl | |
* | -------------- |
* | |
* --------------------
*
* // Returns outerEl because outerEl contains innerEl
* ...outerEl().contains().innerEl()
* // Returns no element because innerEl contains no outerEl
* ...innerEl().contains().outerEl()
* ```
* 
*
* @return {FluentFilters}
*/
contains() {
this._textStr = '';
this._textStr += 'contains';
return new FluentFilters(this);
}
exec(execOptions) {
const originStacktrace = { stack: '' };
Error.captureStackTrace(originStacktrace, this.exec);
return this.fluentCommandStringBuilder(execOptions === null || execOptions === void 0 ? void 0 : execOptions.modelComposition, execOptions === null || execOptions === void 0 ? void 0 : execOptions.skipCache, execOptions === null || execOptions === void 0 ? void 0 : execOptions.retryStrategy).catch((err) => Promise.reject(rewriteStackTraceForError(err, originStacktrace.stack)));
}
}
// Filters
export class FluentFiltersCondition extends FluentBase {
/**
* Filters for a UI element 'other element'.
*
* @return {FluentFiltersOrRelationsCondition}
*/
otherElement() {
this._textStr = '';
this._textStr += 'other';
this._textStr += ' element';
return new FluentFiltersOrRelationsCondition(this);
}
/**
* Filters for a UI element 'switch'.
*
* @return {FluentFiltersOrRelationsCondition}
*/
switch() {
this._textStr = '';
this._textStr += 'switch';
return new FluentFiltersOrRelationsCondition(this);
}
/**
* Filters for a UI element 'container'.
*
* @return {FluentFiltersOrRelationsCondition}
*/
container() {
this._textStr = '';
this._textStr += 'container';
return new FluentFiltersOrRelationsCondition(this);
}
/**
* Filters for a UI element 'checkbox'.
*
* @return {FluentFiltersOrRelationsCondition}
*/
checkbox() {
this._textStr = '';
this._textStr += 'checkbox';
return new FluentFiltersOrRelationsCondition(this);
}
/**
* Filters for any UI element on the screen.
*
* **Examples:**
* ```typescript
* await aui.moveMouseTo().element().exec()
* ```
*
* @return {FluentFiltersOrRelationsCondition}
*/
element() {
this._textStr = '';
this._textStr += 'element';
return new FluentFiltersOrRelationsCondition(this);
}
/**
* Filters special elements defined over a specifically trained custom element descriptor.
*
* Custom element descriptors are trained on your elements that were not detected with our
* default models. Please contact us for further details on how to do this. We are working on
* a solution to provide this in our User Portal.
*
* In the example below circle refers to a circle shaped icon with specific properties.
*
* **Examples:**
* ```typescript
* await aui.moveMouseTo().element().special("circle").exec()
* ```
*
* @param {string} text - A text to be matched.
*
* @return {FluentFiltersOrRelationsCondition}
*/
special(text) {
this._textStr = '';
this._textStr += 'special';
this._textStr += ` ${Separators.STRING}${text}${Separators.STRING}`;
return new FluentFiltersOrRelationsCondition(this);
}
/**
* Filters for a UI element 'button'.
*
* **Examples:**
* ```typescript
* await aui.click().button().contains().text().withText('Google Search').exec()
* ```
*
* 
*
* @return {FluentFiltersOrRelationsCondition}
*/
button() {
this._textStr = '';
this._textStr += 'button';
return new FluentFiltersOrRelationsCondition(this);
}
/**
* Filters for a UI element 'table'.
*
* @return {FluentFiltersOrRelationsCondition}
*/
table() {
this._textStr = '';
this._textStr += 'table';
return new FluentFiltersOrRelationsCondition(this);
}
/**
* Filters for an UI element 'text'.
*
* Takes an optional parameter to filter for a specific text.
* See the examples below.
*
* See also the filters `withTextRegex()` and `withExactText()`
*
* **Examples:**
* ```typescript
* await aui.click().text().exec();
* await aui.click().text('Username').exec();
*
* // Matching with an exact text
* await aui.click().text().withExactText('Username').exec();
*
* // Matching with a regex
* await aui.click().text().withTextRegex('\b[Ss]\w+').exec();
* ```
*
* @param {string} [text] - A text to be matched.
*
* @return {FluentFiltersOrRelationsCondition}
*/
text(text) {
this._textStr = '';
this._textStr += 'text';
if (text !== undefined) {
this._textStr += ` ${Separators.STRING}${text}${Separators.STRING}`;
}
return new FluentFiltersOrRelationsCondition(this);
}
/**
* Filters for an UI element 'wordlevel'.
*
* Takes an optional parameter to filter for a specific text.
* See the examples below.
*
* **Examples:**
* ```typescript
* await aui.fluentCommandExecutor('Click on button right of wordlevel 'User_PRG in:'');
* ```
*
* @param {string} [wordlevel] - A text to be matched.
*
* @return {FluentFiltersOrRelationsCondition}
*/
wordlevel(wordlevel) {
this._textStr = '';
this._textStr += 'wordlevel';
if (wordlevel !== undefined) {
this._textStr += ` ${Separators.STRING}${wordlevel}${Separators.STRING}`;
}
return new FluentFiltersOrRelationsCondition(this);
}
/**
* Filters for a UI element 'icon'.
*
* You can combine it with the element-description 'withText()' to look for a specific icon.
*
* **Examples:**
* ```typescript
* icon().withText('plus')
* ```
*
* 
*
* **Note:** This is an alpha feature. The prediction of the icon name is sometimes unstable. Use custom elements as an alternative.
*
* @return {FluentFiltersOrRelationsCondition}
*/
icon() {
this._textStr = '';
this._textStr += 'icon';
return new FluentFiltersOrRelationsCondition(this);
}
/**
* Filters for a 'custom element', that is a UI element that is defined by providing an image and other parameters such as degree of rotation. It allows filtering for a UI element based on an image instead of using text or element descriptions like `button().withText('Submit')` in `await aui.click().button().withText('Submit').exec()`.
*
* See the tutorial - [Custom Element](https://docs.askui.com/docs/general/Element%20Selection/custom-elements) for more details.
*
* **Example**
* ```typescript
* await aui
* .click()
* .customElement({
* customImage: './logo.png', // required
* name: 'myLogo', // optional
* threshold: 0.5, // optional, defaults to 0.5
* stopThreshold: 0.9, // optional, defaults to 0.9
* rotationDegreePerStep: 0, // optional, defaults to 0
* imageCompareFormat: 'grayscale', // optional, defaults to 'grayscale'
* // mask:{x:0, y:0}[] // optional, a polygon to match only a certain area of the custom element
* })
* .exec();
* ```
*
* **Arguments**
*
* - **customImage** (*`string`, required*):
* - A cropped image in the form of a base64 string or file path.
* - **name** (*`string`, optional*):
* - A unique name that can be used for filtering for the custom element. If not given, any text inside the custom image will be detected via OCR.
* - **threshold** (*`number`, optional*):
* - A threshold for how much a UI element needs to be similar to the custom element as defined by the image. Takes values between `0.0` (== all elements are recognized as the custom element which is probably not what you want) and `1.0` (== elements need to look exac