askui
Version:
Reliable, automated end-to-end-testing that depends on what is shown on your screen instead of the technology you are running on
57 lines (56 loc) • 2.76 kB
TypeScript
import { BetaMessage, BetaMessageParam } from '@anthropic-ai/sdk/resources/beta/messages';
import { ControlCommand } from '../core/ui-control-commands';
import { UiControllerClient } from './ui-controller-client';
import { InferenceClient } from './inference-client';
import { Annotation } from '../core/annotation/annotation';
import { CustomElementJson } from '../core/model/custom-element-json';
import { DetectedElement } from '../core/model/annotation-result/detected-element';
import { UiControllerClientConnectionState } from './ui-controller-client-connection-state';
import { Instruction, StepReporter } from '../core/reporting';
import { RetryStrategy } from './retry-strategies/retry-strategy';
import { ModelCompositionBranch } from './model-composition-branch';
export declare class ExecutionRuntime {
private uiControllerClient;
private inferenceClient;
private stepReporter;
retryStrategy: RetryStrategy;
constructor(uiControllerClient: UiControllerClient, inferenceClient: InferenceClient, stepReporter: StepReporter, retryStrategy: RetryStrategy);
connect(): Promise<UiControllerClientConnectionState>;
disconnect(): void;
startVideoRecording(): Promise<void>;
stopVideoRecording(): Promise<void>;
readVideoRecording(): Promise<string>;
requestControl(controlCommand: ControlCommand): Promise<void>;
executeInstruction(instruction: Instruction, modelComposition: ModelCompositionBranch[]): Promise<void>;
private readonly EXEC_REPETITION_COUNT;
private executeCommandRepeatedly;
/**
* Command prediction may fail, e.g., due to application still loading
* --> retry with linear back-off
*/
private predictCommandWithRetry;
private isImageRequiredByConfig;
private isImageRequired;
private isAnnotationRequired;
getScreenshot(): Promise<string>;
private buildSnapshot;
private predictCommand;
annotateInteractively(): Promise<void>;
takeScreenshotIfImageisNotProvided(imagePath?: string): Promise<string>;
getStartingArguments(): Promise<Record<string, string | number | boolean>>;
getDetectedElements(instruction: string, customElementJson?: CustomElementJson[]): Promise<DetectedElement[]>;
annotateImage(imagePath?: string, customElementJson?: CustomElementJson[], elements?: DetectedElement[]): Promise<Annotation>;
predictVQA(prompt: string, config?: object): Promise<any>;
predictActResponse(params: {
max_tokens: number;
messages: BetaMessageParam[];
model: string;
system?: string;
tools?: object[];
betas?: string[];
tool_choice?: {
type: 'tool' | 'any' | 'auto';
name?: string;
};
}): Promise<BetaMessage>;
}