magnitude-core/dist/connectors/desktopConnector.d.ts

Magnitude e2e testing agent

import { AgentConnector } from ".";
import { ActionDefinition } from '@/actions';
import { Observation } from "@/memory/observation";
/**
 * Generic desktop automation interface.
 * Implementations can use any desktop automation technology
 * (Lume, PyAutoGUI, Windows UI Automation, etc.)
 */
export interface DesktopInterface {
    click(x: number, y: number): Promise<void>;
    rightClick(x: number, y: number): Promise<void>;
    doubleClick(x: number, y: number): Promise<void>;
    moveCursor(x: number, y: number): Promise<void>;
    drag(fromX: number, fromY: number, toX: number, toY: number): Promise<void>;
    scroll(x: number, y: number, deltaX: number, deltaY: number): Promise<void>;
    type(text: string): Promise<void>;
    key(key: string): Promise<void>;
    hotkey(keys: string[]): Promise<void>;
    screenshot(): Promise<Buffer>;
    getScreenSize(): Promise<{
        width: number;
        height: number;
    }>;
    navigate?(url: string): Promise<void>;
    getActiveWindow?(): Promise<{
        title: string;
        app: string;
    }>;
    getOpenWindows?(): Promise<Array<{
        title: string;
        app: string;
        isActive: boolean;
    }>>;
    focusWindow?(title: string): Promise<void>;
    openApplication?(name: string): Promise<void>;
    closeApplication?(name: string): Promise<void>;
}
export interface DesktopConnectorOptions {
    desktopInterface: DesktopInterface;
    virtualScreenDimensions?: {
        width: number;
        height: number;
    };
    minScreenshots?: number;
}
export declare class DesktopConnector implements AgentConnector {
    readonly id: string;
    private desktopInterface;
    private options;
    private logger;
    constructor(options: DesktopConnectorOptions);
    onStart(): Promise<void>;
    onStop(): Promise<void>;
    getActionSpace(): ActionDefinition<any>[];
    collectObservations(): Promise<Observation[]>;
    private formatWindowInfo;
    getInterface(): DesktopInterface;
}