shellx-ai/dist/protocol.d.ts

shellx is a powerful WebSocket-based client for controlling shell commands and UI automation on remote devices.

type Sid = number;
type Uid = number;
/** UI Element bounds */
export type UIBounds = {
    left: number;
    top: number;
    right: number;
    bottom: number;
};
/** UI Element information */
export type UIElement = {
    elementId: string;
    className: string;
    resourceId: string;
    text: string;
    describe: string;
    bounds: UIBounds;
    visible: boolean;
    clickable: boolean;
};
/** UI Hierarchy information */
export type UIHierarchy = {
    elements: UIElement[];
    timestamp: number;
    found: boolean;
    totalMatches?: number;
    success?: boolean;
    errorMessage?: string;
};
export type WAITElement = {
    element: UIElement;
    timestamp: bigint;
    elapsedTimeMs: boolean;
    success?: boolean;
    errorMessage?: string;
};
/** Screen capture response */
export type ScreenShotResponse = {
    imageData: string;
    format: 'png' | 'jpeg';
    timestamp: bigint;
    dimensions: ScreenDimensions;
    success?: boolean;
    errorMessage?: string;
};
export type ScreenInfoResponse = {
    displayId?: number;
    width?: number;
    height?: number;
    density?: number;
    name?: string;
    visible?: boolean;
    foregroundPackageName?: string;
    foregroundActivityName?: string;
    screenOn?: boolean;
    screenUnlocked?: boolean;
    accurateForegroundActivity?: string;
    accurateForegroundApp?: string;
    timestamp: bigint;
    success?: boolean;
    errorMessage?: string;
};
export type App = {
    packageName: string;
    appName: string;
    versionName: string;
    versionCode: number;
    isSystemApp: boolean;
    isEnabled: boolean;
    firstInstallTime: number;
    lastUpdateTime: number;
    sourceDir: string;
};
export type AppListResponse = {
    apps: App[];
    totalCount: number;
    systemAppCount: number;
    userAppCount: number;
    timestamp: bigint;
    success?: boolean;
    errorMessage?: string;
};
/** Screen change event response */
export type ScreenChangeEvent = {
    eventType: 'SCREEN_CHANGED';
    timestamp: bigint;
    changeRatio: number;
    imageData?: string;
};
export interface ActionSequence {
    title?: string;
    description?: string;
    actions: Array<ShellCommandAction | ClickAction | InputAction | SwipeAction | KeyAction | WaitAction | findAction | getAppInfoAction | completeAction>;
    options?: {
        continueOnError?: boolean;
        timeoutMs?: number;
    };
    deleted?: boolean;
}
export type JSONData = {
    findElement?: UIHierarchy;
    waitElement?: WAITElement;
    screenShot?: ScreenShotResponse;
    screenInfo?: ScreenInfoResponse;
    screenChange?: ScreenChangeEvent;
    appList?: AppListResponse;
    promptflowx?: ActionSequence[];
    action_event?: ClickAction | KeyAction | SwipeAction;
};
/** Position and size of a window, see the Rust version. */
export type WsWinsize = {
    x: number;
    y: number;
    rows: number;
    cols: number;
    mode?: number;
};
/** Information about a user, see the Rust version */
export type WsUser = {
    name: string;
    cursor: [number, number] | null;
    focus: number | null;
};
/** Screen capture dimensions */
export type ScreenDimensions = {
    width: number;
    height: number;
};
/** Server message type, see the Rust version. */
export type WsServer = {
    input?: string;
    hello?: Uid;
    invalidAuth?: [];
    users?: [Uid, WsUser][];
    userDiff?: [Uid, WsUser | null];
    shells?: [Sid, WsWinsize][];
    chunks?: [Sid, number, Uint8Array[]];
    hear?: [Uid, string, string];
    shellLatency?: number | bigint;
    pong?: number | bigint;
    error?: string;
    jsonData?: JSONData;
    uiClick?: [string, number, number];
    uiRefresh?: boolean;
};
/** Screen capture region */
export type ScreenRegion = {
    left: number;
    top: number;
    width: number;
    height: number;
};
/** Screen capture options */
export type ScreenShotOptions = {
    format: 'png' | 'jpeg';
    quality?: number;
    scale?: number;
    region?: ScreenRegion;
};
/** Client message type, see the Rust version. */
export type WsClient = {
    authenticate?: string;
    setName?: string;
    setCursor?: [number, number] | null;
    setFocus?: number | null;
    create?: [number, number];
    close?: Sid;
    move?: [Sid, WsWinsize | null];
    data?: [Sid, Uint8Array, bigint];
    subscribe?: [Sid, number];
    chat?: string;
    ping?: bigint;
    findElement?: findAction;
    waitElement?: WaitAction;
    screenShot?: ScreenShotOptions;
    screenInfo?: {};
    screenChange?: {
        interval?: number;
        compareThreshold?: number;
        includeScreenshot?: boolean;
        enable?: boolean;
    };
    action?: ShellXActions;
    actions?: ActionSequence;
    appInfo?: {
        packageName: string;
    };
    appList?: {
        includeSystemApps?: boolean;
        includeDisabledApps?: boolean;
    };
    promptflowx?: ActionSequence;
    switchNode?: string;
};
export type ShellXActions = {
    title?: string;
    thought?: string;
    action: ShellCommandAction | ClickAction | InputAction | SwipeAction | KeyAction | WaitAction | findAction | getAppInfoAction | completeAction;
};
interface ShellCommandAction {
    title?: string;
    type: "command";
    command: string;
}
interface getAppInfoAction {
    title?: string;
    type: "get_app_info";
    packageName: string;
}
interface ClickAction {
    title?: string;
    type: 'click';
    target: {
        type: "elementId";
        value: string;
    } | {
        type: "resourceId";
        value: string;
    } | {
        type: "coordinate";
        value: {
            "x": number;
            "y": number;
        };
    };
    options?: {
        timeoutMs?: number;
        durationMs?: number;
        waitAfterMs?: number;
        clickType?: 'normal' | 'long' | 'double';
    };
}
interface InputAction {
    title?: string;
    type: "input";
    text: string;
    target: {
        type?: "elementId";
        value?: string;
    } | {
        type: "resourceId";
        value: string;
    };
    options?: {
        waitAfterMs?: number;
        replaceExisting?: boolean;
        hideKeyboardAfter?: boolean;
    };
}
interface SwipeAction {
    title?: string;
    type: 'swipe';
    "from": {
        "x": number;
        "y": number;
    };
    "to": {
        "x": number;
        "y": number;
    };
    options?: {
        durationMs?: number;
        steps?: number;
        inertia?: boolean;
        waitAfterMs?: number;
        replaceExisting?: boolean;
        hideKeyboardAfter?: boolean;
    };
}
interface KeyAction {
    title?: string;
    type: "key";
    keyCode?: string;
    options?: {
        longPress?: boolean;
    };
}
export interface WaitAction {
    title?: string;
    type?: "wait";
    selector: ElementSelector;
    options?: WaitOptions;
    condition?: string;
}
export interface findAction {
    title?: string;
    type?: "find";
    selector: ElementSelector;
    options?: FindOptions;
}
export type completeAction = {
    title?: string;
    type: "complete";
    result?: string;
};
export type ElementSelector = {
    elementId?: string;
    className?: string;
    resourceId?: string;
    text?: string;
    textContains?: string;
    textMatches?: string;
    visible?: boolean;
    clickable?: boolean;
};
export type WaitOptions = {
    timeoutMs?: number;
    intervalMs?: number;
    failOnTimeout?: boolean;
};
export type FindOptions = {
    timeout?: number;
    visibleOnly?: boolean;
    clickableOnly?: boolean;
    multiple?: boolean;
    waitForVisible?: boolean;
    maxResults?: number;
};
export {};