smooth-operator-agent-tools

/** * Type definitions for Smooth Operator Agent Tools */ /** * Specifies the mechanism to use for AI-based UI element interaction. */ export declare enum MechanismType { ScreenGrasp2 = "screengrasp2", ScreenGrasp2Low = "screengrasp2-low", ScreenGraspMedium = "screengrasp-medium", ScreenGraspHigh = "screengrasp-high", LLabs = "llabs", AnthropicComputerUse = "anthropic-computer-use", OpenAIComputerUse = "openai-computer-use", Qwen25Vl72b = "qwen25-vl-72b" } type ControlDTOPlain = { id: string; name: string | null; creationDate: string; controlType: string | null; supportsSetValue: boolean | null; supportsInvoke: boolean | null; currentValue: string | null; children: (ControlDTOPlain | null)[] | null; isSmoothOperator: boolean; }; /** * Response from the screenshot endpoint */ export declare class ScreenshotResponse { /** Whether the operation was successful */ success: boolean; /** Base64-encoded image data */ imageBase64: string; /** Timestamp of when the screenshot was taken (ISO 8601 format) */ timestamp: string; /** Message describing the result */ message: string | null; constructor(data: { success: boolean; imageBase64: string; timestamp: string; message?: string | null; }); /** Raw image bytes */ get imageBytes(): Buffer; /** Mime type of the image */ get imageMimeType(): string; toJSON(): any; } /** * Represents a point on the screen with X and Y coordinates */ export declare class Point { /** X coordinate */ x: number; /** Y coordinate */ y: number; constructor(data: { x: number; y: number; }); toJSON(): any; } /** * Generic response for action endpoints */ export declare class ActionResponse { /** Whether the operation was successful */ success: boolean; /** Message describing the result */ message: string | null; /** Additional data (if applicable) */ data: Record<string, any> | null; constructor(data: { success: boolean; message?: string | null; data?: Record<string, any> | null; }); toJSON(): any; } /** * Response from the ScreenGrasp2 endpoint (find-ui-element-by-description) */ export declare class ScreenGrasp2Response extends ActionResponse { /** X coordinate of the found element (if applicable) */ x: number | null; /** Y coordinate of the found element (if applicable) */ y: number | null; /** Status message from the underlying service */ status: string | null; constructor(data: { success: boolean; message?: string | null; data?: Record<string, any> | null; x?: number | null; y?: number | null; status?: string | null; }); toJSON(): any; } /** * Information about a Chrome browser tab */ export declare class ChromeTab { /** Tab ID */ id: string; /** Tab title */ title: string; /** Tab URL */ url: string; /** Whether the tab is active */ isActive: boolean; constructor(data: { id: string; title: string; url: string; isActive: boolean; }); toJSON(): any; } /** * Detailed information about a Chrome tab */ export declare class ChromeTabDetails { /** Tab title */ title: string; /** Tab URL */ url: string; /** Page content */ content: string; /** List of elements on the page */ elements: Record<string, any>[]; /** Summary of the page content */ summary: string; constructor(data: { title: string; url: string; content: string; elements: Record<string, any>[]; summary: string; }); toJSON(): any; } /** * Response from Chrome script execution */ export declare class ChromeScriptResponse extends ActionResponse { /** Result of the script execution */ result: string | null; constructor(data: { success: boolean; message?: string | null; data?: Record<string, any> | null; result?: string | null; }); toJSON(): any; } /** * Response from C# code execution */ export declare class CSharpCodeResponse extends ActionResponse { /** Result of the code execution */ result: string | null; constructor(data: { success: boolean; message?: string | null; data?: Record<string, any> | null; result?: string | null; }); toJSON(): any; } /** * Simple response with a message */ export declare class SimpleResponse { /** Message describing the result */ message: string | null; /** Internal message (not usually exposed to users) */ internalMessage: string | null; constructor(data: { message?: string | null; internalMessage?: string | null; }); toJSON(): any; } /** * Information about a desktop icon */ export declare class DesktopIconDTO { /** Icon name */ name: string; /** Path to the icon's target */ path: string; constructor(data: { name: string; path: string; }); toJSON(): any; } /** * Information about a taskbar icon */ export declare class TaskbarIconDTO { /** Icon name */ name: string; /** Path to the icon's target */ path: string; constructor(data: { name: string; path: string; }); toJSON(): any; } /** * Information about an installed program */ export declare class InstalledProgramDTO { /** Program name */ name: string; /** Path to the executable */ executablePath: string; constructor(data: { name: string; executablePath: string; }); toJSON(): any; } /** * Information about a UI control */ export declare class ControlDTO { /** Control ID */ id: string; /** Control name */ name: string | null; /** Creation date (ISO 8601 format) */ creationDate: string; /** Control type */ controlType: string | null; /** Whether the control supports setting a value */ supportsSetValue: boolean | null; /** Whether the control supports invoking */ supportsInvoke: boolean | null; /** Current value of the control */ currentValue: string | null; /** Child controls */ children: ControlDTO[] | null; /** Parent control */ parent: ControlDTO | null; /** Whether this is a Smooth Operator control */ isSmoothOperator: boolean; constructor(data: { id: string; creationDate: string; isSmoothOperator: boolean; name?: string | null; controlType?: string | null; supportsSetValue?: boolean | null; supportsInvoke?: boolean | null; currentValue?: string | null; children?: ControlDTO[] | null; parent?: ControlDTO | null; }); /** Recursively get all descendant controls. */ get childrenRecursive(): ControlDTO[]; /** Get all ancestor controls. */ get parentsRecursive(): ControlDTO[]; /** Find the closest ancestor control that is a Window. */ get parentWindow(): ControlDTO | null; /** Custom serialization to avoid circular references and large payloads */ toJSON(): ControlDTOPlain; } /** * Information about a window */ export declare class WindowInfoDTO { /** Window ID */ id: string; /** Window title */ title: string | null; /** Path to the executable */ executablePath: string | null; /** Whether the window is in the foreground */ isForeground: boolean | null; /** Process name */ processName: string | null; /** Whether the window is minimized */ isMinimized: boolean | null; /** Detailed information about the window */ detailInfos: WindowDetailResponse | null; constructor(data: { id: string; title?: string | null; executablePath?: string | null; isForeground?: boolean | null; processName?: string | null; isMinimized?: boolean | null; detailInfos?: any | null; }); toJSON(): any; } /** * Response containing window details */ export declare class WindowDetailResponse { /** Detailed information about the window */ details: WindowDetailInfosDTO | null; /** Message describing the result */ message: string | null; constructor(data: { details?: any | null; message?: string | null; }); toJSON(): any; } /** * Detailed information about a window */ export declare class WindowDetailInfosDTO { /** Note about the window */ note: string | null; /** Window information */ window: WindowInfoDTO; /** User interface elements in the window */ userInterfaceElements: ControlDTO; constructor(data: { window: any; userInterfaceElements: any; note?: string | null; }); toJSON(): any; } /** * Information about a Chrome instance */ export declare class ChromeOverview { /** Instance ID */ instanceId: string; /** Tabs in the Chrome instance */ tabs: TabData[]; /** Last update timestamp (ISO 8601 format) */ lastUpdate: string; constructor(data: { instanceId: string; tabs?: any[] | null; lastUpdate: string; }); toJSON(): any; } /** * Information about a Chrome tab */ export declare class TabData { /** Tab ID */ id: string; /** Tab URL */ url: string | null; /** Whether the tab is active */ isActive: boolean | null; /** HTML content of the tab */ html: string | null; /** Text content of the tab */ text: string | null; /** ID string */ idString: string | null; /** Tab number */ tabNr: number; constructor(data: { id: string; tabNr: number; url?: string | null; isActive?: boolean | null; html?: string | null; text?: string | null; idString?: string | null; }); toJSON(): any; } /** * Information about the focused element */ export declare class FocusInformation { /** Focused element */ focusedElement: ControlDTO | null; /** Parent window of the focused element */ focusedElementParentWindow: WindowInfoDTO | null; /** Other elements in the same window that might be relevant */ someOtherElementsInSameWindowThatMightBeRelevant: ControlDTO[] | null; /** Most relevant elements in the current Chrome tab */ currentChromeTabMostRelevantElements: ChromeElementInfo[] | null; /** Whether the focused window is Chrome */ isChrome: boolean; /** Note about the focus */ note: string | null; constructor(data: { isChrome: boolean; focusedElement?: any | null; focusedElementParentWindow?: any | null; someOtherElementsInSameWindowThatMightBeRelevant?: any[] | null; currentChromeTabMostRelevantElements?: any[] | null; note?: string | null; }); toJSON(): any; } /** * Information about a Chrome element */ export declare class ChromeElementInfo { /** Smooth Operator ID */ smoothOpId: string; /** HTML tag name */ tagName: string; /** CSS selector */ cssSelector: string; /** Inner text */ innerText: string | null; /** Whether the element is visible */ isVisible: boolean | null; /** Relevance score */ score: number; /** ARIA role */ role: string | null; /** Element value */ value: string | null; /** Element type */ type: string | null; /** Element name */ name: string | null; /** CSS class name */ className: string | null; /** Semantic meaning */ semantic: string | null; /** Data attributes */ dataAttributes: string | null; /** Truncated HTML */ truncatedHtml: string | null; /** Bounding rectangle [x, y, width, height] */ boundingRect: number[] | null; /** Center point */ centerPoint: Point | null; constructor(data: { smoothOpId: string; tagName: string; cssSelector: string; score: number; innerText?: string | null; isVisible?: boolean | null; role?: string | null; value?: string | null; type?: string | null; name?: string | null; className?: string | null; semantic?: string | null; dataAttributes?: string | null; truncatedHtml?: string | null; boundingRect?: number[] | null; centerPoint?: any | null; }); toJSON(): any; } /** * System overview response */ export declare class OverviewResponse { /** Windows in the system */ windows: WindowInfoDTO[]; /** Chrome instances */ chromeInstances: ChromeOverview[]; /** Focus information */ focusInfo: FocusInformation | null; /** Top pinned taskbar icons */ topPinnedTaskbarIcons: TaskbarIconDTO[]; /** Top desktop icons */ topDesktopIcons: DesktopIconDTO[]; /** Top installed programs */ topInstalledPrograms: InstalledProgramDTO[]; /** Important note */ importantNote: string | null; constructor(data: { windows?: any[] | null; chromeInstances?: any[] | null; focusInfo?: any | null; topPinnedTaskbarIcons?: any[] | null; topDesktopIcons?: any[] | null; topInstalledPrograms?: any[] | null; importantNote?: string | null; }); toJSON(): any; } export {};