@gui-agent/operator-aio/dist/AIOComputer.mjs.map

AIO (All-in-One) operator for GUI Agent

{"version":3,"file":"AIOComputer.mjs","sources":["webpack://@gui-agent/operator-aio/./src/AIOComputer.ts"],"sourcesContent":["/*\n * Copyright (c) 2025 Bytedance, Inc. and its affiliates.\n * SPDX-License-Identifier: Apache-2.0\n */\nimport { ConsoleLogger } from '@agent-infra/logger';\nimport type {\n  AIOAction,\n  ActionResponse,\n  ScreenshotResponse,\n  AIOHybridOptions,\n  MoveToAction,\n  ClickAction,\n  MouseDownAction,\n  MouseUpAction,\n  RightClickAction,\n  DoubleClickAction,\n  DragToAction,\n  ScrollAction,\n  TypingAction,\n  PressAction,\n  KeyDownAction,\n  KeyUpAction,\n  HotkeyAction,\n} from './types';\n\nconst logger = new ConsoleLogger('AIOComputer');\n\nexport class AIOComputer {\n  private baseURL: string;\n  private timeout: number;\n  private headers: Record<string, string>;\n\n  constructor(options: AIOHybridOptions) {\n    this.baseURL = options.baseURL.replace(/\\/$/, ''); // Remove trailing slash\n    this.timeout = options.timeout || 30000; // Default 30 seconds timeout\n    this.headers = {\n      'Content-Type': 'application/json',\n      ...options.headers,\n    };\n  }\n\n  /**\n   * Execute HTTP request\n   */\n  private async request(action: AIOAction): Promise<ActionResponse> {\n    const url = `${this.baseURL}/v1/browser/actions`;\n\n    try {\n      logger.info('[AIOComputer] Executing action:', action.action_type, action);\n\n      const response = await fetch(url, {\n        method: 'POST',\n        headers: this.headers,\n        body: JSON.stringify(action),\n        signal: AbortSignal.timeout(this.timeout),\n      });\n\n      if (!response.ok) {\n        throw new Error(`HTTP ${response.status}: ${response.statusText}`);\n      }\n\n      const result = await response.json();\n      logger.info('[AIOComputer] Action result:', result);\n\n      return {\n        success: true,\n        data: result,\n      };\n    } catch (error) {\n      logger.error('[AIOComputer] Action failed:', error);\n      return {\n        success: false,\n        message: error instanceof Error ? error.message : 'Unknown error',\n      };\n    }\n  }\n\n  /**\n   * Take screenshot with optional delay\n   * @param delay Optional delay in milliseconds before taking screenshot\n   */\n  async screenshot(delay = 1000): Promise<ScreenshotResponse> {\n    const url = `${this.baseURL}/v1/browser/screenshot`;\n\n    try {\n      logger.info('[AIOComputer] Taking screenshot' + (delay ? ` with ${delay}ms delay` : ''));\n\n      if (delay > 0) {\n        await new Promise((resolve) => setTimeout(resolve, delay));\n      }\n\n      const response = await fetch(url, {\n        method: 'GET',\n        headers: this.headers,\n        signal: AbortSignal.timeout(this.timeout),\n      });\n\n      if (!response.ok) {\n        throw new Error(`HTTP ${response.status}: ${response.statusText}`);\n      }\n\n      // Check if response is image data\n      const contentType = response.headers.get('content-type');\n      if (contentType && contentType.startsWith('image/')) {\n        // If response is image data, convert to base64\n        const arrayBuffer = await response.arrayBuffer();\n        const base64 = Buffer.from(arrayBuffer).toString('base64');\n\n        logger.info('[AIOComputer] Screenshot taken successfully');\n\n        return {\n          success: true,\n          data: {\n            base64,\n            scaleFactor: 1, // 默认缩放因子，可能需要从API获取实际值\n            contentType,\n          },\n        };\n      } else {\n        // If response is JSON\n        const result = await response.json();\n        logger.info('[AIOComputer] Screenshot result:', result);\n\n        return {\n          success: true,\n          data: result,\n        };\n      }\n    } catch (error) {\n      logger.error('[AIOComputer] Screenshot failed:', error);\n      return {\n        success: false,\n        message: error instanceof Error ? error.message : 'Unknown error',\n      };\n    }\n  }\n\n  /**\n   * Move mouse to specified position\n   */\n  async moveTo(x: number, y: number): Promise<ActionResponse> {\n    const action: MoveToAction = {\n      action_type: 'MOVE_TO',\n      x,\n      y,\n    };\n    return this.request(action);\n  }\n\n  /**\n   * Click operation\n   */\n  async click(\n    x?: number,\n    y?: number,\n    button?: string,\n    numClicks?: number,\n  ): Promise<ActionResponse> {\n    const action: ClickAction = {\n      action_type: 'CLICK',\n      ...(x !== undefined && { x }),\n      ...(y !== undefined && { y }),\n      ...(button && { button }),\n      ...(numClicks && { num_clicks: numClicks }),\n    };\n    return this.request(action);\n  }\n\n  /**\n   * Press mouse button\n   */\n  async mouseDown(button?: string): Promise<ActionResponse> {\n    const action: MouseDownAction = {\n      action_type: 'MOUSE_DOWN',\n      ...(button && { button }),\n    };\n    return this.request(action);\n  }\n\n  /**\n   * Release mouse button\n   */\n  async mouseUp(button?: string): Promise<ActionResponse> {\n    const action: MouseUpAction = {\n      action_type: 'MOUSE_UP',\n      ...(button && { button }),\n    };\n    return this.request(action);\n  }\n\n  /**\n   * Right click\n   */\n  async rightClick(x?: number, y?: number): Promise<ActionResponse> {\n    const action: RightClickAction = {\n      action_type: 'RIGHT_CLICK',\n      ...(x !== undefined && { x }),\n      ...(y !== undefined && { y }),\n    };\n    return this.request(action);\n  }\n\n  /**\n   * Double click\n   */\n  async doubleClick(x?: number, y?: number): Promise<ActionResponse> {\n    const action: DoubleClickAction = {\n      action_type: 'DOUBLE_CLICK',\n      ...(x !== undefined && { x }),\n      ...(y !== undefined && { y }),\n    };\n    return this.request(action);\n  }\n\n  /**\n   * Drag to specified position\n   */\n  async dragTo(x: number, y: number): Promise<ActionResponse> {\n    const action: DragToAction = {\n      action_type: 'DRAG_TO',\n      x,\n      y,\n    };\n    return this.request(action);\n  }\n\n  /**\n   * Scroll operation\n   */\n  async scroll(dx?: number, dy?: number): Promise<ActionResponse> {\n    const action: ScrollAction = {\n      action_type: 'SCROLL',\n      ...(dx !== undefined && { dx }),\n      ...(dy !== undefined && { dy }),\n    };\n    return this.request(action);\n  }\n\n  /**\n   * Type text\n   */\n  async type(text: string): Promise<ActionResponse> {\n    const action: TypingAction = {\n      action_type: 'TYPING',\n      text,\n    };\n    return this.request(action);\n  }\n\n  /**\n   * Press key\n   */\n  async press(key: string): Promise<ActionResponse> {\n    const action: PressAction = {\n      action_type: 'PRESS',\n      key,\n    };\n    return this.request(action);\n  }\n\n  /**\n   * Press down key\n   */\n  async keyDown(key: string): Promise<ActionResponse> {\n    const action: KeyDownAction = {\n      action_type: 'KEY_DOWN',\n      key,\n    };\n    return this.request(action);\n  }\n\n  /**\n   * Release key\n   */\n  async keyUp(key: string): Promise<ActionResponse> {\n    const action: KeyUpAction = {\n      action_type: 'KEY_UP',\n      key,\n    };\n    return this.request(action);\n  }\n\n  /**\n   * Hotkey combination\n   */\n  async hotkey(keys: string[]): Promise<ActionResponse> {\n    // Convert all keys to lowercase\n    const lowercaseKeys = keys.map((key: string) => key.toLowerCase());\n    const action: HotkeyAction = {\n      action_type: 'HOTKEY',\n      keys: lowercaseKeys,\n    };\n    return this.request(action);\n  }\n\n  /**\n   * Generic execute method\n   */\n  async execute(action: AIOAction): Promise<ActionResponse> {\n    return this.request(action);\n  }\n}\n"],"names":["logger","ConsoleLogger","AIOComputer","action","url","response","fetch","JSON","AbortSignal","Error","result","error","delay","Promise","resolve","setTimeout","contentType","arrayBuffer","base64","Buffer","x","y","button","numClicks","undefined","dx","dy","text","key","keys","lowercaseKeys","options"],"mappings":";;;;;AAGC;;;;;;;;;;AAsBD,MAAMA,SAAS,IAAIC,cAAc;AAE1B,MAAMC;IAiBX,MAAc,QAAQC,MAAiB,EAA2B;QAChE,MAAMC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC;QAEhD,IAAI;YACFJ,OAAO,IAAI,CAAC,mCAAmCG,OAAO,WAAW,EAAEA;YAEnE,MAAME,WAAW,MAAMC,MAAMF,KAAK;gBAChC,QAAQ;gBACR,SAAS,IAAI,CAAC,OAAO;gBACrB,MAAMG,KAAK,SAAS,CAACJ;gBACrB,QAAQK,YAAY,OAAO,CAAC,IAAI,CAAC,OAAO;YAC1C;YAEA,IAAI,CAACH,SAAS,EAAE,EACd,MAAM,IAAII,MAAM,CAAC,KAAK,EAAEJ,SAAS,MAAM,CAAC,EAAE,EAAEA,SAAS,UAAU,EAAE;YAGnE,MAAMK,SAAS,MAAML,SAAS,IAAI;YAClCL,OAAO,IAAI,CAAC,gCAAgCU;YAE5C,OAAO;gBACL,SAAS;gBACT,MAAMA;YACR;QACF,EAAE,OAAOC,OAAO;YACdX,OAAO,KAAK,CAAC,gCAAgCW;YAC7C,OAAO;gBACL,SAAS;gBACT,SAASA,iBAAiBF,QAAQE,MAAM,OAAO,GAAG;YACpD;QACF;IACF;IAMA,MAAM,WAAWC,QAAQ,IAAI,EAA+B;QAC1D,MAAMR,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC;QAEnD,IAAI;YACFJ,OAAO,IAAI,CAAC,oCAAqCY,CAAAA,QAAQ,CAAC,MAAM,EAAEA,MAAM,QAAQ,CAAC,GAAG,EAAC;YAErF,IAAIA,QAAQ,GACV,MAAM,IAAIC,QAAQ,CAACC,UAAYC,WAAWD,SAASF;YAGrD,MAAMP,WAAW,MAAMC,MAAMF,KAAK;gBAChC,QAAQ;gBACR,SAAS,IAAI,CAAC,OAAO;gBACrB,QAAQI,YAAY,OAAO,CAAC,IAAI,CAAC,OAAO;YAC1C;YAEA,IAAI,CAACH,SAAS,EAAE,EACd,MAAM,IAAII,MAAM,CAAC,KAAK,EAAEJ,SAAS,MAAM,CAAC,EAAE,EAAEA,SAAS,UAAU,EAAE;YAInE,MAAMW,cAAcX,SAAS,OAAO,CAAC,GAAG,CAAC;YACzC,IAAIW,eAAeA,YAAY,UAAU,CAAC,WAAW;gBAEnD,MAAMC,cAAc,MAAMZ,SAAS,WAAW;gBAC9C,MAAMa,SAASC,OAAO,IAAI,CAACF,aAAa,QAAQ,CAAC;gBAEjDjB,OAAO,IAAI,CAAC;gBAEZ,OAAO;oBACL,SAAS;oBACT,MAAM;wBACJkB;wBACA,aAAa;wBACbF;oBACF;gBACF;YACF;YAAO;gBAEL,MAAMN,SAAS,MAAML,SAAS,IAAI;gBAClCL,OAAO,IAAI,CAAC,oCAAoCU;gBAEhD,OAAO;oBACL,SAAS;oBACT,MAAMA;gBACR;YACF;QACF,EAAE,OAAOC,OAAO;YACdX,OAAO,KAAK,CAAC,oCAAoCW;YACjD,OAAO;gBACL,SAAS;gBACT,SAASA,iBAAiBF,QAAQE,MAAM,OAAO,GAAG;YACpD;QACF;IACF;IAKA,MAAM,OAAOS,CAAS,EAAEC,CAAS,EAA2B;QAC1D,MAAMlB,SAAuB;YAC3B,aAAa;YACbiB;YACAC;QACF;QACA,OAAO,IAAI,CAAC,OAAO,CAAClB;IACtB;IAKA,MAAM,MACJiB,CAAU,EACVC,CAAU,EACVC,MAAe,EACfC,SAAkB,EACO;QACzB,MAAMpB,SAAsB;YAC1B,aAAa;YACb,GAAIiB,AAAMI,WAANJ,KAAmB;gBAAEA;YAAE,CAAC;YAC5B,GAAIC,AAAMG,WAANH,KAAmB;gBAAEA;YAAE,CAAC;YAC5B,GAAIC,UAAU;gBAAEA;YAAO,CAAC;YACxB,GAAIC,aAAa;gBAAE,YAAYA;YAAU,CAAC;QAC5C;QACA,OAAO,IAAI,CAAC,OAAO,CAACpB;IACtB;IAKA,MAAM,UAAUmB,MAAe,EAA2B;QACxD,MAAMnB,SAA0B;YAC9B,aAAa;YACb,GAAImB,UAAU;gBAAEA;YAAO,CAAC;QAC1B;QACA,OAAO,IAAI,CAAC,OAAO,CAACnB;IACtB;IAKA,MAAM,QAAQmB,MAAe,EAA2B;QACtD,MAAMnB,SAAwB;YAC5B,aAAa;YACb,GAAImB,UAAU;gBAAEA;YAAO,CAAC;QAC1B;QACA,OAAO,IAAI,CAAC,OAAO,CAACnB;IACtB;IAKA,MAAM,WAAWiB,CAAU,EAAEC,CAAU,EAA2B;QAChE,MAAMlB,SAA2B;YAC/B,aAAa;YACb,GAAIiB,AAAMI,WAANJ,KAAmB;gBAAEA;YAAE,CAAC;YAC5B,GAAIC,AAAMG,WAANH,KAAmB;gBAAEA;YAAE,CAAC;QAC9B;QACA,OAAO,IAAI,CAAC,OAAO,CAAClB;IACtB;IAKA,MAAM,YAAYiB,CAAU,EAAEC,CAAU,EAA2B;QACjE,MAAMlB,SAA4B;YAChC,aAAa;YACb,GAAIiB,AAAMI,WAANJ,KAAmB;gBAAEA;YAAE,CAAC;YAC5B,GAAIC,AAAMG,WAANH,KAAmB;gBAAEA;YAAE,CAAC;QAC9B;QACA,OAAO,IAAI,CAAC,OAAO,CAAClB;IACtB;IAKA,MAAM,OAAOiB,CAAS,EAAEC,CAAS,EAA2B;QAC1D,MAAMlB,SAAuB;YAC3B,aAAa;YACbiB;YACAC;QACF;QACA,OAAO,IAAI,CAAC,OAAO,CAAClB;IACtB;IAKA,MAAM,OAAOsB,EAAW,EAAEC,EAAW,EAA2B;QAC9D,MAAMvB,SAAuB;YAC3B,aAAa;YACb,GAAIsB,AAAOD,WAAPC,MAAoB;gBAAEA;YAAG,CAAC;YAC9B,GAAIC,AAAOF,WAAPE,MAAoB;gBAAEA;YAAG,CAAC;QAChC;QACA,OAAO,IAAI,CAAC,OAAO,CAACvB;IACtB;IAKA,MAAM,KAAKwB,IAAY,EAA2B;QAChD,MAAMxB,SAAuB;YAC3B,aAAa;YACbwB;QACF;QACA,OAAO,IAAI,CAAC,OAAO,CAACxB;IACtB;IAKA,MAAM,MAAMyB,GAAW,EAA2B;QAChD,MAAMzB,SAAsB;YAC1B,aAAa;YACbyB;QACF;QACA,OAAO,IAAI,CAAC,OAAO,CAACzB;IACtB;IAKA,MAAM,QAAQyB,GAAW,EAA2B;QAClD,MAAMzB,SAAwB;YAC5B,aAAa;YACbyB;QACF;QACA,OAAO,IAAI,CAAC,OAAO,CAACzB;IACtB;IAKA,MAAM,MAAMyB,GAAW,EAA2B;QAChD,MAAMzB,SAAsB;YAC1B,aAAa;YACbyB;QACF;QACA,OAAO,IAAI,CAAC,OAAO,CAACzB;IACtB;IAKA,MAAM,OAAO0B,IAAc,EAA2B;QAEpD,MAAMC,gBAAgBD,KAAK,GAAG,CAAC,CAACD,MAAgBA,IAAI,WAAW;QAC/D,MAAMzB,SAAuB;YAC3B,aAAa;YACb,MAAM2B;QACR;QACA,OAAO,IAAI,CAAC,OAAO,CAAC3B;IACtB;IAKA,MAAM,QAAQA,MAAiB,EAA2B;QACxD,OAAO,IAAI,CAAC,OAAO,CAACA;IACtB;IA5QA,YAAY4B,OAAyB,CAAE;QAJvC,uBAAQ,WAAR;QACA,uBAAQ,WAAR;QACA,uBAAQ,WAAR;QAGE,IAAI,CAAC,OAAO,GAAGA,QAAQ,OAAO,CAAC,OAAO,CAAC,OAAO;QAC9C,IAAI,CAAC,OAAO,GAAGA,QAAQ,OAAO,IAAI;QAClC,IAAI,CAAC,OAAO,GAAG;YACb,gBAAgB;YAChB,GAAGA,QAAQ,OAAO;QACpB;IACF;AAsQF"}