UNPKG

@gui-agent/operator-aio

Version:
1 lines 17.5 kB
{"version":3,"file":"AIOHybridOperator.mjs","sources":["webpack://@gui-agent/operator-aio/./src/AIOHybridOperator.ts"],"sourcesContent":["/*\n * Copyright (c) 2025 Bytedance, Inc. and its affiliates.\n * SPDX-License-Identifier: Apache-2.0\n */\nimport {\n type ScreenshotOutput,\n type ExecuteParams,\n type ExecuteOutput,\n Operator,\n StatusEnum,\n} from '@ui-tars/sdk/core';\nimport { ConsoleLogger } from '@agent-infra/logger';\nimport { Base64ImageParser } from '@agent-infra/media-utils';\nimport { sleep } from '@ui-tars/shared/utils';\nimport { parseBoxToScreenCoords } from './utils';\nimport { AIOComputer } from './AIOComputer';\nimport type { AIOHybridOptions } from './types';\nimport { AIOBrowser } from './AIOBrowser';\nimport { log } from 'console';\n\nconst logger = new ConsoleLogger('AioHybridOperator');\n\nexport class AIOHybridOperator extends Operator {\n static MANUAL = {\n ACTION_SPACES: [\n `click(start_box='[x1, y1, x2, y2]')`,\n `left_double(start_box='[x1, y1, x2, y2]')`,\n `right_single(start_box='[x1, y1, x2, y2]')`,\n `drag(start_box='[x1, y1, x2, y2]', end_box='[x3, y3, x4, y4]')`,\n `hotkey(key='')`,\n `type(content='') #If you want to submit your input, use \"\\\\n\" at the end of \\`content\\`.`,\n `scroll(start_box='[x1, y1, x2, y2]', direction='down or up or right or left')`,\n `wait() #Sleep for 5s and take a screenshot to check for any changes.`,\n `finished()`,\n `call_user() # Submit the task and call the user when the task is unsolvable, or when you need the user's help.`,\n ],\n };\n\n private static currentInstance: AIOHybridOperator | null = null;\n private aioBrowser: AIOBrowser | null = null;\n private aioComputer: AIOComputer;\n\n private screenshotWidth = 1280;\n private screenshotHeight = 1024;\n\n public static async create(options: AIOHybridOptions): Promise<AIOHybridOperator> {\n logger.info('[AioHybridOperator] construct:', options.baseURL);\n const instance = new AIOHybridOperator(options);\n await instance.initialize(options);\n this.currentInstance = instance;\n return instance;\n }\n\n private constructor(options: AIOHybridOptions) {\n super();\n this.aioComputer = new AIOComputer(options);\n }\n\n private async initialize(options: AIOHybridOptions): Promise<void> {\n this.aioComputer.screenshot(0); // Ping the aio sandbox\n this.aioBrowser = await AIOBrowser.create({\n baseURl: options.baseURL,\n logger: logger,\n });\n await this.aioBrowser?.launch({\n timeout: 1000,\n defaultViewport: { width: 1280, height: 1024 },\n });\n logger.info('[AioHybridOperator] AIOBrowser launched successfully');\n logger.info('[AioHybridOperator] AIOBrowser initialized successfully');\n }\n\n public async getMeta(): Promise<{ url: string }> {\n let url = '';\n try {\n const retUrl = await this.aioBrowser?.getActiveUrl();\n if (retUrl) {\n url = retUrl;\n }\n } catch (error) {\n logger.error('Failed to get page meta:', error);\n }\n return {\n url,\n };\n }\n\n public async screenshot(): Promise<ScreenshotOutput> {\n logger.info('[AioHybridOperator] Taking screenshot');\n\n try {\n const result = await this.aioComputer.screenshot();\n\n if (!result.success) {\n throw new Error(result.message || 'Screenshot failed');\n }\n\n // Convert the response to ScreenshotOutput format expected by the SDK\n if (result.data?.base64) {\n const base64Tool = new Base64ImageParser(result.data?.base64);\n const dimensions = base64Tool.getDimensions();\n if (dimensions) {\n this.screenshotWidth = dimensions?.width;\n this.screenshotHeight = dimensions?.height;\n }\n logger.info('[AioHybridOperator] screenshot dimensions:', JSON.stringify(dimensions));\n return {\n base64: result.data.base64,\n scaleFactor: result.data.scaleFactor || 1,\n };\n } else {\n throw new Error('No base64 image data received from screenshot API');\n }\n } catch (error) {\n logger.error('[AioHybridOperator] Screenshot failed:', error);\n throw error;\n }\n }\n\n async execute(params: ExecuteParams): Promise<ExecuteOutput> {\n const { parsedPrediction, screenWidth, screenHeight, scaleFactor } = params;\n const { action_type, action_inputs } = parsedPrediction;\n const startBoxStr = action_inputs?.start_box || '';\n\n logger.info(\n '[AioHybridOperator] Executing action',\n action_type,\n action_inputs,\n ', screen context',\n this.screenshotWidth,\n this.screenshotHeight,\n );\n\n const {\n x: rawX,\n y: rawY,\n percentX: rawPercentX,\n percentY: rawPercentY,\n } = parseBoxToScreenCoords({\n boxStr: startBoxStr,\n screenWidth: this.screenshotWidth,\n screenHeight: this.screenshotHeight,\n factors: [1000, 1000],\n });\n\n const startX = rawX !== null ? Math.round(rawX) : null;\n const startY = rawY !== null ? Math.round(rawY) : null;\n\n logger.info(`[AioHybridOperator] Action position: (${startX}, ${startY})`);\n logger.info(\n `[AioHybridOperator] Action position percent raw: (${rawPercentX}, ${rawPercentY})`,\n );\n\n let startXPercent = null,\n startYPercent = null;\n\n try {\n switch (action_type) {\n case 'navigate':\n logger.info('[AioHybridOperator] Navigating to', action_inputs?.content);\n await this.aioBrowser?.handleNavigate({ url: action_inputs?.content || '' });\n break;\n case 'navigate_back':\n logger.info('[AioHybridOperator] Navigating back');\n await this.aioBrowser?.handleNavigateBack();\n break;\n case 'wait':\n logger.info('[AioHybridOperator] Waiting for 5 seconds');\n await sleep(5000);\n break;\n\n case 'mouse_move':\n case 'hover':\n if (startX !== null && startY !== null) {\n await this.aioComputer.moveTo(startX, startY);\n startXPercent = rawPercentX;\n startYPercent = rawPercentY;\n }\n break;\n\n case 'click':\n case 'left_click':\n case 'left_single':\n if (startX !== null && startY !== null) {\n await this.aioComputer.click(startX, startY);\n startXPercent = rawPercentX;\n startYPercent = rawPercentY;\n }\n break;\n\n case 'left_double':\n case 'double_click':\n if (startX !== null && startY !== null) {\n await this.aioComputer.doubleClick(startX, startY);\n startXPercent = rawPercentX;\n startYPercent = rawPercentY;\n }\n break;\n\n case 'right_click':\n case 'right_single':\n if (startX !== null && startY !== null) {\n await this.aioComputer.rightClick(startX, startY);\n startXPercent = rawPercentX;\n startYPercent = rawPercentY;\n }\n break;\n\n case 'middle_click':\n if (startX !== null && startY !== null) {\n await this.aioComputer.click(startX, startY, 'middle');\n startXPercent = rawPercentX;\n startYPercent = rawPercentY;\n }\n break;\n\n case 'left_click_drag':\n case 'drag':\n case 'select': {\n if (action_inputs?.end_box) {\n const { x: rawEndX, y: rawEndY } = parseBoxToScreenCoords({\n boxStr: action_inputs.end_box,\n screenWidth,\n screenHeight,\n });\n const endX = rawEndX !== null ? Math.round(rawEndX) : null;\n const endY = rawEndY !== null ? Math.round(rawEndY) : null;\n\n if (startX && startY && endX && endY) {\n // Move to start position, press mouse, drag to end position, release mouse\n await this.aioComputer.moveTo(startX, startY);\n await this.aioComputer.mouseDown();\n await this.aioComputer.dragTo(endX, endY);\n await this.aioComputer.mouseUp();\n }\n }\n break;\n }\n\n case 'type': {\n const content = action_inputs.content?.trim();\n if (content) {\n const stripContent = content.replace(/\\\\n$/, '').replace(/\\n$/, '');\n await this.aioComputer.type(stripContent);\n }\n break;\n }\n\n case 'hotkey':\n case 'press': {\n const keyStr = action_inputs?.key || action_inputs?.hotkey;\n if (keyStr) {\n // 处理组合键\n const keys = keyStr.split(/[\\s+]/).filter((k) => k.length > 0);\n if (keys.length > 1) {\n await this.aioComputer.hotkey(keys);\n } else {\n await this.aioComputer.press(keyStr);\n }\n }\n break;\n }\n\n case 'scroll': {\n const { direction } = action_inputs;\n if (startX !== null && startY !== null && direction) {\n const normalizedDirection = direction.toLowerCase();\n let dx = 0,\n dy = 0;\n\n switch (normalizedDirection) {\n case 'up':\n dy = 10;\n break;\n case 'down':\n dy = -10;\n break;\n case 'left':\n dx = 10;\n break;\n case 'right':\n dx = -10;\n break;\n }\n\n if (dx !== 0 || dy !== 0) {\n await this.aioComputer.scroll(dx, dy);\n }\n }\n break;\n }\n\n case 'error_env':\n case 'call_user':\n case 'finished':\n case 'user_stop':\n break;\n\n default:\n logger.warn(`Unsupported action type: ${action_type}`);\n }\n\n // const { startXPercent, startYPercent } = parseBoxToScreenCoordsPercent({\n // startX,\n // startY,\n // screenWidth,\n // screenHeight,\n // deviceScaleFactor: scaleFactor,\n // });\n logger.info(\n `[AioHybridOperator] position percent return: (${startXPercent}, ${startYPercent})`,\n );\n\n // return { status: StatusEnum.INIT };\n return {\n // Hand it over to the upper layer to avoid redundancy\n // @ts-expect-error fix type later\n startX,\n startY,\n // Add percentage coordinates for new GUI Agent design\n startXPercent,\n startYPercent,\n action_inputs,\n };\n } catch (error) {\n logger.error('[AioHybridOperator] 执行失败:', error);\n return { status: StatusEnum.ERROR };\n }\n }\n}\n"],"names":["logger","ConsoleLogger","AIOHybridOperator","Operator","options","instance","_this_aioBrowser","AIOBrowser","url","retUrl","error","_result_data","result","Error","_result_data1","base64Tool","Base64ImageParser","dimensions","JSON","params","parsedPrediction","screenWidth","screenHeight","scaleFactor","action_type","action_inputs","startBoxStr","rawX","rawY","rawPercentX","rawPercentY","parseBoxToScreenCoords","startX","Math","startY","startXPercent","startYPercent","_this_aioBrowser1","sleep","rawEndX","rawEndY","endX","endY","_action_inputs_content","content","stripContent","keyStr","keys","k","direction","normalizedDirection","dx","dy","StatusEnum","AIOComputer"],"mappings":";;;;;;;;;;;AAGC;;;;;;;;;;AAiBD,MAAMA,SAAS,IAAIC,cAAc;AAE1B,MAAMC,0BAA0BC;IAuBrC,aAAoB,OAAOC,OAAyB,EAA8B;QAChFJ,OAAO,IAAI,CAAC,kCAAkCI,QAAQ,OAAO;QAC7D,MAAMC,WAAW,IAAIH,kBAAkBE;QACvC,MAAMC,SAAS,UAAU,CAACD;QAC1B,IAAI,CAAC,eAAe,GAAGC;QACvB,OAAOA;IACT;IAOA,MAAc,WAAWD,OAAyB,EAAiB;YAM3DE;QALN,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC;QAC5B,IAAI,CAAC,UAAU,GAAG,MAAMC,WAAW,MAAM,CAAC;YACxC,SAASH,QAAQ,OAAO;YACxB,QAAQJ;QACV;QACA,eAAMM,CAAAA,mBAAAA,IAAI,CAAC,UAAU,AAAD,IAAdA,KAAAA,IAAAA,iBAAiB,MAAM,CAAC;YAC5B,SAAS;YACT,iBAAiB;gBAAE,OAAO;gBAAM,QAAQ;YAAK;QAC/C,EAAC;QACDN,OAAO,IAAI,CAAC;QACZA,OAAO,IAAI,CAAC;IACd;IAEA,MAAa,UAAoC;QAC/C,IAAIQ,MAAM;QACV,IAAI;gBACmBF;YAArB,MAAMG,SAAS,eAAMH,CAAAA,mBAAAA,IAAI,CAAC,UAAU,AAAD,IAAdA,KAAAA,IAAAA,iBAAiB,YAAY,EAAC;YACnD,IAAIG,QACFD,MAAMC;QAEV,EAAE,OAAOC,OAAO;YACdV,OAAO,KAAK,CAAC,4BAA4BU;QAC3C;QACA,OAAO;YACLF;QACF;IACF;IAEA,MAAa,aAAwC;QACnDR,OAAO,IAAI,CAAC;QAEZ,IAAI;gBAQEW;YAPJ,MAAMC,SAAS,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU;YAEhD,IAAI,CAACA,OAAO,OAAO,EACjB,MAAM,IAAIC,MAAMD,OAAO,OAAO,IAAI;YAIpC,IAAI,QAAAD,CAAAA,eAAAA,OAAO,IAAI,AAAD,IAAVA,KAAAA,IAAAA,aAAa,MAAM,EAAE;oBACkBG;gBAAzC,MAAMC,aAAa,IAAIC,kBAAkB,QAAAF,CAAAA,gBAAAA,OAAO,IAAI,AAAD,IAAVA,KAAAA,IAAAA,cAAa,MAAM;gBAC5D,MAAMG,aAAaF,WAAW,aAAa;gBAC3C,IAAIE,YAAY;oBACd,IAAI,CAAC,eAAe,GAAGA,QAAAA,aAAAA,KAAAA,IAAAA,WAAY,KAAK;oBACxC,IAAI,CAAC,gBAAgB,GAAGA,QAAAA,aAAAA,KAAAA,IAAAA,WAAY,MAAM;gBAC5C;gBACAjB,OAAO,IAAI,CAAC,8CAA8CkB,KAAK,SAAS,CAACD;gBACzE,OAAO;oBACL,QAAQL,OAAO,IAAI,CAAC,MAAM;oBAC1B,aAAaA,OAAO,IAAI,CAAC,WAAW,IAAI;gBAC1C;YACF;YACE,MAAM,IAAIC,MAAM;QAEpB,EAAE,OAAOH,OAAO;YACdV,OAAO,KAAK,CAAC,0CAA0CU;YACvD,MAAMA;QACR;IACF;IAEA,MAAM,QAAQS,MAAqB,EAA0B;QAC3D,MAAM,EAAEC,gBAAgB,EAAEC,WAAW,EAAEC,YAAY,EAAEC,WAAW,EAAE,GAAGJ;QACrE,MAAM,EAAEK,WAAW,EAAEC,aAAa,EAAE,GAAGL;QACvC,MAAMM,cAAcD,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,SAAS,AAAD,KAAK;QAEhDzB,OAAO,IAAI,CACT,wCACAwB,aACAC,eACA,oBACA,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,gBAAgB;QAGvB,MAAM,EACJ,GAAGE,IAAI,EACP,GAAGC,IAAI,EACP,UAAUC,WAAW,EACrB,UAAUC,WAAW,EACtB,GAAGC,uBAAuB;YACzB,QAAQL;YACR,aAAa,IAAI,CAAC,eAAe;YACjC,cAAc,IAAI,CAAC,gBAAgB;YACnC,SAAS;gBAAC;gBAAM;aAAK;QACvB;QAEA,MAAMM,SAASL,AAAS,SAATA,OAAgBM,KAAK,KAAK,CAACN,QAAQ;QAClD,MAAMO,SAASN,AAAS,SAATA,OAAgBK,KAAK,KAAK,CAACL,QAAQ;QAElD5B,OAAO,IAAI,CAAC,CAAC,sCAAsC,EAAEgC,OAAO,EAAE,EAAEE,OAAO,CAAC,CAAC;QACzElC,OAAO,IAAI,CACT,CAAC,kDAAkD,EAAE6B,YAAY,EAAE,EAAEC,YAAY,CAAC,CAAC;QAGrF,IAAIK,gBAAgB,MAClBC,gBAAgB;QAElB,IAAI;YACF,OAAQZ;gBACN,KAAK;wBAEGlB;oBADNN,OAAO,IAAI,CAAC,qCAAqCyB,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,OAAO;oBACvE,eAAMnB,CAAAA,mBAAAA,IAAI,CAAC,UAAU,AAAD,IAAdA,KAAAA,IAAAA,iBAAiB,cAAc,CAAC;wBAAE,KAAKmB,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,OAAO,AAAD,KAAK;oBAAG,EAAC;oBAC3E;gBACF,KAAK;wBAEGY;oBADNrC,OAAO,IAAI,CAAC;oBACZ,eAAMqC,CAAAA,oBAAAA,IAAI,CAAC,UAAU,AAAD,IAAdA,KAAAA,IAAAA,kBAAiB,kBAAkB,EAAC;oBAC1C;gBACF,KAAK;oBACHrC,OAAO,IAAI,CAAC;oBACZ,MAAMsC,MAAM;oBACZ;gBAEF,KAAK;gBACL,KAAK;oBACH,IAAIN,AAAW,SAAXA,UAAmBE,AAAW,SAAXA,QAAiB;wBACtC,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,CAACF,QAAQE;wBACtCC,gBAAgBN;wBAChBO,gBAAgBN;oBAClB;oBACA;gBAEF,KAAK;gBACL,KAAK;gBACL,KAAK;oBACH,IAAIE,AAAW,SAAXA,UAAmBE,AAAW,SAAXA,QAAiB;wBACtC,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAACF,QAAQE;wBACrCC,gBAAgBN;wBAChBO,gBAAgBN;oBAClB;oBACA;gBAEF,KAAK;gBACL,KAAK;oBACH,IAAIE,AAAW,SAAXA,UAAmBE,AAAW,SAAXA,QAAiB;wBACtC,MAAM,IAAI,CAAC,WAAW,CAAC,WAAW,CAACF,QAAQE;wBAC3CC,gBAAgBN;wBAChBO,gBAAgBN;oBAClB;oBACA;gBAEF,KAAK;gBACL,KAAK;oBACH,IAAIE,AAAW,SAAXA,UAAmBE,AAAW,SAAXA,QAAiB;wBACtC,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAACF,QAAQE;wBAC1CC,gBAAgBN;wBAChBO,gBAAgBN;oBAClB;oBACA;gBAEF,KAAK;oBACH,IAAIE,AAAW,SAAXA,UAAmBE,AAAW,SAAXA,QAAiB;wBACtC,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAACF,QAAQE,QAAQ;wBAC7CC,gBAAgBN;wBAChBO,gBAAgBN;oBAClB;oBACA;gBAEF,KAAK;gBACL,KAAK;gBACL,KAAK;oBACH,IAAIL,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,OAAO,EAAE;wBAC1B,MAAM,EAAE,GAAGc,OAAO,EAAE,GAAGC,OAAO,EAAE,GAAGT,uBAAuB;4BACxD,QAAQN,cAAc,OAAO;4BAC7BJ;4BACAC;wBACF;wBACA,MAAMmB,OAAOF,AAAY,SAAZA,UAAmBN,KAAK,KAAK,CAACM,WAAW;wBACtD,MAAMG,OAAOF,AAAY,SAAZA,UAAmBP,KAAK,KAAK,CAACO,WAAW;wBAEtD,IAAIR,UAAUE,UAAUO,QAAQC,MAAM;4BAEpC,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,CAACV,QAAQE;4BACtC,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS;4BAChC,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,CAACO,MAAMC;4BACpC,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO;wBAChC;oBACF;oBACA;gBAGF,KAAK;oBAAQ;4BACKC;wBAAhB,MAAMC,UAAU,QAAAD,CAAAA,yBAAAA,cAAc,OAAO,AAAD,IAApBA,KAAAA,IAAAA,uBAAuB,IAAI;wBAC3C,IAAIC,SAAS;4BACX,MAAMC,eAAeD,QAAQ,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,OAAO;4BAChE,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAACC;wBAC9B;wBACA;oBACF;gBAEA,KAAK;gBACL,KAAK;oBAAS;wBACZ,MAAMC,SAASrB,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,GAAG,AAAD,KAAKA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,MAAM,AAAD;wBACzD,IAAIqB,QAAQ;4BAEV,MAAMC,OAAOD,OAAO,KAAK,CAAC,SAAS,MAAM,CAAC,CAACE,IAAMA,EAAE,MAAM,GAAG;4BAC5D,IAAID,KAAK,MAAM,GAAG,GAChB,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,CAACA;iCAE9B,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAACD;wBAEjC;wBACA;oBACF;gBAEA,KAAK;oBAAU;wBACb,MAAM,EAAEG,SAAS,EAAE,GAAGxB;wBACtB,IAAIO,AAAW,SAAXA,UAAmBE,AAAW,SAAXA,UAAmBe,WAAW;4BACnD,MAAMC,sBAAsBD,UAAU,WAAW;4BACjD,IAAIE,KAAK,GACPC,KAAK;4BAEP,OAAQF;gCACN,KAAK;oCACHE,KAAK;oCACL;gCACF,KAAK;oCACHA,KAAK;oCACL;gCACF,KAAK;oCACHD,KAAK;oCACL;gCACF,KAAK;oCACHA,KAAK;oCACL;4BACJ;4BAEA,IAAIA,AAAO,MAAPA,MAAYC,AAAO,MAAPA,IACd,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,CAACD,IAAIC;wBAEtC;wBACA;oBACF;gBAEA,KAAK;gBACL,KAAK;gBACL,KAAK;gBACL,KAAK;oBACH;gBAEF;oBACEpD,OAAO,IAAI,CAAC,CAAC,yBAAyB,EAAEwB,aAAa;YACzD;YASAxB,OAAO,IAAI,CACT,CAAC,8CAA8C,EAAEmC,cAAc,EAAE,EAAEC,cAAc,CAAC,CAAC;YAIrF,OAAO;gBAGLJ;gBACAE;gBAEAC;gBACAC;gBACAX;YACF;QACF,EAAE,OAAOf,OAAO;YACdV,OAAO,KAAK,CAAC,iDAA6BU;YAC1C,OAAO;gBAAE,QAAQ2C,WAAW,KAAK;YAAC;QACpC;IACF;IAnRA,YAAoBjD,OAAyB,CAAE;QAC7C,KAAK,IAfP,uBAAQ,cAAgC,OACxC,uBAAQ,eAAR,SAEA,uBAAQ,mBAAkB,OAC1B,uBAAQ,oBAAmB;QAYzB,IAAI,CAAC,WAAW,GAAG,IAAIkD,YAAYlD;IACrC;AAiRF;AAlTE,iBADWF,mBACJ,UAAS;IACd,eAAe;QACb;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;KACD;AACH;AAEA,iBAhBWA,mBAgBI,mBAA4C"}