@ui-tars/operator-nut-js/dist/index.mjs.map

Operator Nut JS SDK for UI-TARS

{"version":3,"file":"index.mjs","sources":["webpack://@ui-tars/operator-nut-js/./src/index.ts"],"sourcesContent":["/*\n * Copyright (c) 2025 Bytedance, Inc. and its affiliates.\n * SPDX-License-Identifier: Apache-2.0\n */\nimport {\n  Operator,\n  useContext,\n  parseBoxToScreenCoords,\n  StatusEnum,\n  type ScreenshotOutput,\n  type ExecuteParams,\n  type ExecuteOutput,\n} from '@ui-tars/sdk/core';\nimport { Jimp } from 'jimp';\nimport {\n  screen,\n  Button,\n  Key,\n  Point,\n  Region,\n  centerOf,\n  keyboard,\n  mouse,\n  sleep,\n  straightTo,\n  clipboard,\n} from '@computer-use/nut-js';\nimport Big from 'big.js';\n\nconst moveStraightTo = async (startX: number | null, startY: number | null) => {\n  if (startX === null || startY === null) {\n    return;\n  }\n  await mouse.move(straightTo(new Point(startX, startY)));\n};\nexport class NutJSOperator extends Operator {\n  static MANUAL = {\n    ACTION_SPACES: [\n      `click(start_box='[x1, y1, x2, y2]')`,\n      `left_double(start_box='[x1, y1, x2, y2]')`,\n      `right_single(start_box='[x1, y1, x2, y2]')`,\n      `drag(start_box='[x1, y1, x2, y2]', end_box='[x3, y3, x4, y4]')`,\n      `hotkey(key='')`,\n      `type(content='') #If you want to submit your input, use \"\\\\n\" at the end of \\`content\\`.`,\n      `scroll(start_box='[x1, y1, x2, y2]', direction='down or up or right or left')`,\n      `wait() #Sleep for 5s and take a screenshot to check for any changes.`,\n      `finished()`,\n      `call_user() # Submit the task and call the user when the task is unsolvable, or when you need the user's help.`,\n    ],\n  };\n\n  public async screenshot(): Promise<ScreenshotOutput> {\n    const { logger } = useContext();\n    const grabImage = await screen.grab();\n    const screenWithScale = await grabImage.toRGB(); // widthScale = screenWidth * scaleX\n\n    const scaleFactor = screenWithScale.pixelDensity.scaleX;\n\n    logger.info(\n      '[NutjsOperator]',\n      'scaleX',\n      screenWithScale.pixelDensity.scaleX,\n      'scaleY',\n      screenWithScale.pixelDensity.scaleY,\n    );\n\n    const screenWithScaleImage = await Jimp.fromBitmap({\n      width: screenWithScale.width,\n      height: screenWithScale.height,\n      data: Buffer.from(screenWithScale.data),\n    });\n\n    const width = screenWithScale.width / screenWithScale.pixelDensity.scaleX;\n    const height = screenWithScale.height / screenWithScale.pixelDensity.scaleY;\n\n    const physicalScreenImage = await screenWithScaleImage\n      .resize({\n        w: width,\n        h: height,\n      })\n      .getBuffer('image/png'); // Use png format to avoid compression\n\n    const output = {\n      base64: physicalScreenImage.toString('base64'),\n      scaleFactor,\n    };\n\n    logger?.info(\n      `[NutjsOperator] screenshot: ${width}x${height}, scaleFactor: ${scaleFactor}`,\n    );\n    return output;\n  }\n\n  async execute(params: ExecuteParams): Promise<ExecuteOutput> {\n    const { logger } = useContext();\n    const { parsedPrediction, screenWidth, screenHeight, scaleFactor } = params;\n\n    const { action_type, action_inputs } = parsedPrediction;\n    const startBoxStr = action_inputs?.start_box || '';\n\n    logger.info('[NutjsOperator] execute', scaleFactor);\n    const { x: startX, y: startY } = parseBoxToScreenCoords({\n      boxStr: startBoxStr,\n      screenWidth,\n      screenHeight,\n    });\n\n    logger.info(`[NutjsOperator Position]: (${startX}, ${startY})`);\n\n    // execute configs\n    mouse.config.mouseSpeed = 3600;\n\n    // if (startBoxStr) {\n    //   const region = await nutScreen.highlight(\n    //     new Region(startX, startY, 100, 100),\n    //   );\n    //   logger.info('[execute] [Region]', region);\n    // }\n\n    const getHotkeys = (keyStr: string | undefined): Key[] => {\n      if (keyStr) {\n        const platformCommandKey =\n          process.platform === 'darwin' ? Key.LeftCmd : Key.LeftWin;\n        const platformCtrlKey =\n          process.platform === 'darwin' ? Key.LeftCmd : Key.LeftControl;\n        const keyMap = {\n          return: Key.Enter,\n          ctrl: platformCtrlKey,\n          shift: Key.LeftShift,\n          alt: Key.LeftAlt,\n          'page down': Key.PageDown,\n          'page up': Key.PageUp,\n          meta: platformCommandKey,\n          win: platformCommandKey,\n          command: platformCommandKey,\n          cmd: platformCommandKey,\n          ',': Key.Comma,\n          arrowup: Key.Up,\n          arrowdown: Key.Down,\n          arrowleft: Key.Left,\n          arrowright: Key.Right,\n        } as const;\n\n        const lowercaseKeyMap = Object.fromEntries(\n          Object.entries(Key).map(([k, v]) => [k.toLowerCase(), v]),\n        ) as {\n          [K in keyof typeof Key as Lowercase<K>]: (typeof Key)[K];\n        };\n\n        const keys = keyStr\n          .split(/[\\s+]/)\n          .map((k) => k.toLowerCase())\n          .map(\n            (k) =>\n              keyMap[k as keyof typeof keyMap] ??\n              lowercaseKeyMap[k as Lowercase<keyof typeof Key>],\n          )\n          .filter(Boolean);\n        logger.info('[NutjsOperator] hotkey: ', keys);\n        return keys;\n      } else {\n        logger.error(\n          '[NutjsOperator] hotkey error: ',\n          `${keyStr} is not a valid key`,\n        );\n        return [];\n      }\n    };\n\n    switch (action_type) {\n      case 'wait':\n        logger.info('[NutjsOperator] wait', action_inputs);\n        await sleep(5000);\n        break;\n\n      case 'mouse_move':\n      case 'hover':\n        logger.info('[NutjsOperator] mouse_move');\n        await moveStraightTo(startX, startY);\n        break;\n\n      case 'click':\n      case 'left_click':\n      case 'left_single':\n        logger.info('[NutjsOperator] left_click');\n        await moveStraightTo(startX, startY);\n        await sleep(100);\n        await mouse.click(Button.LEFT);\n        break;\n\n      case 'left_double':\n      case 'double_click':\n        logger.info(`[NutjsOperator] ${action_type}(${startX}, ${startY})`);\n        await moveStraightTo(startX, startY);\n        await sleep(100);\n        await mouse.doubleClick(Button.LEFT);\n        break;\n\n      case 'right_click':\n      case 'right_single':\n        logger.info('[NutjsOperator] right_click');\n        await moveStraightTo(startX, startY);\n        await sleep(100);\n        await mouse.click(Button.RIGHT);\n        break;\n\n      case 'middle_click':\n        logger.info('[NutjsOperator] middle_click');\n        await moveStraightTo(startX, startY);\n        await mouse.click(Button.MIDDLE);\n        break;\n\n      case 'left_click_drag':\n      case 'drag':\n      case 'select': {\n        logger.info('[NutjsOperator] drag', action_inputs);\n        // end_box\n        if (action_inputs?.end_box) {\n          const { x: endX, y: endY } = parseBoxToScreenCoords({\n            boxStr: action_inputs.end_box,\n            screenWidth,\n            screenHeight,\n          });\n\n          if (startX && startY && endX && endY) {\n            // calculate x and y direction difference\n            const diffX = Big(endX).minus(startX).toNumber();\n            const diffY = Big(endY).minus(startY).toNumber();\n\n            await mouse.drag(\n              straightTo(centerOf(new Region(startX, startY, diffX, diffY))),\n            );\n          }\n        }\n        break;\n      }\n\n      case 'type': {\n        const content = action_inputs.content?.trim();\n        logger.info('[NutjsOperator] type', content);\n        if (content) {\n          const stripContent = content.replace(/\\\\n$/, '').replace(/\\n$/, '');\n          keyboard.config.autoDelayMs = 0;\n          if (process.platform === 'win32') {\n            const originalClipboard = await clipboard.getContent();\n            await clipboard.setContent(stripContent);\n            await keyboard.pressKey(Key.LeftControl, Key.V);\n            await sleep(50);\n            await keyboard.releaseKey(Key.LeftControl, Key.V);\n            await sleep(50);\n            await clipboard.setContent(originalClipboard);\n          } else {\n            await keyboard.type(stripContent);\n          }\n\n          if (content.endsWith('\\n') || content.endsWith('\\\\n')) {\n            await keyboard.pressKey(Key.Enter);\n            await keyboard.releaseKey(Key.Enter);\n          }\n\n          keyboard.config.autoDelayMs = 500;\n        }\n        break;\n      }\n\n      case 'hotkey': {\n        const keyStr = action_inputs?.key || action_inputs?.hotkey;\n        const keys = getHotkeys(keyStr);\n        if (keys.length > 0) {\n          await keyboard.pressKey(...keys);\n          await keyboard.releaseKey(...keys);\n        }\n        break;\n      }\n\n      case 'press': {\n        const keyStr = action_inputs?.key || action_inputs?.hotkey;\n        const keys = getHotkeys(keyStr);\n        if (keys.length > 0) {\n          await keyboard.pressKey(...keys);\n        }\n        break;\n      }\n\n      case 'release': {\n        const keyStr = action_inputs?.key || action_inputs?.hotkey;\n        const keys = getHotkeys(keyStr);\n        if (keys.length > 0) {\n          await keyboard.releaseKey(...keys);\n        }\n        break;\n      }\n\n      case 'scroll': {\n        const { direction } = action_inputs;\n        // if startX and startY is not null, move mouse to startX, startY\n        if (startX !== null && startY !== null) {\n          await moveStraightTo(startX, startY);\n        }\n\n        switch (direction?.toLowerCase()) {\n          case 'up':\n            await mouse.scrollUp(5 * 100);\n            break;\n          case 'down':\n            await mouse.scrollDown(5 * 100);\n            break;\n          default:\n            console.warn(\n              `[NutjsOperator] Unsupported scroll direction: ${direction}`,\n            );\n        }\n        break;\n      }\n\n      case 'error_env':\n      case 'call_user':\n      case 'finished':\n      case 'user_stop':\n        return { status: StatusEnum.END };\n\n      default:\n        logger.warn(`Unsupported action: ${action_type}`);\n    }\n  }\n}\n"],"names":["moveStraightTo","startX","startY","mouse","straightTo","Point","NutJSOperator","Operator","logger","useContext","grabImage","screen","screenWithScale","scaleFactor","screenWithScaleImage","Jimp","Buffer","width","height","physicalScreenImage","output","params","parsedPrediction","screenWidth","screenHeight","action_type","action_inputs","startBoxStr","parseBoxToScreenCoords","getHotkeys","keyStr","platformCommandKey","process","Key","platformCtrlKey","keyMap","lowercaseKeyMap","Object","k","v","keys","Boolean","sleep","Button","endX","endY","diffX","Big","diffY","centerOf","Region","_action_inputs_content","content","stripContent","keyboard","originalClipboard","clipboard","direction","console","StatusEnum"],"mappings":";;;;;;;;AAGC;;;;;;;;;;AA0BD,MAAMA,iBAAiB,OAAOC,QAAuBC;IACnD,IAAID,AAAW,SAAXA,UAAmBC,AAAW,SAAXA,QACrB;IAEF,MAAMC,MAAM,IAAI,CAACC,WAAW,IAAIC,MAAMJ,QAAQC;AAChD;AACO,MAAMI,sBAAsBC;IAgBjC,MAAa,aAAwC;QACnD,MAAM,EAAEC,MAAM,EAAE,GAAGC;QACnB,MAAMC,YAAY,MAAMC,OAAO,IAAI;QACnC,MAAMC,kBAAkB,MAAMF,UAAU,KAAK;QAE7C,MAAMG,cAAcD,gBAAgB,YAAY,CAAC,MAAM;QAEvDJ,OAAO,IAAI,CACT,mBACA,UACAI,gBAAgB,YAAY,CAAC,MAAM,EACnC,UACAA,gBAAgB,YAAY,CAAC,MAAM;QAGrC,MAAME,uBAAuB,MAAMC,KAAK,UAAU,CAAC;YACjD,OAAOH,gBAAgB,KAAK;YAC5B,QAAQA,gBAAgB,MAAM;YAC9B,MAAMI,OAAO,IAAI,CAACJ,gBAAgB,IAAI;QACxC;QAEA,MAAMK,QAAQL,gBAAgB,KAAK,GAAGA,gBAAgB,YAAY,CAAC,MAAM;QACzE,MAAMM,SAASN,gBAAgB,MAAM,GAAGA,gBAAgB,YAAY,CAAC,MAAM;QAE3E,MAAMO,sBAAsB,MAAML,qBAC/B,MAAM,CAAC;YACN,GAAGG;YACH,GAAGC;QACL,GACC,SAAS,CAAC;QAEb,MAAME,SAAS;YACb,QAAQD,oBAAoB,QAAQ,CAAC;YACrCN;QACF;QAEAL,QAAAA,UAAAA,OAAQ,IAAI,CACV,CAAC,4BAA4B,EAAES,MAAM,CAAC,EAAEC,OAAO,eAAe,EAAEL,aAAa;QAE/E,OAAOO;IACT;IAEA,MAAM,QAAQC,MAAqB,EAA0B;QAC3D,MAAM,EAAEb,MAAM,EAAE,GAAGC;QACnB,MAAM,EAAEa,gBAAgB,EAAEC,WAAW,EAAEC,YAAY,EAAEX,WAAW,EAAE,GAAGQ;QAErE,MAAM,EAAEI,WAAW,EAAEC,aAAa,EAAE,GAAGJ;QACvC,MAAMK,cAAcD,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,SAAS,AAAD,KAAK;QAEhDlB,OAAO,IAAI,CAAC,2BAA2BK;QACvC,MAAM,EAAE,GAAGZ,MAAM,EAAE,GAAGC,MAAM,EAAE,GAAG0B,uBAAuB;YACtD,QAAQD;YACRJ;YACAC;QACF;QAEAhB,OAAO,IAAI,CAAC,CAAC,2BAA2B,EAAEP,OAAO,EAAE,EAAEC,OAAO,CAAC,CAAC;QAG9DC,MAAM,MAAM,CAAC,UAAU,GAAG;QAS1B,MAAM0B,aAAa,CAACC;YAClB,IAAIA,QAAQ;gBACV,MAAMC,qBACJC,AAAqB,aAArBA,QAAQ,QAAQ,GAAgBC,IAAI,OAAO,GAAGA,IAAI,OAAO;gBAC3D,MAAMC,kBACJF,AAAqB,aAArBA,QAAQ,QAAQ,GAAgBC,IAAI,OAAO,GAAGA,IAAI,WAAW;gBAC/D,MAAME,SAAS;oBACb,QAAQF,IAAI,KAAK;oBACjB,MAAMC;oBACN,OAAOD,IAAI,SAAS;oBACpB,KAAKA,IAAI,OAAO;oBAChB,aAAaA,IAAI,QAAQ;oBACzB,WAAWA,IAAI,MAAM;oBACrB,MAAMF;oBACN,KAAKA;oBACL,SAASA;oBACT,KAAKA;oBACL,KAAKE,IAAI,KAAK;oBACd,SAASA,IAAI,EAAE;oBACf,WAAWA,IAAI,IAAI;oBACnB,WAAWA,IAAI,IAAI;oBACnB,YAAYA,IAAI,KAAK;gBACvB;gBAEA,MAAMG,kBAAkBC,OAAO,WAAW,CACxCA,OAAO,OAAO,CAACJ,KAAK,GAAG,CAAC,CAAC,CAACK,GAAGC,EAAE,GAAK;wBAACD,EAAE,WAAW;wBAAIC;qBAAE;gBAK1D,MAAMC,OAAOV,OACV,KAAK,CAAC,SACN,GAAG,CAAC,CAACQ,IAAMA,EAAE,WAAW,IACxB,GAAG,CACF,CAACA,IACCH,MAAM,CAACG,EAAyB,IAChCF,eAAe,CAACE,EAAiC,EAEpD,MAAM,CAACG;gBACVjC,OAAO,IAAI,CAAC,4BAA4BgC;gBACxC,OAAOA;YACT;YACEhC,OAAO,KAAK,CACV,kCACA,GAAGsB,OAAO,mBAAmB,CAAC;YAEhC,OAAO,EAAE;QAEb;QAEA,OAAQL;YACN,KAAK;gBACHjB,OAAO,IAAI,CAAC,wBAAwBkB;gBACpC,MAAMgB,MAAM;gBACZ;YAEF,KAAK;YACL,KAAK;gBACHlC,OAAO,IAAI,CAAC;gBACZ,MAAMR,eAAeC,QAAQC;gBAC7B;YAEF,KAAK;YACL,KAAK;YACL,KAAK;gBACHM,OAAO,IAAI,CAAC;gBACZ,MAAMR,eAAeC,QAAQC;gBAC7B,MAAMwC,MAAM;gBACZ,MAAMvC,MAAM,KAAK,CAACwC,OAAO,IAAI;gBAC7B;YAEF,KAAK;YACL,KAAK;gBACHnC,OAAO,IAAI,CAAC,CAAC,gBAAgB,EAAEiB,YAAY,CAAC,EAAExB,OAAO,EAAE,EAAEC,OAAO,CAAC,CAAC;gBAClE,MAAMF,eAAeC,QAAQC;gBAC7B,MAAMwC,MAAM;gBACZ,MAAMvC,MAAM,WAAW,CAACwC,OAAO,IAAI;gBACnC;YAEF,KAAK;YACL,KAAK;gBACHnC,OAAO,IAAI,CAAC;gBACZ,MAAMR,eAAeC,QAAQC;gBAC7B,MAAMwC,MAAM;gBACZ,MAAMvC,MAAM,KAAK,CAACwC,OAAO,KAAK;gBAC9B;YAEF,KAAK;gBACHnC,OAAO,IAAI,CAAC;gBACZ,MAAMR,eAAeC,QAAQC;gBAC7B,MAAMC,MAAM,KAAK,CAACwC,OAAO,MAAM;gBAC/B;YAEF,KAAK;YACL,KAAK;YACL,KAAK;gBACHnC,OAAO,IAAI,CAAC,wBAAwBkB;gBAEpC,IAAIA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,OAAO,EAAE;oBAC1B,MAAM,EAAE,GAAGkB,IAAI,EAAE,GAAGC,IAAI,EAAE,GAAGjB,uBAAuB;wBAClD,QAAQF,cAAc,OAAO;wBAC7BH;wBACAC;oBACF;oBAEA,IAAIvB,UAAUC,UAAU0C,QAAQC,MAAM;wBAEpC,MAAMC,QAAQC,IAAIH,MAAM,KAAK,CAAC3C,QAAQ,QAAQ;wBAC9C,MAAM+C,QAAQD,IAAIF,MAAM,KAAK,CAAC3C,QAAQ,QAAQ;wBAE9C,MAAMC,MAAM,IAAI,CACdC,WAAW6C,SAAS,IAAIC,OAAOjD,QAAQC,QAAQ4C,OAAOE;oBAE1D;gBACF;gBACA;YAGF,KAAK;gBAAQ;wBACKG;oBAAhB,MAAMC,UAAU,QAAAD,CAAAA,yBAAAA,cAAc,OAAO,AAAD,IAApBA,KAAAA,IAAAA,uBAAuB,IAAI;oBAC3C3C,OAAO,IAAI,CAAC,wBAAwB4C;oBACpC,IAAIA,SAAS;wBACX,MAAMC,eAAeD,QAAQ,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,OAAO;wBAChEE,SAAS,MAAM,CAAC,WAAW,GAAG;wBAC9B,IAAItB,AAAqB,YAArBA,QAAQ,QAAQ,EAAc;4BAChC,MAAMuB,oBAAoB,MAAMC,UAAU,UAAU;4BACpD,MAAMA,UAAU,UAAU,CAACH;4BAC3B,MAAMC,SAAS,QAAQ,CAACrB,IAAI,WAAW,EAAEA,IAAI,CAAC;4BAC9C,MAAMS,MAAM;4BACZ,MAAMY,SAAS,UAAU,CAACrB,IAAI,WAAW,EAAEA,IAAI,CAAC;4BAChD,MAAMS,MAAM;4BACZ,MAAMc,UAAU,UAAU,CAACD;wBAC7B,OACE,MAAMD,SAAS,IAAI,CAACD;wBAGtB,IAAID,QAAQ,QAAQ,CAAC,SAASA,QAAQ,QAAQ,CAAC,QAAQ;4BACrD,MAAME,SAAS,QAAQ,CAACrB,IAAI,KAAK;4BACjC,MAAMqB,SAAS,UAAU,CAACrB,IAAI,KAAK;wBACrC;wBAEAqB,SAAS,MAAM,CAAC,WAAW,GAAG;oBAChC;oBACA;gBACF;YAEA,KAAK;gBAAU;oBACb,MAAMxB,SAASJ,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,GAAG,AAAD,KAAKA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,MAAM,AAAD;oBACzD,MAAMc,OAAOX,WAAWC;oBACxB,IAAIU,KAAK,MAAM,GAAG,GAAG;wBACnB,MAAMc,SAAS,QAAQ,IAAId;wBAC3B,MAAMc,SAAS,UAAU,IAAId;oBAC/B;oBACA;gBACF;YAEA,KAAK;gBAAS;oBACZ,MAAMV,SAASJ,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,GAAG,AAAD,KAAKA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,MAAM,AAAD;oBACzD,MAAMc,OAAOX,WAAWC;oBACxB,IAAIU,KAAK,MAAM,GAAG,GAChB,MAAMc,SAAS,QAAQ,IAAId;oBAE7B;gBACF;YAEA,KAAK;gBAAW;oBACd,MAAMV,SAASJ,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,GAAG,AAAD,KAAKA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,MAAM,AAAD;oBACzD,MAAMc,OAAOX,WAAWC;oBACxB,IAAIU,KAAK,MAAM,GAAG,GAChB,MAAMc,SAAS,UAAU,IAAId;oBAE/B;gBACF;YAEA,KAAK;gBAAU;oBACb,MAAM,EAAEiB,SAAS,EAAE,GAAG/B;oBAEtB,IAAIzB,AAAW,SAAXA,UAAmBC,AAAW,SAAXA,QACrB,MAAMF,eAAeC,QAAQC;oBAG/B,OAAQuD,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,WAAW;wBAC5B,KAAK;4BACH,MAAMtD,MAAM,QAAQ,CAAC;4BACrB;wBACF,KAAK;4BACH,MAAMA,MAAM,UAAU,CAAC;4BACvB;wBACF;4BACEuD,QAAQ,IAAI,CACV,CAAC,8CAA8C,EAAED,WAAW;oBAElE;oBACA;gBACF;YAEA,KAAK;YACL,KAAK;YACL,KAAK;YACL,KAAK;gBACH,OAAO;oBAAE,QAAQE,WAAW,GAAG;gBAAC;YAElC;gBACEnD,OAAO,IAAI,CAAC,CAAC,oBAAoB,EAAEiB,aAAa;QACpD;IACF;AACF;AAjSE,iBADWnB,eACJ,UAAS;IACd,eAAe;QACb;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;KACD;AACH"}