@ui-tars/sdk/dist/Model.mjs.map

A powerful cross-platform(ANY device/platform) toolkit for building GUI automation agents for UI-TARS

{"version":3,"file":"Model.mjs","sources":["webpack://@ui-tars/sdk/./src/Model.ts"],"sourcesContent":["/*\n * Copyright (c) 2025 Bytedance, Inc. and its affiliates.\n * SPDX-License-Identifier: Apache-2.0\n */\nimport OpenAI, { type ClientOptions } from 'openai';\nimport {\n  type ChatCompletionCreateParamsNonStreaming,\n  type ChatCompletionCreateParamsBase,\n  type ChatCompletionMessageParam,\n} from 'openai/resources/chat/completions';\nimport { actionParser } from '@ui-tars/action-parser';\n\nimport { useContext } from './context/useContext';\nimport { Model, type InvokeParams, type InvokeOutput } from './types';\n\nimport { preprocessResizeImage, convertToOpenAIMessages } from './utils';\nimport { DEFAULT_FACTORS } from './constants';\nimport {\n  UITarsModelVersion,\n  MAX_PIXELS_V1_0,\n  MAX_PIXELS_V1_5,\n  MAX_PIXELS_DOUBAO,\n} from '@ui-tars/shared/types';\n\ntype OpenAIChatCompletionCreateParams = Omit<ClientOptions, 'maxRetries'> &\n  Pick<\n    ChatCompletionCreateParamsBase,\n    'model' | 'max_tokens' | 'temperature' | 'top_p'\n  >;\n\nexport interface UITarsModelConfig extends OpenAIChatCompletionCreateParams {}\n\nexport interface ThinkingVisionProModelConfig\n  extends ChatCompletionCreateParamsNonStreaming {\n  thinking?: {\n    type: 'enabled' | 'disabled';\n  };\n}\n\nexport class UITarsModel extends Model {\n  constructor(protected readonly modelConfig: UITarsModelConfig) {\n    super();\n    this.modelConfig = modelConfig;\n  }\n\n  /** [widthFactor, heightFactor] */\n  get factors(): [number, number] {\n    return DEFAULT_FACTORS;\n  }\n\n  get modelName(): string {\n    return this.modelConfig.model ?? 'unknown';\n  }\n\n  /**\n   * call real LLM / VLM Model\n   * @param params\n   * @param options\n   * @returns\n   */\n  protected async invokeModelProvider(\n    uiTarsVersion: UITarsModelVersion = UITarsModelVersion.V1_0,\n    params: {\n      messages: Array<ChatCompletionMessageParam>;\n    },\n    options: {\n      signal?: AbortSignal;\n    },\n  ): Promise<{\n    prediction: string;\n    costTime?: number;\n    costTokens?: number;\n  }> {\n    const { messages } = params;\n    const {\n      baseURL,\n      apiKey,\n      model,\n      max_tokens = uiTarsVersion == UITarsModelVersion.V1_5 ? 65535 : 1000,\n      temperature = 0,\n      top_p = 0.7,\n      ...restOptions\n    } = this.modelConfig;\n\n    const openai = new OpenAI({\n      ...restOptions,\n      maxRetries: 0,\n      baseURL,\n      apiKey,\n    });\n\n    const createCompletionPrams: ChatCompletionCreateParamsNonStreaming = {\n      model,\n      messages,\n      stream: false,\n      seed: null,\n      stop: null,\n      frequency_penalty: null,\n      presence_penalty: null,\n      // custom options\n      max_tokens,\n      temperature,\n      top_p,\n    };\n\n    const createCompletionPramsThinkingVp: ThinkingVisionProModelConfig = {\n      ...createCompletionPrams,\n      thinking: {\n        type: 'disabled',\n      },\n    };\n\n    const startTime = Date.now();\n    const result = await openai.chat.completions.create(\n      createCompletionPramsThinkingVp,\n      options,\n    );\n    const costTime = Date.now() - startTime;\n\n    return {\n      prediction: result.choices?.[0]?.message?.content ?? '',\n      costTime: costTime,\n      costTokens: result.usage?.total_tokens ?? 0,\n    };\n  }\n\n  async invoke(params: InvokeParams): Promise<InvokeOutput> {\n    const { conversations, images, screenContext, scaleFactor, uiTarsVersion } =\n      params;\n    const { logger, signal } = useContext();\n\n    logger?.info(\n      `[UITarsModel] invoke: screenContext=${JSON.stringify(screenContext)}, scaleFactor=${scaleFactor}, uiTarsVersion=${uiTarsVersion}`,\n    );\n\n    const maxPixels =\n      uiTarsVersion === UITarsModelVersion.V1_5\n        ? MAX_PIXELS_V1_5\n        : uiTarsVersion === UITarsModelVersion.DOUBAO_1_5_15B ||\n            uiTarsVersion === UITarsModelVersion.DOUBAO_1_5_20B\n          ? MAX_PIXELS_DOUBAO\n          : MAX_PIXELS_V1_0;\n    const compressedImages = await Promise.all(\n      images.map((image) => preprocessResizeImage(image, maxPixels)),\n    );\n\n    const messages = convertToOpenAIMessages({\n      conversations,\n      images: compressedImages,\n    });\n\n    const startTime = Date.now();\n    const result = await this.invokeModelProvider(\n      uiTarsVersion,\n      {\n        messages,\n      },\n      {\n        signal,\n      },\n    )\n      .catch((e) => {\n        logger?.error('[UITarsModel] error', e);\n        throw e;\n      })\n      .finally(() => {\n        logger?.info(`[UITarsModel cost]: ${Date.now() - startTime}ms`);\n      });\n\n    if (!result.prediction) {\n      const err = new Error();\n      err.name = 'vlm response error';\n      err.stack = JSON.stringify(result) ?? 'no message';\n      logger?.error(err);\n      throw err;\n    }\n\n    const { prediction, costTime, costTokens } = result;\n\n    try {\n      const { parsed: parsedPredictions } = await actionParser({\n        prediction,\n        factor: this.factors,\n        screenContext,\n        scaleFactor,\n        modelVer: uiTarsVersion,\n      });\n      return {\n        prediction,\n        parsedPredictions,\n        costTime,\n        costTokens,\n      };\n    } catch (error) {\n      logger?.error('[UITarsModel] error', error);\n      return {\n        prediction,\n        parsedPredictions: [],\n      };\n    }\n  }\n}\n"],"names":["UITarsModel","Model","DEFAULT_FACTORS","uiTarsVersion","UITarsModelVersion","params","options","_result_choices__message","_result_usage","messages","baseURL","apiKey","model","max_tokens","temperature","top_p","restOptions","openai","OpenAI","createCompletionPrams","createCompletionPramsThinkingVp","startTime","Date","result","costTime","conversations","images","screenContext","scaleFactor","logger","signal","useContext","JSON","maxPixels","MAX_PIXELS_V1_5","MAX_PIXELS_DOUBAO","MAX_PIXELS_V1_0","compressedImages","Promise","image","preprocessResizeImage","convertToOpenAIMessages","e","err","Error","prediction","costTokens","parsedPredictions","actionParser","error","modelConfig"],"mappings":";;;;;;;;;;;AAGC;;;;;;;;;;AAoCM,MAAMA,oBAAoBC,gDAAAA,KAAKA;IAOpC,IAAI,UAA4B;QAC9B,OAAOC,oDAAAA,eAAeA;IACxB;IAEA,IAAI,YAAoB;QACtB,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,IAAI;IACnC;IAQA,MAAgB,oBACdC,gBAAoCC,2DAAAA,kBAAAA,CAAAA,IAAuB,EAC3DC,MAEC,EACDC,OAEC,EAKA;YAgDaC,0BAAAA,kBAAAA,iBAEAC;QAjDd,MAAM,EAAEC,QAAQ,EAAE,GAAGJ;QACrB,MAAM,EACJK,OAAO,EACPC,MAAM,EACNC,KAAK,EACLC,aAAaV,iBAAiBC,2DAAAA,kBAAAA,CAAAA,IAAuB,GAAG,QAAQ,IAAI,EACpEU,cAAc,CAAC,EACfC,QAAQ,GAAG,EACX,GAAGC,aACJ,GAAG,IAAI,CAAC,WAAW;QAEpB,MAAMC,SAAS,IAAIC,kCAAAA,CAAAA,UAAMA,CAAC;YACxB,GAAGF,WAAW;YACd,YAAY;YACZN;YACAC;QACF;QAEA,MAAMQ,wBAAgE;YACpEP;YACAH;YACA,QAAQ;YACR,MAAM;YACN,MAAM;YACN,mBAAmB;YACnB,kBAAkB;YAElBI;YACAC;YACAC;QACF;QAEA,MAAMK,kCAAgE;YACpE,GAAGD,qBAAqB;YACxB,UAAU;gBACR,MAAM;YACR;QACF;QAEA,MAAME,YAAYC,KAAK,GAAG;QAC1B,MAAMC,SAAS,MAAMN,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,CACjDG,iCACAd;QAEF,MAAMkB,WAAWF,KAAK,GAAG,KAAKD;QAE9B,OAAO;YACL,YAAYd,AAAAA,CAAc,SAAdA,CAAAA,kBAAAA,OAAO,OAAO,AAAD,KAAbA,AAAAA,KAAAA,MAAAA,kBAAAA,KAAAA,IAAAA,AAAmB,SAAnBA,CAAAA,mBAAAA,eAAgB,CAAC,EAAE,AAAD,KAAlBA,AAAAA,KAAAA,MAAAA,mBAAAA,KAAAA,IAAAA,AAA4B,SAA5BA,CAAAA,2BAAAA,iBAAqB,OAAO,AAAD,KAA3BA,AAAAA,KAAAA,MAAAA,2BAAAA,KAAAA,IAAAA,yBAA8B,OAAO,AAAD,KAAK;YACrD,UAAUiB;YACV,YAAYhB,AAAAA,CAAY,SAAZA,CAAAA,gBAAAA,OAAO,KAAK,AAAD,KAAXA,AAAAA,KAAAA,MAAAA,gBAAAA,KAAAA,IAAAA,cAAc,YAAY,AAAD,KAAK;QAC5C;IACF;IAEA,MAAM,OAAOH,MAAoB,EAAyB;QACxD,MAAM,EAAEoB,aAAa,EAAEC,MAAM,EAAEC,aAAa,EAAEC,WAAW,EAAEzB,aAAa,EAAE,GACxEE;QACF,MAAM,EAAEwB,MAAM,EAAEC,MAAM,EAAE,GAAGC,AAAAA,IAAAA,6DAAAA,UAAAA,AAAAA;QAE3BF,QAAAA,UAAAA,OAAQ,IAAI,CACV,CAAC,oCAAoC,EAAEG,KAAK,SAAS,CAACL,eAAe,cAAc,EAAEC,YAAY,gBAAgB,EAAEzB,eAAe;QAGpI,MAAM8B,YACJ9B,kBAAkBC,2DAAAA,kBAAAA,CAAAA,IAAuB,GACrC8B,2DAAAA,eAAeA,GACf/B,kBAAkBC,2DAAAA,kBAAAA,CAAAA,cAAiC,IACjDD,kBAAkBC,2DAAAA,kBAAAA,CAAAA,cAAiC,GACnD+B,2DAAAA,iBAAiBA,GACjBC,2DAAAA,eAAeA;QACvB,MAAMC,mBAAmB,MAAMC,QAAQ,GAAG,CACxCZ,OAAO,GAAG,CAAC,CAACa,QAAUC,AAAAA,IAAAA,gDAAAA,qBAAAA,AAAAA,EAAsBD,OAAON;QAGrD,MAAMxB,WAAWgC,AAAAA,IAAAA,gDAAAA,uBAAAA,AAAAA,EAAwB;YACvChB;YACA,QAAQY;QACV;QAEA,MAAMhB,YAAYC,KAAK,GAAG;QAC1B,MAAMC,SAAS,MAAM,IAAI,CAAC,mBAAmB,CAC3CpB,eACA;YACEM;QACF,GACA;YACEqB;QACF,GAEC,KAAK,CAAC,CAACY;YACNb,QAAAA,UAAAA,OAAQ,KAAK,CAAC,uBAAuBa;YACrC,MAAMA;QACR,GACC,OAAO,CAAC;YACPb,QAAAA,UAAAA,OAAQ,IAAI,CAAC,CAAC,oBAAoB,EAAEP,KAAK,GAAG,KAAKD,UAAU,EAAE,CAAC;QAChE;QAEF,IAAI,CAACE,OAAO,UAAU,EAAE;YACtB,MAAMoB,MAAM,IAAIC;YAChBD,IAAI,IAAI,GAAG;YACXA,IAAI,KAAK,GAAGX,KAAK,SAAS,CAACT,WAAW;YACtCM,QAAAA,UAAAA,OAAQ,KAAK,CAACc;YACd,MAAMA;QACR;QAEA,MAAM,EAAEE,UAAU,EAAErB,QAAQ,EAAEsB,UAAU,EAAE,GAAGvB;QAE7C,IAAI;YACF,MAAM,EAAE,QAAQwB,iBAAiB,EAAE,GAAG,MAAMC,AAAAA,IAAAA,4DAAAA,YAAAA,AAAAA,EAAa;gBACvDH;gBACA,QAAQ,IAAI,CAAC,OAAO;gBACpBlB;gBACAC;gBACA,UAAUzB;YACZ;YACA,OAAO;gBACL0C;gBACAE;gBACAvB;gBACAsB;YACF;QACF,EAAE,OAAOG,OAAO;YACdpB,QAAAA,UAAAA,OAAQ,KAAK,CAAC,uBAAuBoB;YACrC,OAAO;gBACLJ;gBACA,mBAAmB,EAAE;YACvB;QACF;IACF;IAhKA,YAA+BK,WAA8B,CAAE;QAC7D,KAAK,wDADwBA,WAAW,GAAXA;QAE7B,IAAI,CAAC,WAAW,GAAGA;IACrB;AA8JF"}