@genkit-ai/vertexai
Version:
Genkit AI framework plugin for Google Cloud Vertex AI APIs including Gemini APIs, Imagen, and more.
1 lines • 18.2 kB
Source Map (JSON)
{"version":3,"sources":["../../src/modelgarden/openai_compatibility.ts"],"sourcesContent":["/**\n * Copyright 2024 Google LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Genkit, Message, StreamingCallback, z } from 'genkit';\nimport {\n GenerateResponseChunkData,\n GenerateResponseData,\n GenerationCommonConfigSchema,\n ModelAction,\n ModelReference,\n type CandidateData,\n type GenerateRequest,\n type MessageData,\n type Part,\n type Role,\n type ToolDefinition,\n type ToolRequestPart,\n} from 'genkit/model';\nimport OpenAI from 'openai';\nimport {\n type ChatCompletion,\n type ChatCompletionChunk,\n type ChatCompletionContentPart,\n type ChatCompletionCreateParamsNonStreaming,\n type ChatCompletionMessageParam,\n type ChatCompletionMessageToolCall,\n type ChatCompletionRole,\n type ChatCompletionTool,\n type CompletionChoice,\n} from 'openai/resources/index.mjs';\n\n/**\n * See https://platform.openai.com/docs/api-reference/chat/create.\n */\nexport const OpenAIConfigSchema = GenerationCommonConfigSchema.extend({\n // TODO: topK is not supported and some of the other common config options\n // have different names in the above doc. Eg: max_completion_tokens.\n // Update to use the parameters in above doc.\n frequencyPenalty: z\n .number()\n .min(-2)\n .max(2)\n .describe(\n 'Positive values penalize new tokens based on their ' +\n \"existing frequency in the text so far, decreasing the model's \" +\n 'likelihood to repeat the same line verbatim.'\n )\n .optional(),\n logitBias: z\n .record(\n z.string().describe('Token string.'),\n z.number().min(-100).max(100).describe('Associated bias value.')\n )\n .describe(\n 'Controls the likelihood of specified tokens appearing ' +\n 'in the generated output. Map of tokens to an associated bias ' +\n 'value from -100 (which will in most cases block that token ' +\n 'from being generated) to 100 (exclusive selection of the ' +\n 'token which makes it more likely to be generated). Moderate ' +\n 'values like -1 and 1 will change the probability of a token ' +\n 'being selected to a lesser degree.'\n )\n .optional(),\n logProbs: z\n .boolean()\n .describe(\n 'Whether to return log probabilities of the output tokens or not.'\n )\n .optional(),\n presencePenalty: z\n .number()\n .min(-2)\n .max(2)\n .describe(\n 'Positive values penalize new tokens based on whether ' +\n \"they appear in the text so far, increasing the model's \" +\n 'likelihood to talk about new topics.'\n )\n .optional(),\n seed: z\n .number()\n .int()\n .describe(\n 'If specified, the system will make a best effort to sample ' +\n 'deterministically, such that repeated requests with the same seed ' +\n 'and parameters should return the same result. Determinism is not ' +\n 'guaranteed, and you should refer to the system_fingerprint response ' +\n 'parameter to monitor changes in the backend.'\n )\n .optional(),\n topLogProbs: z\n .number()\n .int()\n .min(0)\n .max(20)\n .describe(\n 'An integer specifying the number of most likely tokens to ' +\n 'return at each token position, each with an associated log ' +\n 'probability. logprobs must be set to true if this parameter is used.'\n )\n .optional(),\n user: z\n .string()\n .describe(\n 'A unique identifier representing your end-user to monitor and detect abuse.'\n )\n .optional(),\n});\n\nexport function toOpenAIRole(role: Role): ChatCompletionRole {\n switch (role) {\n case 'user':\n return 'user';\n case 'model':\n return 'assistant';\n case 'system':\n return 'system';\n case 'tool':\n return 'tool';\n default:\n throw new Error(`role ${role} doesn't map to an OpenAI role.`);\n }\n}\n\nfunction toOpenAiTool(tool: ToolDefinition): ChatCompletionTool {\n return {\n type: 'function',\n function: {\n name: tool.name,\n parameters: tool.inputSchema || undefined,\n },\n };\n}\n\nexport function toOpenAiTextAndMedia(part: Part): ChatCompletionContentPart {\n if (part.text) {\n return {\n type: 'text',\n text: part.text,\n };\n } else if (part.media) {\n return {\n type: 'image_url',\n image_url: {\n url: part.media.url,\n },\n };\n }\n throw Error(\n `Unsupported genkit part fields encountered for current message role: ${JSON.stringify(part)}.`\n );\n}\n\nexport function toOpenAiMessages(\n messages: MessageData[]\n): ChatCompletionMessageParam[] {\n const openAiMsgs: ChatCompletionMessageParam[] = [];\n for (const message of messages) {\n const msg = new Message(message);\n const role = toOpenAIRole(message.role);\n switch (role) {\n case 'user':\n openAiMsgs.push({\n role: role,\n content: msg.content.map((part) => toOpenAiTextAndMedia(part)),\n });\n break;\n case 'system':\n openAiMsgs.push({\n role: role,\n content: msg.text,\n });\n break;\n case 'assistant': {\n const toolCalls: ChatCompletionMessageToolCall[] = msg.content\n .filter(\n (\n part\n ): part is Part & {\n toolRequest: NonNullable<Part['toolRequest']>;\n } => Boolean(part.toolRequest)\n )\n .map((part) => ({\n id: part.toolRequest.ref ?? '',\n type: 'function',\n function: {\n name: part.toolRequest.name,\n arguments: JSON.stringify(part.toolRequest.input),\n },\n }));\n if (toolCalls.length > 0) {\n openAiMsgs.push({\n role: role,\n tool_calls: toolCalls,\n });\n } else {\n openAiMsgs.push({\n role: role,\n content: msg.text,\n });\n }\n break;\n }\n case 'tool': {\n const toolResponseParts = msg.toolResponseParts();\n toolResponseParts.map((part) => {\n openAiMsgs.push({\n role: role,\n tool_call_id: part.toolResponse.ref ?? '',\n content:\n typeof part.toolResponse.output === 'string'\n ? part.toolResponse.output\n : JSON.stringify(part.toolResponse.output),\n });\n });\n break;\n }\n }\n }\n return openAiMsgs;\n}\n\nconst finishReasonMap: Record<\n CompletionChoice['finish_reason'] | 'tool_calls',\n CandidateData['finishReason']\n> = {\n length: 'length',\n stop: 'stop',\n tool_calls: 'stop',\n content_filter: 'blocked',\n};\n\nexport function fromOpenAiToolCall(\n toolCall:\n | ChatCompletionMessageToolCall\n | ChatCompletionChunk.Choice.Delta.ToolCall\n): ToolRequestPart {\n if (!toolCall.function) {\n throw Error(\n `Unexpected openAI chunk choice. tool_calls was provided but one or more tool_calls is missing.`\n );\n }\n const f = toolCall.function;\n return {\n toolRequest: {\n name: f.name!,\n ref: toolCall.id,\n input: f.arguments ? JSON.parse(f.arguments) : f.arguments,\n },\n };\n}\n\nexport function fromOpenAiChoice(\n choice: ChatCompletion.Choice,\n jsonMode = false\n): CandidateData {\n const toolRequestParts = choice.message.tool_calls?.map(fromOpenAiToolCall);\n return {\n index: choice.index,\n finishReason: finishReasonMap[choice.finish_reason] || 'other',\n message: {\n role: 'model',\n content: toolRequestParts\n ? // Note: Not sure why I have to cast here exactly.\n // Otherwise it thinks toolRequest must be 'undefined' if provided\n (toolRequestParts as ToolRequestPart[])\n : [\n jsonMode\n ? { data: JSON.parse(choice.message.content!) }\n : { text: choice.message.content! },\n ],\n },\n custom: {},\n };\n}\n\nexport function fromOpenAiChunkChoice(\n choice: ChatCompletionChunk.Choice,\n jsonMode = false\n): CandidateData {\n const toolRequestParts = choice.delta.tool_calls?.map(fromOpenAiToolCall);\n return {\n index: choice.index,\n finishReason: choice.finish_reason\n ? finishReasonMap[choice.finish_reason] || 'other'\n : 'unknown',\n message: {\n role: 'model',\n content: toolRequestParts\n ? (toolRequestParts as ToolRequestPart[])\n : [\n jsonMode\n ? { data: JSON.parse(choice.delta.content!) }\n : { text: choice.delta.content! },\n ],\n },\n custom: {},\n };\n}\n\nexport function toRequestBody(\n model: ModelReference<typeof OpenAIConfigSchema>,\n request: GenerateRequest<typeof OpenAIConfigSchema>\n) {\n const openAiMessages = toOpenAiMessages(request.messages);\n const mappedModelName =\n request.config?.version || model.version || model.name;\n const body = {\n model: mappedModelName,\n messages: openAiMessages,\n temperature: request.config?.temperature,\n max_tokens: request.config?.maxOutputTokens,\n top_p: request.config?.topP,\n stop: request.config?.stopSequences,\n frequency_penalty: request.config?.frequencyPenalty,\n logit_bias: request.config?.logitBias,\n logprobs: request.config?.logProbs,\n presence_penalty: request.config?.presencePenalty,\n seed: request.config?.seed,\n top_logprobs: request.config?.topLogProbs,\n user: request.config?.user,\n tools: request.tools?.map(toOpenAiTool),\n n: request.candidates,\n } as ChatCompletionCreateParamsNonStreaming;\n const response_format = request.output?.format;\n if (response_format) {\n if (\n response_format === 'json' &&\n model.info?.supports?.output?.includes('json')\n ) {\n body.response_format = {\n type: 'json_object',\n };\n } else if (\n response_format === 'text' &&\n model.info?.supports?.output?.includes('text')\n ) {\n // this is default format, don't need to set it\n // body.response_format = {\n // type: 'text',\n // };\n } else {\n throw new Error(`${response_format} format is not supported currently`);\n }\n }\n for (const key in body) {\n if (!body[key] || (Array.isArray(body[key]) && !body[key].length))\n delete body[key];\n }\n return body;\n}\n\nexport function openaiCompatibleModel<C extends typeof OpenAIConfigSchema>(\n ai: Genkit,\n model: ModelReference<any>,\n clientFactory: (request: GenerateRequest<C>) => Promise<OpenAI>\n): ModelAction<C> {\n const modelId = model.name;\n if (!model) throw new Error(`Unsupported model: ${name}`);\n\n return ai.defineModel(\n {\n name: modelId,\n ...model.info,\n configSchema: model.configSchema,\n },\n async (\n request: GenerateRequest<C>,\n sendChunk?: StreamingCallback<GenerateResponseChunkData>\n ): Promise<GenerateResponseData> => {\n let response: ChatCompletion;\n const client = await clientFactory(request);\n const body = toRequestBody(model, request);\n if (sendChunk) {\n const stream = client.beta.chat.completions.stream({\n ...body,\n stream: true,\n });\n for await (const chunk of stream) {\n chunk.choices?.forEach((chunk) => {\n const c = fromOpenAiChunkChoice(chunk);\n sendChunk({\n index: c.index,\n content: c.message.content,\n });\n });\n }\n response = await stream.finalChatCompletion();\n } else {\n response = await client.chat.completions.create(body);\n }\n return {\n candidates: response.choices.map((c) =>\n fromOpenAiChoice(c, request.output?.format === 'json')\n ),\n usage: {\n inputTokens: response.usage?.prompt_tokens,\n outputTokens: response.usage?.completion_tokens,\n totalTokens: response.usage?.total_tokens,\n },\n custom: response,\n };\n }\n );\n}\n"],"mappings":"AAgBA,SAAiB,SAA4B,SAAS;AACtD;AAAA,EAGE;AAAA,OAUK;AAiBA,MAAM,qBAAqB,6BAA6B,OAAO;AAAA;AAAA;AAAA;AAAA,EAIpE,kBAAkB,EACf,OAAO,EACP,IAAI,EAAE,EACN,IAAI,CAAC,EACL;AAAA,IACC;AAAA,EAGF,EACC,SAAS;AAAA,EACZ,WAAW,EACR;AAAA,IACC,EAAE,OAAO,EAAE,SAAS,eAAe;AAAA,IACnC,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE,IAAI,GAAG,EAAE,SAAS,wBAAwB;AAAA,EACjE,EACC;AAAA,IACC;AAAA,EAOF,EACC,SAAS;AAAA,EACZ,UAAU,EACP,QAAQ,EACR;AAAA,IACC;AAAA,EACF,EACC,SAAS;AAAA,EACZ,iBAAiB,EACd,OAAO,EACP,IAAI,EAAE,EACN,IAAI,CAAC,EACL;AAAA,IACC;AAAA,EAGF,EACC,SAAS;AAAA,EACZ,MAAM,EACH,OAAO,EACP,IAAI,EACJ;AAAA,IACC;AAAA,EAKF,EACC,SAAS;AAAA,EACZ,aAAa,EACV,OAAO,EACP,IAAI,EACJ,IAAI,CAAC,EACL,IAAI,EAAE,EACN;AAAA,IACC;AAAA,EAGF,EACC,SAAS;AAAA,EACZ,MAAM,EACH,OAAO,EACP;AAAA,IACC;AAAA,EACF,EACC,SAAS;AACd,CAAC;AAEM,SAAS,aAAa,MAAgC;AAC3D,UAAQ,MAAM;AAAA,IACZ,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT;AACE,YAAM,IAAI,MAAM,QAAQ,IAAI,iCAAiC;AAAA,EACjE;AACF;AAEA,SAAS,aAAa,MAA0C;AAC9D,SAAO;AAAA,IACL,MAAM;AAAA,IACN,UAAU;AAAA,MACR,MAAM,KAAK;AAAA,MACX,YAAY,KAAK,eAAe;AAAA,IAClC;AAAA,EACF;AACF;AAEO,SAAS,qBAAqB,MAAuC;AAC1E,MAAI,KAAK,MAAM;AACb,WAAO;AAAA,MACL,MAAM;AAAA,MACN,MAAM,KAAK;AAAA,IACb;AAAA,EACF,WAAW,KAAK,OAAO;AACrB,WAAO;AAAA,MACL,MAAM;AAAA,MACN,WAAW;AAAA,QACT,KAAK,KAAK,MAAM;AAAA,MAClB;AAAA,IACF;AAAA,EACF;AACA,QAAM;AAAA,IACJ,wEAAwE,KAAK,UAAU,IAAI,CAAC;AAAA,EAC9F;AACF;AAEO,SAAS,iBACd,UAC8B;AAC9B,QAAM,aAA2C,CAAC;AAClD,aAAW,WAAW,UAAU;AAC9B,UAAM,MAAM,IAAI,QAAQ,OAAO;AAC/B,UAAM,OAAO,aAAa,QAAQ,IAAI;AACtC,YAAQ,MAAM;AAAA,MACZ,KAAK;AACH,mBAAW,KAAK;AAAA,UACd;AAAA,UACA,SAAS,IAAI,QAAQ,IAAI,CAAC,SAAS,qBAAqB,IAAI,CAAC;AAAA,QAC/D,CAAC;AACD;AAAA,MACF,KAAK;AACH,mBAAW,KAAK;AAAA,UACd;AAAA,UACA,SAAS,IAAI;AAAA,QACf,CAAC;AACD;AAAA,MACF,KAAK,aAAa;AAChB,cAAM,YAA6C,IAAI,QACpD;AAAA,UACC,CACE,SAGG,QAAQ,KAAK,WAAW;AAAA,QAC/B,EACC,IAAI,CAAC,UAAU;AAAA,UACd,IAAI,KAAK,YAAY,OAAO;AAAA,UAC5B,MAAM;AAAA,UACN,UAAU;AAAA,YACR,MAAM,KAAK,YAAY;AAAA,YACvB,WAAW,KAAK,UAAU,KAAK,YAAY,KAAK;AAAA,UAClD;AAAA,QACF,EAAE;AACJ,YAAI,UAAU,SAAS,GAAG;AACxB,qBAAW,KAAK;AAAA,YACd;AAAA,YACA,YAAY;AAAA,UACd,CAAC;AAAA,QACH,OAAO;AACL,qBAAW,KAAK;AAAA,YACd;AAAA,YACA,SAAS,IAAI;AAAA,UACf,CAAC;AAAA,QACH;AACA;AAAA,MACF;AAAA,MACA,KAAK,QAAQ;AACX,cAAM,oBAAoB,IAAI,kBAAkB;AAChD,0BAAkB,IAAI,CAAC,SAAS;AAC9B,qBAAW,KAAK;AAAA,YACd;AAAA,YACA,cAAc,KAAK,aAAa,OAAO;AAAA,YACvC,SACE,OAAO,KAAK,aAAa,WAAW,WAChC,KAAK,aAAa,SAClB,KAAK,UAAU,KAAK,aAAa,MAAM;AAAA,UAC/C,CAAC;AAAA,QACH,CAAC;AACD;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;AAEA,MAAM,kBAGF;AAAA,EACF,QAAQ;AAAA,EACR,MAAM;AAAA,EACN,YAAY;AAAA,EACZ,gBAAgB;AAClB;AAEO,SAAS,mBACd,UAGiB;AACjB,MAAI,CAAC,SAAS,UAAU;AACtB,UAAM;AAAA,MACJ;AAAA,IACF;AAAA,EACF;AACA,QAAM,IAAI,SAAS;AACnB,SAAO;AAAA,IACL,aAAa;AAAA,MACX,MAAM,EAAE;AAAA,MACR,KAAK,SAAS;AAAA,MACd,OAAO,EAAE,YAAY,KAAK,MAAM,EAAE,SAAS,IAAI,EAAE;AAAA,IACnD;AAAA,EACF;AACF;AAEO,SAAS,iBACd,QACA,WAAW,OACI;AACf,QAAM,mBAAmB,OAAO,QAAQ,YAAY,IAAI,kBAAkB;AAC1E,SAAO;AAAA,IACL,OAAO,OAAO;AAAA,IACd,cAAc,gBAAgB,OAAO,aAAa,KAAK;AAAA,IACvD,SAAS;AAAA,MACP,MAAM;AAAA,MACN,SAAS;AAAA;AAAA;AAAA,QAGJ;AAAA,UACD;AAAA,QACE,WACI,EAAE,MAAM,KAAK,MAAM,OAAO,QAAQ,OAAQ,EAAE,IAC5C,EAAE,MAAM,OAAO,QAAQ,QAAS;AAAA,MACtC;AAAA,IACN;AAAA,IACA,QAAQ,CAAC;AAAA,EACX;AACF;AAEO,SAAS,sBACd,QACA,WAAW,OACI;AACf,QAAM,mBAAmB,OAAO,MAAM,YAAY,IAAI,kBAAkB;AACxE,SAAO;AAAA,IACL,OAAO,OAAO;AAAA,IACd,cAAc,OAAO,gBACjB,gBAAgB,OAAO,aAAa,KAAK,UACzC;AAAA,IACJ,SAAS;AAAA,MACP,MAAM;AAAA,MACN,SAAS,mBACJ,mBACD;AAAA,QACE,WACI,EAAE,MAAM,KAAK,MAAM,OAAO,MAAM,OAAQ,EAAE,IAC1C,EAAE,MAAM,OAAO,MAAM,QAAS;AAAA,MACpC;AAAA,IACN;AAAA,IACA,QAAQ,CAAC;AAAA,EACX;AACF;AAEO,SAAS,cACd,OACA,SACA;AACA,QAAM,iBAAiB,iBAAiB,QAAQ,QAAQ;AACxD,QAAM,kBACJ,QAAQ,QAAQ,WAAW,MAAM,WAAW,MAAM;AACpD,QAAM,OAAO;AAAA,IACX,OAAO;AAAA,IACP,UAAU;AAAA,IACV,aAAa,QAAQ,QAAQ;AAAA,IAC7B,YAAY,QAAQ,QAAQ;AAAA,IAC5B,OAAO,QAAQ,QAAQ;AAAA,IACvB,MAAM,QAAQ,QAAQ;AAAA,IACtB,mBAAmB,QAAQ,QAAQ;AAAA,IACnC,YAAY,QAAQ,QAAQ;AAAA,IAC5B,UAAU,QAAQ,QAAQ;AAAA,IAC1B,kBAAkB,QAAQ,QAAQ;AAAA,IAClC,MAAM,QAAQ,QAAQ;AAAA,IACtB,cAAc,QAAQ,QAAQ;AAAA,IAC9B,MAAM,QAAQ,QAAQ;AAAA,IACtB,OAAO,QAAQ,OAAO,IAAI,YAAY;AAAA,IACtC,GAAG,QAAQ;AAAA,EACb;AACA,QAAM,kBAAkB,QAAQ,QAAQ;AACxC,MAAI,iBAAiB;AACnB,QACE,oBAAoB,UACpB,MAAM,MAAM,UAAU,QAAQ,SAAS,MAAM,GAC7C;AACA,WAAK,kBAAkB;AAAA,QACrB,MAAM;AAAA,MACR;AAAA,IACF,WACE,oBAAoB,UACpB,MAAM,MAAM,UAAU,QAAQ,SAAS,MAAM,GAC7C;AAAA,IAKF,OAAO;AACL,YAAM,IAAI,MAAM,GAAG,eAAe,oCAAoC;AAAA,IACxE;AAAA,EACF;AACA,aAAW,OAAO,MAAM;AACtB,QAAI,CAAC,KAAK,GAAG,KAAM,MAAM,QAAQ,KAAK,GAAG,CAAC,KAAK,CAAC,KAAK,GAAG,EAAE;AACxD,aAAO,KAAK,GAAG;AAAA,EACnB;AACA,SAAO;AACT;AAEO,SAAS,sBACd,IACA,OACA,eACgB;AAChB,QAAM,UAAU,MAAM;AACtB,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,sBAAsB,IAAI,EAAE;AAExD,SAAO,GAAG;AAAA,IACR;AAAA,MACE,MAAM;AAAA,MACN,GAAG,MAAM;AAAA,MACT,cAAc,MAAM;AAAA,IACtB;AAAA,IACA,OACE,SACA,cACkC;AAClC,UAAI;AACJ,YAAM,SAAS,MAAM,cAAc,OAAO;AAC1C,YAAM,OAAO,cAAc,OAAO,OAAO;AACzC,UAAI,WAAW;AACb,cAAM,SAAS,OAAO,KAAK,KAAK,YAAY,OAAO;AAAA,UACjD,GAAG;AAAA,UACH,QAAQ;AAAA,QACV,CAAC;AACD,yBAAiB,SAAS,QAAQ;AAChC,gBAAM,SAAS,QAAQ,CAACA,WAAU;AAChC,kBAAM,IAAI,sBAAsBA,MAAK;AACrC,sBAAU;AAAA,cACR,OAAO,EAAE;AAAA,cACT,SAAS,EAAE,QAAQ;AAAA,YACrB,CAAC;AAAA,UACH,CAAC;AAAA,QACH;AACA,mBAAW,MAAM,OAAO,oBAAoB;AAAA,MAC9C,OAAO;AACL,mBAAW,MAAM,OAAO,KAAK,YAAY,OAAO,IAAI;AAAA,MACtD;AACA,aAAO;AAAA,QACL,YAAY,SAAS,QAAQ;AAAA,UAAI,CAAC,MAChC,iBAAiB,GAAG,QAAQ,QAAQ,WAAW,MAAM;AAAA,QACvD;AAAA,QACA,OAAO;AAAA,UACL,aAAa,SAAS,OAAO;AAAA,UAC7B,cAAc,SAAS,OAAO;AAAA,UAC9B,aAAa,SAAS,OAAO;AAAA,QAC/B;AAAA,QACA,QAAQ;AAAA,MACV;AAAA,IACF;AAAA,EACF;AACF;","names":["chunk"]}