@langchain/openai
Version:
OpenAI integrations for LangChain.js
1,314 lines • 99 kB
JavaScript
import { OpenAI as OpenAIClient } from "openai";
import { AIMessage, AIMessageChunk, ChatMessage, ChatMessageChunk, FunctionMessageChunk, HumanMessageChunk, SystemMessageChunk, ToolMessageChunk, isAIMessage, parseBase64DataUrl, parseMimeType, convertToProviderContentBlock, isDataContentBlock, } from "@langchain/core/messages";
import { ChatGenerationChunk, } from "@langchain/core/outputs";
import { getEnvironmentVariable } from "@langchain/core/utils/env";
import { BaseChatModel, } from "@langchain/core/language_models/chat_models";
import { isOpenAITool, } from "@langchain/core/language_models/base";
import { RunnableLambda, RunnablePassthrough, RunnableSequence, } from "@langchain/core/runnables";
import { JsonOutputParser, StructuredOutputParser, } from "@langchain/core/output_parsers";
import { JsonOutputKeyToolsParser, convertLangChainToolCallToOpenAI, makeInvalidToolCall, parseToolCall, } from "@langchain/core/output_parsers/openai_tools";
import { getSchemaDescription, isInteropZodSchema, } from "@langchain/core/utils/types";
import { toJsonSchema, } from "@langchain/core/utils/json_schema";
import { getEndpoint } from "./utils/azure.js";
import { formatToOpenAIToolChoice, interopZodResponseFormat, wrapOpenAIClientError, } from "./utils/openai.js";
import { formatFunctionDefinitions, } from "./utils/openai-format-fndef.js";
import { _convertToOpenAITool } from "./utils/tools.js";
const _FUNCTION_CALL_IDS_MAP_KEY = "__openai_function_call_ids__";
function isBuiltInTool(tool) {
return "type" in tool && tool.type !== "function";
}
function isBuiltInToolChoice(tool_choice) {
return (tool_choice != null &&
typeof tool_choice === "object" &&
"type" in tool_choice &&
tool_choice.type !== "function");
}
function isReasoningModel(model) {
return model && /^o\d/.test(model);
}
function isStructuredOutputMethodParams(x
// eslint-disable-next-line @typescript-eslint/no-explicit-any
) {
return (x !== undefined &&
// eslint-disable-next-line @typescript-eslint/no-explicit-any
typeof x.schema ===
"object");
}
function extractGenericMessageCustomRole(message) {
if (message.role !== "system" &&
message.role !== "developer" &&
message.role !== "assistant" &&
message.role !== "user" &&
message.role !== "function" &&
message.role !== "tool") {
console.warn(`Unknown message role: ${message.role}`);
}
return message.role;
}
export function messageToOpenAIRole(message) {
const type = message._getType();
switch (type) {
case "system":
return "system";
case "ai":
return "assistant";
case "human":
return "user";
case "function":
return "function";
case "tool":
return "tool";
case "generic": {
if (!ChatMessage.isInstance(message))
throw new Error("Invalid generic chat message");
return extractGenericMessageCustomRole(message);
}
default:
throw new Error(`Unknown message type: ${type}`);
}
}
const completionsApiContentBlockConverter = {
providerName: "ChatOpenAI",
fromStandardTextBlock(block) {
return { type: "text", text: block.text };
},
fromStandardImageBlock(block) {
if (block.source_type === "url") {
return {
type: "image_url",
image_url: {
url: block.url,
...(block.metadata?.detail
? { detail: block.metadata.detail }
: {}),
},
};
}
if (block.source_type === "base64") {
const url = `data:${block.mime_type ?? ""};base64,${block.data}`;
return {
type: "image_url",
image_url: {
url,
...(block.metadata?.detail
? { detail: block.metadata.detail }
: {}),
},
};
}
throw new Error(`Image content blocks with source_type ${block.source_type} are not supported for ChatOpenAI`);
},
fromStandardAudioBlock(block) {
if (block.source_type === "url") {
const data = parseBase64DataUrl({ dataUrl: block.url });
if (!data) {
throw new Error(`URL audio blocks with source_type ${block.source_type} must be formatted as a data URL for ChatOpenAI`);
}
const rawMimeType = data.mime_type || block.mime_type || "";
let mimeType;
try {
mimeType = parseMimeType(rawMimeType);
}
catch {
throw new Error(`Audio blocks with source_type ${block.source_type} must have mime type of audio/wav or audio/mp3`);
}
if (mimeType.type !== "audio" ||
(mimeType.subtype !== "wav" && mimeType.subtype !== "mp3")) {
throw new Error(`Audio blocks with source_type ${block.source_type} must have mime type of audio/wav or audio/mp3`);
}
return {
type: "input_audio",
input_audio: {
format: mimeType.subtype,
data: data.data,
},
};
}
if (block.source_type === "base64") {
let mimeType;
try {
mimeType = parseMimeType(block.mime_type ?? "");
}
catch {
throw new Error(`Audio blocks with source_type ${block.source_type} must have mime type of audio/wav or audio/mp3`);
}
if (mimeType.type !== "audio" ||
(mimeType.subtype !== "wav" && mimeType.subtype !== "mp3")) {
throw new Error(`Audio blocks with source_type ${block.source_type} must have mime type of audio/wav or audio/mp3`);
}
return {
type: "input_audio",
input_audio: {
format: mimeType.subtype,
data: block.data,
},
};
}
throw new Error(`Audio content blocks with source_type ${block.source_type} are not supported for ChatOpenAI`);
},
fromStandardFileBlock(block) {
if (block.source_type === "url") {
const data = parseBase64DataUrl({ dataUrl: block.url });
if (!data) {
throw new Error(`URL file blocks with source_type ${block.source_type} must be formatted as a data URL for ChatOpenAI`);
}
return {
type: "file",
file: {
file_data: block.url, // formatted as base64 data URL
...(block.metadata?.filename || block.metadata?.name
? {
filename: (block.metadata?.filename ||
block.metadata?.name),
}
: {}),
},
};
}
if (block.source_type === "base64") {
return {
type: "file",
file: {
file_data: `data:${block.mime_type ?? ""};base64,${block.data}`,
...(block.metadata?.filename ||
block.metadata?.name ||
block.metadata?.title
? {
filename: (block.metadata?.filename ||
block.metadata?.name ||
block.metadata?.title),
}
: {}),
},
};
}
if (block.source_type === "id") {
return {
type: "file",
file: {
file_id: block.id,
},
};
}
throw new Error(`File content blocks with source_type ${block.source_type} are not supported for ChatOpenAI`);
},
};
// Used in LangSmith, export is important here
// TODO: put this conversion elsewhere
export function _convertMessagesToOpenAIParams(messages, model) {
// TODO: Function messages do not support array content, fix cast
return messages.flatMap((message) => {
let role = messageToOpenAIRole(message);
if (role === "system" && isReasoningModel(model)) {
role = "developer";
}
const content = typeof message.content === "string"
? message.content
: message.content.map((m) => {
if (isDataContentBlock(m)) {
return convertToProviderContentBlock(m, completionsApiContentBlockConverter);
}
return m;
});
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const completionParam = {
role,
content,
};
if (message.name != null) {
completionParam.name = message.name;
}
if (message.additional_kwargs.function_call != null) {
completionParam.function_call = message.additional_kwargs.function_call;
completionParam.content = "";
}
if (isAIMessage(message) && !!message.tool_calls?.length) {
completionParam.tool_calls = message.tool_calls.map(convertLangChainToolCallToOpenAI);
completionParam.content = "";
}
else {
if (message.additional_kwargs.tool_calls != null) {
completionParam.tool_calls = message.additional_kwargs.tool_calls;
}
if (message.tool_call_id != null) {
completionParam.tool_call_id = message.tool_call_id;
}
}
if (message.additional_kwargs.audio &&
typeof message.additional_kwargs.audio === "object" &&
"id" in message.additional_kwargs.audio) {
const audioMessage = {
role: "assistant",
audio: {
id: message.additional_kwargs.audio.id,
},
};
return [
completionParam,
audioMessage,
];
}
return completionParam;
});
}
/** @internal */
class BaseChatOpenAI extends BaseChatModel {
_llmType() {
return "openai";
}
static lc_name() {
return "ChatOpenAI";
}
get callKeys() {
return [
...super.callKeys,
"options",
"function_call",
"functions",
"tools",
"tool_choice",
"promptIndex",
"response_format",
"seed",
"reasoning",
"service_tier",
];
}
get lc_secrets() {
return {
apiKey: "OPENAI_API_KEY",
organization: "OPENAI_ORGANIZATION",
};
}
get lc_aliases() {
return {
apiKey: "openai_api_key",
modelName: "model",
};
}
get lc_serializable_keys() {
return [
"configuration",
"logprobs",
"topLogprobs",
"prefixMessages",
"supportsStrictToolCalling",
"modalities",
"audio",
"temperature",
"maxTokens",
"topP",
"frequencyPenalty",
"presencePenalty",
"n",
"logitBias",
"user",
"streaming",
"streamUsage",
"model",
"modelName",
"modelKwargs",
"stop",
"stopSequences",
"timeout",
"apiKey",
"cache",
"maxConcurrency",
"maxRetries",
"verbose",
"callbacks",
"tags",
"metadata",
"disableStreaming",
"zdrEnabled",
"reasoning",
];
}
getLsParams(options) {
const params = this.invocationParams(options);
return {
ls_provider: "openai",
ls_model_name: this.model,
ls_model_type: "chat",
ls_temperature: params.temperature ?? undefined,
ls_max_tokens: params.max_tokens ?? undefined,
ls_stop: options.stop,
};
}
/** @ignore */
_identifyingParams() {
return {
model_name: this.model,
...this.invocationParams(),
...this.clientConfig,
};
}
/**
* Get the identifying parameters for the model
*/
identifyingParams() {
return this._identifyingParams();
}
constructor(fields) {
super(fields ?? {});
Object.defineProperty(this, "temperature", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "topP", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "frequencyPenalty", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "presencePenalty", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "n", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "logitBias", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "model", {
enumerable: true,
configurable: true,
writable: true,
value: "gpt-3.5-turbo"
});
Object.defineProperty(this, "modelKwargs", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "stop", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "stopSequences", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "user", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "timeout", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "streaming", {
enumerable: true,
configurable: true,
writable: true,
value: false
});
Object.defineProperty(this, "streamUsage", {
enumerable: true,
configurable: true,
writable: true,
value: true
});
Object.defineProperty(this, "maxTokens", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "logprobs", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "topLogprobs", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "apiKey", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "organization", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "__includeRawResponse", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "client", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "clientConfig", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
/**
* Whether the model supports the `strict` argument when passing in tools.
* If `undefined` the `strict` argument will not be passed to OpenAI.
*/
Object.defineProperty(this, "supportsStrictToolCalling", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "audio", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "modalities", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "reasoning", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
/**
* Must be set to `true` in tenancies with Zero Data Retention. Setting to `true` will disable
* output storage in the Responses API, but this DOES NOT enable Zero Data Retention in your
* OpenAI organization or project. This must be configured directly with OpenAI.
*
* See:
* https://help.openai.com/en/articles/10503543-data-residency-for-the-openai-api
* https://platform.openai.com/docs/api-reference/responses/create#responses-create-store
*
* @default false
*/
Object.defineProperty(this, "zdrEnabled", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
/**
* Service tier to use for this request. Can be "auto", "default", or "flex" or "priority".
* Specifies the service tier for prioritization and latency optimization.
*/
Object.defineProperty(this, "service_tier", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "lc_serializable", {
enumerable: true,
configurable: true,
writable: true,
value: true
});
this.apiKey =
fields?.apiKey ??
fields?.configuration?.apiKey ??
getEnvironmentVariable("OPENAI_API_KEY");
this.organization =
fields?.configuration?.organization ??
getEnvironmentVariable("OPENAI_ORGANIZATION");
this.model = fields?.model ?? fields?.modelName ?? this.model;
this.modelKwargs = fields?.modelKwargs ?? {};
this.timeout = fields?.timeout;
this.temperature = fields?.temperature ?? this.temperature;
this.topP = fields?.topP ?? this.topP;
this.frequencyPenalty = fields?.frequencyPenalty ?? this.frequencyPenalty;
this.presencePenalty = fields?.presencePenalty ?? this.presencePenalty;
this.logprobs = fields?.logprobs;
this.topLogprobs = fields?.topLogprobs;
this.n = fields?.n ?? this.n;
this.logitBias = fields?.logitBias;
this.stop = fields?.stopSequences ?? fields?.stop;
this.stopSequences = this.stop;
this.user = fields?.user;
this.__includeRawResponse = fields?.__includeRawResponse;
this.audio = fields?.audio;
this.modalities = fields?.modalities;
this.reasoning = fields?.reasoning;
this.maxTokens = fields?.maxCompletionTokens ?? fields?.maxTokens;
this.disableStreaming = fields?.disableStreaming ?? this.disableStreaming;
this.streaming = fields?.streaming ?? false;
if (this.disableStreaming)
this.streaming = false;
this.streamUsage = fields?.streamUsage ?? this.streamUsage;
if (this.disableStreaming)
this.streamUsage = false;
this.clientConfig = {
apiKey: this.apiKey,
organization: this.organization,
dangerouslyAllowBrowser: true,
...fields?.configuration,
};
// If `supportsStrictToolCalling` is explicitly set, use that value.
// Else leave undefined so it's not passed to OpenAI.
if (fields?.supportsStrictToolCalling !== undefined) {
this.supportsStrictToolCalling = fields.supportsStrictToolCalling;
}
if (fields?.service_tier !== undefined) {
this.service_tier = fields.service_tier;
}
this.zdrEnabled = fields?.zdrEnabled ?? false;
}
/**
* Returns backwards compatible reasoning parameters from constructor params and call options
* @internal
*/
_getReasoningParams(options) {
if (!isReasoningModel(this.model)) {
return;
}
// apply options in reverse order of importance -- newer options supersede older options
let reasoning;
if (this.reasoning !== undefined) {
reasoning = {
...reasoning,
...this.reasoning,
};
}
if (options?.reasoning !== undefined) {
reasoning = {
...reasoning,
...options.reasoning,
};
}
return reasoning;
}
/**
* Returns an openai compatible response format from a set of options
* @internal
*/
_getResponseFormat(resFormat) {
if (resFormat &&
resFormat.type === "json_schema" &&
resFormat.json_schema.schema &&
isInteropZodSchema(resFormat.json_schema.schema)) {
return interopZodResponseFormat(resFormat.json_schema.schema, resFormat.json_schema.name, {
description: resFormat.json_schema.description,
});
}
return resFormat;
}
_getClientOptions(options) {
if (!this.client) {
const openAIEndpointConfig = {
baseURL: this.clientConfig.baseURL,
};
const endpoint = getEndpoint(openAIEndpointConfig);
const params = {
...this.clientConfig,
baseURL: endpoint,
timeout: this.timeout,
maxRetries: 0,
};
if (!params.baseURL) {
delete params.baseURL;
}
this.client = new OpenAIClient(params);
}
const requestOptions = {
...this.clientConfig,
...options,
};
return requestOptions;
}
// TODO: move to completions class
_convertChatOpenAIToolToCompletionsTool(tool, fields) {
if (isOpenAITool(tool)) {
if (fields?.strict !== undefined) {
return {
...tool,
function: {
...tool.function,
strict: fields.strict,
},
};
}
return tool;
}
return _convertToOpenAITool(tool, fields);
}
bindTools(tools, kwargs) {
let strict;
if (kwargs?.strict !== undefined) {
strict = kwargs.strict;
}
else if (this.supportsStrictToolCalling !== undefined) {
strict = this.supportsStrictToolCalling;
}
return this.withConfig({
tools: tools.map((tool) => isBuiltInTool(tool)
? tool
: this._convertChatOpenAIToolToCompletionsTool(tool, { strict })),
...kwargs,
});
}
/** @ignore */
_combineLLMOutput(...llmOutputs) {
return llmOutputs.reduce((acc, llmOutput) => {
if (llmOutput && llmOutput.tokenUsage) {
acc.tokenUsage.completionTokens +=
llmOutput.tokenUsage.completionTokens ?? 0;
acc.tokenUsage.promptTokens += llmOutput.tokenUsage.promptTokens ?? 0;
acc.tokenUsage.totalTokens += llmOutput.tokenUsage.totalTokens ?? 0;
}
return acc;
}, {
tokenUsage: {
completionTokens: 0,
promptTokens: 0,
totalTokens: 0,
},
});
}
async getNumTokensFromMessages(messages) {
let totalCount = 0;
let tokensPerMessage = 0;
let tokensPerName = 0;
// From: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb
if (this.model === "gpt-3.5-turbo-0301") {
tokensPerMessage = 4;
tokensPerName = -1;
}
else {
tokensPerMessage = 3;
tokensPerName = 1;
}
const countPerMessage = await Promise.all(messages.map(async (message) => {
const textCount = await this.getNumTokens(message.content);
const roleCount = await this.getNumTokens(messageToOpenAIRole(message));
const nameCount = message.name !== undefined
? tokensPerName + (await this.getNumTokens(message.name))
: 0;
let count = textCount + tokensPerMessage + roleCount + nameCount;
// From: https://github.com/hmarr/openai-chat-tokens/blob/main/src/index.ts messageTokenEstimate
const openAIMessage = message;
if (openAIMessage._getType() === "function") {
count -= 2;
}
if (openAIMessage.additional_kwargs?.function_call) {
count += 3;
}
if (openAIMessage?.additional_kwargs.function_call?.name) {
count += await this.getNumTokens(openAIMessage.additional_kwargs.function_call?.name);
}
if (openAIMessage.additional_kwargs.function_call?.arguments) {
try {
count += await this.getNumTokens(
// Remove newlines and spaces
JSON.stringify(JSON.parse(openAIMessage.additional_kwargs.function_call?.arguments)));
}
catch (error) {
console.error("Error parsing function arguments", error, JSON.stringify(openAIMessage.additional_kwargs.function_call));
count += await this.getNumTokens(openAIMessage.additional_kwargs.function_call?.arguments);
}
}
totalCount += count;
return count;
}));
totalCount += 3; // every reply is primed with <|start|>assistant<|message|>
return { totalCount, countPerMessage };
}
/** @internal */
async _getNumTokensFromGenerations(generations) {
const generationUsages = await Promise.all(generations.map(async (generation) => {
if (generation.message.additional_kwargs?.function_call) {
return (await this.getNumTokensFromMessages([generation.message]))
.countPerMessage[0];
}
else {
return await this.getNumTokens(generation.message.content);
}
}));
return generationUsages.reduce((a, b) => a + b, 0);
}
/** @internal */
async _getEstimatedTokenCountFromPrompt(messages, functions, function_call) {
// It appears that if functions are present, the first system message is padded with a trailing newline. This
// was inferred by trying lots of combinations of messages and functions and seeing what the token counts were.
let tokens = (await this.getNumTokensFromMessages(messages)).totalCount;
// If there are functions, add the function definitions as they count towards token usage
if (functions && function_call !== "auto") {
const promptDefinitions = formatFunctionDefinitions(functions);
tokens += await this.getNumTokens(promptDefinitions);
tokens += 9; // Add nine per completion
}
// If there's a system message _and_ functions are present, subtract four tokens. I assume this is because
// functions typically add a system message, but reuse the first one if it's already there. This offsets
// the extra 9 tokens added by the function definitions.
if (functions && messages.find((m) => m._getType() === "system")) {
tokens -= 4;
}
// If function_call is 'none', add one token.
// If it's a FunctionCall object, add 4 + the number of tokens in the function name.
// If it's undefined or 'auto', don't add anything.
if (function_call === "none") {
tokens += 1;
}
else if (typeof function_call === "object") {
tokens += (await this.getNumTokens(function_call.name)) + 4;
}
return tokens;
}
withStructuredOutput(outputSchema, config) {
// ):
// | Runnable<BaseLanguageModelInput, RunOutput>
// | Runnable<
// BaseLanguageModelInput,
// { raw: BaseMessage; parsed: RunOutput }
// > {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
let schema;
let name;
let method;
let includeRaw;
if (isStructuredOutputMethodParams(outputSchema)) {
schema = outputSchema.schema;
name = outputSchema.name;
method = outputSchema.method;
includeRaw = outputSchema.includeRaw;
}
else {
schema = outputSchema;
name = config?.name;
method = config?.method;
includeRaw = config?.includeRaw;
}
let llm;
let outputParser;
if (config?.strict !== undefined && method === "jsonMode") {
throw new Error("Argument `strict` is only supported for `method` = 'function_calling'");
}
if (!this.model.startsWith("gpt-3") &&
!this.model.startsWith("gpt-4-") &&
this.model !== "gpt-4") {
if (method === undefined) {
method = "jsonSchema";
}
}
else if (method === "jsonSchema") {
console.warn(`[WARNING]: JSON Schema is not supported for model "${this.model}". Falling back to tool calling.`);
}
if (method === "jsonMode") {
let outputFormatSchema;
if (isInteropZodSchema(schema)) {
outputParser = StructuredOutputParser.fromZodSchema(schema);
outputFormatSchema = toJsonSchema(schema);
}
else {
outputParser = new JsonOutputParser();
}
llm = this.withConfig({
response_format: { type: "json_object" },
ls_structured_output_format: {
kwargs: { method: "jsonMode" },
schema: outputFormatSchema,
},
});
}
else if (method === "jsonSchema") {
llm = this.withConfig({
response_format: {
type: "json_schema",
json_schema: {
name: name ?? "extract",
description: getSchemaDescription(schema),
schema,
strict: config?.strict,
},
},
ls_structured_output_format: {
kwargs: { method: "jsonSchema" },
schema: toJsonSchema(schema),
},
});
if (isInteropZodSchema(schema)) {
const altParser = StructuredOutputParser.fromZodSchema(schema);
outputParser = RunnableLambda.from((aiMessage) => {
if ("parsed" in aiMessage.additional_kwargs) {
return aiMessage.additional_kwargs.parsed;
}
return altParser;
});
}
else {
outputParser = new JsonOutputParser();
}
}
else {
let functionName = name ?? "extract";
// Is function calling
if (isInteropZodSchema(schema)) {
const asJsonSchema = toJsonSchema(schema);
llm = this.withConfig({
tools: [
{
type: "function",
function: {
name: functionName,
description: asJsonSchema.description,
parameters: asJsonSchema,
},
},
],
tool_choice: {
type: "function",
function: {
name: functionName,
},
},
ls_structured_output_format: {
kwargs: { method: "functionCalling" },
schema: asJsonSchema,
},
// Do not pass `strict` argument to OpenAI if `config.strict` is undefined
...(config?.strict !== undefined ? { strict: config.strict } : {}),
});
outputParser = new JsonOutputKeyToolsParser({
returnSingle: true,
keyName: functionName,
zodSchema: schema,
});
}
else {
let openAIFunctionDefinition;
if (typeof schema.name === "string" &&
typeof schema.parameters === "object" &&
schema.parameters != null) {
openAIFunctionDefinition = schema;
functionName = schema.name;
}
else {
functionName = schema.title ?? functionName;
openAIFunctionDefinition = {
name: functionName,
description: schema.description ?? "",
parameters: schema,
};
}
llm = this.withConfig({
tools: [
{
type: "function",
function: openAIFunctionDefinition,
},
],
tool_choice: {
type: "function",
function: {
name: functionName,
},
},
ls_structured_output_format: {
kwargs: { method: "functionCalling" },
schema: toJsonSchema(schema),
},
// Do not pass `strict` argument to OpenAI if `config.strict` is undefined
...(config?.strict !== undefined ? { strict: config.strict } : {}),
});
outputParser = new JsonOutputKeyToolsParser({
returnSingle: true,
keyName: functionName,
});
}
}
if (!includeRaw) {
return llm.pipe(outputParser);
}
const parserAssign = RunnablePassthrough.assign({
// eslint-disable-next-line @typescript-eslint/no-explicit-any
parsed: (input, config) => outputParser.invoke(input.raw, config),
});
const parserNone = RunnablePassthrough.assign({
parsed: () => null,
});
const parsedWithFallback = parserAssign.withFallbacks({
fallbacks: [parserNone],
});
return RunnableSequence.from([{ raw: llm }, parsedWithFallback]);
}
}
/**
* OpenAI Responses API implementation.
*
* Will be exported in a later version of @langchain/openai.
*
* @internal
*/
export class ChatOpenAIResponses extends BaseChatOpenAI {
invocationParams(options) {
let strict;
if (options?.strict !== undefined) {
strict = options.strict;
}
else if (this.supportsStrictToolCalling !== undefined) {
strict = this.supportsStrictToolCalling;
}
const params = {
model: this.model,
temperature: this.temperature,
top_p: this.topP,
user: this.user,
// if include_usage is set or streamUsage then stream must be set to true.
stream: this.streaming,
previous_response_id: options?.previous_response_id,
truncation: options?.truncation,
include: options?.include,
tools: options?.tools?.length
? this._reduceChatOpenAITools(options.tools, {
stream: this.streaming,
strict,
})
: undefined,
tool_choice: isBuiltInToolChoice(options?.tool_choice)
? options?.tool_choice
: (() => {
const formatted = formatToOpenAIToolChoice(options?.tool_choice);
if (typeof formatted === "object" && "type" in formatted) {
return { type: "function", name: formatted.function.name };
}
else {
return undefined;
}
})(),
text: (() => {
if (options?.text)
return options.text;
const format = this._getResponseFormat(options?.response_format);
if (format?.type === "json_schema") {
if (format.json_schema.schema != null) {
return {
format: {
type: "json_schema",
schema: format.json_schema.schema,
description: format.json_schema.description,
name: format.json_schema.name,
strict: format.json_schema.strict,
},
};
}
return undefined;
}
return { format };
})(),
parallel_tool_calls: options?.parallel_tool_calls,
max_output_tokens: this.maxTokens === -1 ? undefined : this.maxTokens,
...(this.zdrEnabled ? { store: false } : {}),
...this.modelKwargs,
};
const reasoning = this._getReasoningParams(options);
if (reasoning !== undefined) {
params.reasoning = reasoning;
}
return params;
}
async _generate(messages, options) {
const invocationParams = this.invocationParams(options);
if (invocationParams.stream) {
const stream = this._streamResponseChunks(messages, options);
let finalChunk;
for await (const chunk of stream) {
chunk.message.response_metadata = {
...chunk.generationInfo,
...chunk.message.response_metadata,
};
finalChunk = finalChunk?.concat(chunk) ?? chunk;
}
return {
generations: finalChunk ? [finalChunk] : [],
llmOutput: {
estimatedTokenUsage: finalChunk?.message
?.usage_metadata,
},
};
}
else {
const input = this._convertMessagesToResponsesParams(messages);
const data = await this.completionWithRetry({
input,
...invocationParams,
stream: false,
}, { signal: options?.signal, ...options?.options });
return {
generations: [
{
text: data.output_text,
message: this._convertResponsesMessageToBaseMessage(data),
},
],
llmOutput: {
id: data.id,
estimatedTokenUsage: data.usage
? {
promptTokens: data.usage.input_tokens,
completionTokens: data.usage.output_tokens,
totalTokens: data.usage.total_tokens,
}
: undefined,
},
};
}
}
async *_streamResponseChunks(messages, options) {
const streamIterable = await this.completionWithRetry({
...this.invocationParams(options),
input: this._convertMessagesToResponsesParams(messages),
stream: true,
}, options);
for await (const data of streamIterable) {
const chunk = this._convertResponsesDeltaToBaseMessageChunk(data);
if (chunk == null)
continue;
yield chunk;
}
}
async completionWithRetry(request, requestOptions) {
return this.caller.call(async () => {
const clientOptions = this._getClientOptions(requestOptions);
try {
// use parse if dealing with json_schema
if (request.text?.format?.type === "json_schema" && !request.stream) {
return await this.client.responses.parse(request, clientOptions);
}
return await this.client.responses.create(request, clientOptions);
}
catch (e) {
const error = wrapOpenAIClientError(e);
throw error;
}
});
}
/** @internal */
_convertResponsesMessageToBaseMessage(response) {
if (response.error) {
// TODO: add support for `addLangChainErrorFields`
const error = new Error(response.error.message);
error.name = response.error.code;
throw error;
}
let messageId;
const content = [];
const tool_calls = [];
const invalid_tool_calls = [];
const response_metadata = {
model: response.model,
created_at: response.created_at,
id: response.id,
incomplete_details: response.incomplete_details,
metadata: response.metadata,
object: response.object,
status: response.status,
user: response.user,
service_tier: response.service_tier,
// for compatibility with chat completion calls.
model_name: response.model,
};
const additional_kwargs = {};
for (const item of response.output) {
if (item.type === "message") {
messageId = item.id;
content.push(...item.content.flatMap((part) => {
if (part.type === "output_text") {
if ("parsed" in part && part.parsed != null) {
additional_kwargs.parsed = part.parsed;
}
return {
type: "text",
text: part.text,
annotations: part.annotations,
};
}
if (part.type === "refusal") {
additional_kwargs.refusal = part.refusal;
return [];
}
return part;
}));
}
else if (item.type === "function_call") {
const fnAdapter = {
function: { name: item.name, arguments: item.arguments },
id: item.call_id,
};
try {
tool_calls.push(parseToolCall(fnAdapter, { returnId: true }));
}
catch (e) {
let errMessage;
if (typeof e === "object" &&
e != null &&
"message" in e &&
typeof e.message === "string") {
errMessage = e.message;
}
invalid_tool_calls.push(makeInvalidToolCall(fnAdapter, errMessage));
}
additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY] ??= {};
if (item.id) {
additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY][item.call_id] = item.id;
}
}
else if (item.type === "reasoning") {
additional_kwargs.reasoning = item;
}
else {
additional_kwargs.tool_outputs ??= [];
additional_kwargs.tool_outputs.push(item);
}
}
return new AIMessage({
id: messageId,
content,
tool_calls,
invalid_tool_calls,
usage_metadata: response.usage,
additional_kwargs,
response_metadata,
});
}
/** @internal */
_convertResponsesDeltaToBaseMessageChunk(chunk) {
const content = [];
let generationInfo = {};
let usage_metadata;
const tool_call_chunks = [];
const response_metadata = {};
const additional_kwargs = {};
let id;
if (chunk.type === "response.output_text.delta") {
content.push({
type: "text",
text: chunk.delta,
index: chunk.content_index,
});
}
else if (chunk.type === "response.output_text_annotation.added") {
content.push({
type: "text",
text: "",
annotations: [chunk.annotation],
index: chunk.content_index,
});
}
else if (chunk.type === "response.output_item.added" &&
chunk.item.type === "message") {
id = chunk.item.id;
}
else if (chunk.type === "response.output_item.added" &&
chunk.item.type === "function_call") {
tool_call_chunks.push({
type: "tool_call_chunk",
name: chunk.item.name,
args: chunk.item.arguments,
id: chunk.item.call_id,
index: chunk.output_index,
});
additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY] = {
[chunk.item.call_id]: chunk.item.id,
};
}
else if (chunk.type === "response.output_item.done" &&
[
"web_search_call",
"file_search_call",
"computer_call",
"code_interpreter_call",
"mcp_call",
"mcp_list_tools",
"mcp_approval_request",
"image_generation_call",
].includes(chunk.item.type)) {
additional_kwargs.tool_outputs = [chunk.item];
}
else if (chunk.type === "response.created") {
response_metadata.id = chunk.response.id;
response_metadata.model_name = chunk.response.model;
response_metadata.model = chunk.response.model;
}
else if (chunk.type === "response.completed") {
const msg = this._convertResponsesMessageToBaseMessage(chunk.response);
usage_metadata = chunk.response.usage;
if (chunk.response.text?.format?.type === "json_schema") {
additional_kwargs.parsed ??= JSON.parse(msg.text);
}
for (const [key, value] of Object.entries(chunk.response)) {
if (key !== "id")
response_metadata[key] = value;
}
}
else if (chunk.type === "response.function_call_arguments.delta") {
tool_call_chunks.push({
type: "tool_call_chunk",
args: chunk.delta,
index: chunk.output_index,
});
}
else if (chunk.type === "response.web_search_call.completed" ||
chunk.type === "response.file_search_call.completed") {
generationInfo = {
tool_outputs: {
id: chunk.item_id,
type: chunk.type.replace("response.", "").replace(".completed", ""),
status: "completed",
},
};
}
else if (chunk.type === "response.refusal.done") {
additional_kwargs.refusal = chunk.refusal;
}
else if (chunk.type === "response.output_item.added" &&
"item" in chunk &&
chunk.item.type === "reasoning") {
const summary = chunk
.item.summary
? chunk.item.summary.map((s, index) => ({
...s,
index,
}))
: undefined;
additional_kwargs.reasoning = {
// We only capture ID in the first chunk or else the concatenated result of all chunks will
// h