@langgraph-js/pro
Version:
The Pro SDK for LangGraph - seamlessly integrate your AI agents with frontend interfaces and build complex AI workflows
782 lines (750 loc) • 30.4 kB
text/typescript
import {
AIMessage,
AIMessageChunk,
BaseMessage,
BaseMessageChunk,
ChatMessage,
ChatMessageChunk,
FunctionMessageChunk,
HumanMessageChunk,
OpenAIToolCall,
SystemMessageChunk,
ToolCallChunk,
ToolMessage,
ToolMessageChunk,
parseBase64DataUrl,
parseMimeType,
StandardContentBlockConverter,
isDataContentBlock,
ContentBlock,
iife,
convertToProviderContentBlock,
} from "@langchain/core/messages";
import { convertLangChainToolCallToOpenAI, makeInvalidToolCall, parseToolCall } from "@langchain/core/output_parsers/openai_tools";
import { Converter } from "@langchain/core/utils/format";
import type { ChatCompletionContentPartText, ChatCompletionContentPartImage, ChatCompletionContentPartInputAudio, ChatCompletionContentPart } from "openai/resources/chat/completions";
import { OpenAI as OpenAIClient } from "openai";
import { handleMultiModalOutput } from "./output.js";
import { messageToOpenAIRole } from "@langchain/openai";
/**
* @deprecated This converter is an internal detail of the OpenAI provider. Do not use it directly. This will be revisited in a future release.
*/
export const completionsApiContentBlockConverter: StandardContentBlockConverter<{
text: ChatCompletionContentPartText;
image: ChatCompletionContentPartImage;
audio: ChatCompletionContentPartInputAudio;
file: ChatCompletionContentPart.File;
}> = {
providerName: "ChatOpenAI",
fromStandardTextBlock(block): ChatCompletionContentPartText {
return { type: "text", text: block.text };
},
fromStandardImageBlock(block): ChatCompletionContentPartImage {
if (block.source_type === "url") {
return {
type: "image_url",
image_url: {
url: block.url,
...(block.metadata?.detail ? { detail: block.metadata.detail as "auto" | "low" | "high" } : {}),
},
};
}
if (block.source_type === "base64") {
const url = `data:${block.mime_type ?? ""};base64,${block.data}`;
return {
type: "image_url",
image_url: {
url,
...(block.metadata?.detail ? { detail: block.metadata.detail as "auto" | "low" | "high" } : {}),
},
};
}
throw new Error(`Image content blocks with source_type ${block.source_type} are not supported for ChatOpenAI`);
},
fromStandardAudioBlock(block): ChatCompletionContentPartInputAudio {
if (block.source_type === "url") {
const data = parseBase64DataUrl({ dataUrl: block.url });
if (!data) {
throw new Error(`URL audio blocks with source_type ${block.source_type} must be formatted as a data URL for ChatOpenAI`);
}
const rawMimeType = data.mime_type || block.mime_type || "";
let mimeType: { type: string; subtype: string };
try {
mimeType = parseMimeType(rawMimeType);
} catch {
throw new Error(`Audio blocks with source_type ${block.source_type} must have mime type of audio/wav or audio/mp3`);
}
if (mimeType.type !== "audio" || (mimeType.subtype !== "wav" && mimeType.subtype !== "mp3")) {
throw new Error(`Audio blocks with source_type ${block.source_type} must have mime type of audio/wav or audio/mp3`);
}
return {
type: "input_audio",
input_audio: {
format: mimeType.subtype,
data: data.data,
},
};
}
if (block.source_type === "base64") {
let mimeType: { type: string; subtype: string };
try {
mimeType = parseMimeType(block.mime_type ?? "");
} catch {
throw new Error(`Audio blocks with source_type ${block.source_type} must have mime type of audio/wav or audio/mp3`);
}
if (mimeType.type !== "audio" || (mimeType.subtype !== "wav" && mimeType.subtype !== "mp3")) {
throw new Error(`Audio blocks with source_type ${block.source_type} must have mime type of audio/wav or audio/mp3`);
}
return {
type: "input_audio",
input_audio: {
format: mimeType.subtype,
data: block.data,
},
};
}
throw new Error(`Audio content blocks with source_type ${block.source_type} are not supported for ChatOpenAI`);
},
fromStandardFileBlock(block): ChatCompletionContentPart.File {
if (block.source_type === "url") {
const data = parseBase64DataUrl({ dataUrl: block.url });
if (!data) {
throw new Error(`URL file blocks with source_type ${block.source_type} must be formatted as a data URL for ChatOpenAI`);
}
return {
type: "file",
file: {
file_data: block.url, // formatted as base64 data URL
...(block.metadata?.filename || block.metadata?.name
? {
filename: (block.metadata?.filename || block.metadata?.name) as string,
}
: {}),
},
};
}
if (block.source_type === "base64") {
return {
type: "file",
file: {
file_data: `data:${block.mime_type ?? ""};base64,${block.data}`,
...(block.metadata?.filename || block.metadata?.name || block.metadata?.title
? {
filename: (block.metadata?.filename || block.metadata?.name || block.metadata?.title) as string,
}
: {}),
},
};
}
if (block.source_type === "id") {
return {
type: "file",
file: {
file_id: block.id,
},
};
}
throw new Error(`File content blocks with source_type ${block.source_type} are not supported for ChatOpenAI`);
},
};
/**
* Converts an OpenAI Chat Completions API message to a LangChain BaseMessage.
*
* This converter transforms messages from OpenAI's Chat Completions API format into
* LangChain's internal message representation, handling various message types and
* preserving metadata, tool calls, and other relevant information.
*
* @remarks
* The converter handles the following message roles:
* - `assistant`: Converted to {@link AIMessage} with support for tool calls, function calls,
* audio content, and multi-modal outputs
* - Other roles: Converted to generic {@link ChatMessage}
*
* For assistant messages, the converter:
* - Parses and validates tool calls, separating valid and invalid calls
* - Preserves function call information in additional_kwargs
* - Includes usage statistics and system fingerprint in response_metadata
* - Handles multi-modal content (text, images, audio)
* - Optionally includes the raw API response for debugging
*
* @param params - Conversion parameters
* @param params.message - The OpenAI chat completion message to convert
* @param params.rawResponse - The complete raw response from OpenAI's API, used to extract
* metadata like model name, usage statistics, and system fingerprint
* @param params.includeRawResponse - If true, includes the raw OpenAI response in the
* message's additional_kwargs under the `__raw_response` key. Useful for debugging
* or accessing provider-specific fields. Defaults to false.
*
* @returns A LangChain BaseMessage instance:
* - {@link AIMessage} for assistant messages with tool calls, metadata, and content
* - {@link ChatMessage} for all other message types
*
* @example
* ```typescript
* const baseMessage = convertCompletionsMessageToBaseMessage({
* message: {
* role: "assistant",
* content: "Hello! How can I help you?",
* tool_calls: [
* {
* id: "call_123",
* type: "function",
* function: { name: "get_weather", arguments: '{"location":"NYC"}' }
* }
* ]
* },
* rawResponse: completionResponse,
* includeRawResponse: true
* });
* // Returns an AIMessage with parsed tool calls and metadata
* ```
*
* @throws {Error} If tool call parsing fails, the invalid tool call is captured in
* the `invalid_tool_calls` array rather than throwing an error
*
*/
export const convertCompletionsMessageToBaseMessage: Converter<
{
message: OpenAIClient.Chat.Completions.ChatCompletionMessage;
rawResponse: OpenAIClient.Chat.Completions.ChatCompletion;
includeRawResponse?: boolean;
},
BaseMessage
> = ({ message, rawResponse, includeRawResponse }) => {
const rawToolCalls: OpenAIToolCall[] | undefined = message.tool_calls as OpenAIToolCall[] | undefined;
switch (message.role) {
case "assistant": {
const toolCalls = [];
const invalidToolCalls = [];
for (const rawToolCall of rawToolCalls ?? []) {
try {
toolCalls.push(parseToolCall(rawToolCall, { returnId: true }));
// eslint-disable-next-line @typescript-eslint/no-explicit-any
} catch (e: any) {
invalidToolCalls.push(makeInvalidToolCall(rawToolCall, e.message));
}
}
const additional_kwargs: Record<string, unknown> = {
function_call: message.function_call,
tool_calls: rawToolCalls,
};
if (includeRawResponse !== undefined) {
additional_kwargs.__raw_response = rawResponse;
}
const response_metadata: Record<string, unknown> | undefined = {
model_provider: "openai",
model_name: rawResponse.model,
...(rawResponse.system_fingerprint
? {
usage: { ...rawResponse.usage },
system_fingerprint: rawResponse.system_fingerprint,
}
: {}),
};
if (message.audio) {
additional_kwargs.audio = message.audio;
}
const content = handleMultiModalOutput(message.content || "", rawResponse.choices?.[0]?.message);
return new AIMessage({
content,
tool_calls: toolCalls,
invalid_tool_calls: invalidToolCalls,
additional_kwargs,
response_metadata,
id: rawResponse.id,
});
}
default:
return new ChatMessage(message.content || "", message.role ?? "unknown");
}
};
/**
* Converts an OpenAI Chat Completions API delta (streaming chunk) to a LangChain BaseMessageChunk.
*
* This converter is used during streaming responses to transform incremental updates from OpenAI's
* Chat Completions API into LangChain message chunks. It handles various message types, tool calls,
* function calls, audio content, and role-specific message chunk creation.
*
* @param params - Conversion parameters
* @param params.delta - The delta object from an OpenAI streaming chunk containing incremental
* message updates. May include content, role, tool_calls, function_call, audio, etc.
* @param params.rawResponse - The complete raw ChatCompletionChunk response from OpenAI,
* containing metadata like model info, usage stats, and the delta
* @param params.includeRawResponse - Optional flag to include the raw OpenAI response in the
* message chunk's additional_kwargs. Useful for debugging or accessing provider-specific data
* @param params.defaultRole - Optional default role to use if the delta doesn't specify one.
* Typically used to maintain role consistency across chunks in a streaming response
*
* @returns A BaseMessageChunk subclass appropriate for the message role:
* - HumanMessageChunk for "user" role
* - AIMessageChunk for "assistant" role (includes tool call chunks)
* - SystemMessageChunk for "system" or "developer" roles
* - FunctionMessageChunk for "function" role
* - ToolMessageChunk for "tool" role
* - ChatMessageChunk for any other role
*
* @example
* Basic streaming text chunk:
* ```typescript
* const chunk = convertCompletionsDeltaToBaseMessageChunk({
* delta: { role: "assistant", content: "Hello" },
* rawResponse: { id: "chatcmpl-123", model: "gpt-4", ... }
* });
* // Returns: AIMessageChunk with content "Hello"
* ```
*
* @example
* Streaming chunk with tool call:
* ```typescript
* const chunk = convertCompletionsDeltaToBaseMessageChunk({
* delta: {
* role: "assistant",
* tool_calls: [{
* index: 0,
* id: "call_123",
* function: { name: "get_weather", arguments: '{"location":' }
* }]
* },
* rawResponse: { id: "chatcmpl-123", ... }
* });
* // Returns: AIMessageChunk with tool_call_chunks containing partial tool call data
* ```
*
* @remarks
* - Tool calls are converted to ToolCallChunk objects with incremental data
* - Audio content includes the chunk index from the raw response
* - The "developer" role is mapped to SystemMessageChunk with a special marker
* - Response metadata includes model provider info and usage statistics
* - Function calls and tool calls are stored in additional_kwargs for compatibility
*/
export const convertCompletionsDeltaToBaseMessageChunk: Converter<
{
// eslint-disable-next-line @typescript-eslint/no-explicit-any
delta: Record<string, any>;
rawResponse: OpenAIClient.Chat.Completions.ChatCompletionChunk;
includeRawResponse?: boolean;
defaultRole?: OpenAIClient.Chat.ChatCompletionRole;
},
BaseMessageChunk
> = ({ delta, rawResponse, includeRawResponse, defaultRole }) => {
const role = delta.role ?? defaultRole;
const content = delta.content ?? "";
let additional_kwargs: Record<string, unknown>;
if (delta.function_call) {
additional_kwargs = {
function_call: delta.function_call,
};
} else if (delta.tool_calls) {
additional_kwargs = {
tool_calls: delta.tool_calls,
};
} else {
additional_kwargs = {};
}
if (includeRawResponse) {
additional_kwargs.__raw_response = rawResponse;
}
if (delta.audio) {
additional_kwargs.audio = {
...delta.audio,
index: rawResponse.choices[0].index,
};
}
const response_metadata = {
model_provider: "openai",
usage: { ...rawResponse.usage },
};
if (role === "user") {
return new HumanMessageChunk({ content, response_metadata });
} else if (role === "assistant") {
const toolCallChunks: ToolCallChunk[] = [];
if (Array.isArray(delta.tool_calls)) {
for (const rawToolCall of delta.tool_calls) {
toolCallChunks.push({
name: rawToolCall.function?.name,
args: rawToolCall.function?.arguments,
id: rawToolCall.id,
index: rawToolCall.index,
type: "tool_call_chunk",
});
}
}
return new AIMessageChunk({
content,
tool_call_chunks: toolCallChunks,
additional_kwargs,
id: rawResponse.id,
response_metadata,
});
} else if (role === "system") {
return new SystemMessageChunk({ content, response_metadata });
} else if (role === "developer") {
return new SystemMessageChunk({
content,
response_metadata,
additional_kwargs: {
__openai_role__: "developer",
},
});
} else if (role === "function") {
return new FunctionMessageChunk({
content,
additional_kwargs,
name: delta.name,
response_metadata,
});
} else if (role === "tool") {
return new ToolMessageChunk({
content,
additional_kwargs,
tool_call_id: delta.tool_call_id,
response_metadata,
});
} else {
return new ChatMessageChunk({ content, role, response_metadata });
}
};
/**
* Converts a standard LangChain content block to an OpenAI Completions API content part.
*
* This converter transforms LangChain's standardized content blocks (image, audio, file)
* into the format expected by OpenAI's Chat Completions API. It handles various content
* types including images (URL or base64), audio (base64), and files (data or file ID).
*
* @param block - The standard content block to convert. Can be an image, audio, or file block.
*
* @returns An OpenAI Chat Completions content part object, or undefined if the block
* cannot be converted (e.g., missing required data).
*
* @example
* Image with URL:
* ```typescript
* const block = { type: "image", url: "https://example.com/image.jpg" };
* const part = convertStandardContentBlockToCompletionsContentPart(block);
* // Returns: { type: "image_url", image_url: { url: "https://example.com/image.jpg" } }
* ```
*
* @example
* Image with base64 data:
* ```typescript
* const block = { type: "image", data: "iVBORw0KGgo...", mimeType: "image/png" };
* const part = convertStandardContentBlockToCompletionsContentPart(block);
* // Returns: { type: "image_url", image_url: { url: "data:image/png;base64,iVBORw0KGgo..." } }
* ```
*/
export const convertStandardContentBlockToCompletionsContentPart: Converter<
ContentBlock.Standard,
| OpenAIClient.Chat.Completions.ChatCompletionContentPartImage
| OpenAIClient.Chat.Completions.ChatCompletionContentPartInputAudio
| OpenAIClient.Chat.Completions.ChatCompletionContentPart.File
| undefined
> = (block) => {
if (block.type === "image") {
if (block.url) {
return {
type: "image_url",
image_url: {
url: block.url,
},
};
} else if (block.data) {
return {
type: "image_url",
image_url: {
url: `data:${block.mimeType};base64,${block.data}`,
},
};
}
}
if (block.type === "audio") {
if (block.data) {
const format = iife(() => {
const [, format] = block.mimeType.split("/");
if (format === "wav" || format === "mp3") {
return format;
}
return "wav";
});
return {
type: "input_audio",
input_audio: {
data: block.data.toString(),
format,
},
};
}
}
if (block.type === "file") {
if (block.data) {
return {
type: "file",
file: {
file_data: block.data.toString(),
},
};
}
if (block.fileId) {
return {
type: "file",
file: {
file_id: block.fileId,
},
};
}
}
return undefined;
};
export function isReasoningModel(model?: string) {
if (!model) return false;
if (/^o\d/.test(model ?? "")) return true;
if (model.startsWith("gpt-5") && !model.startsWith("gpt-5-chat")) return true;
return false;
}
/**
* Converts a LangChain BaseMessage with standard content blocks to an OpenAI Chat Completions API message parameter.
*
* This converter transforms LangChain's standardized message format (using contentBlocks) into the format
* expected by OpenAI's Chat Completions API. It handles role mapping, content filtering, and multi-modal
* content conversion for various message types.
*
* @remarks
* The converter performs the following transformations:
* - Maps LangChain message roles to OpenAI API roles (user, assistant, system, developer, tool, function)
* - For reasoning models, automatically converts "system" role to "developer" role
* - Filters content blocks based on message role (most roles only include text blocks)
* - For user messages, converts multi-modal content blocks (images, audio, files) to OpenAI format
* - Preserves tool call IDs for tool messages and function names for function messages
*
* Role-specific behavior:
* - **developer**: Returns only text content blocks (used for reasoning models)
* - **system**: Returns only text content blocks
* - **assistant**: Returns only text content blocks
* - **tool**: Returns only text content blocks with tool_call_id preserved
* - **function**: Returns text content blocks joined as a single string with function name
* - **user** (default): Returns multi-modal content including text, images, audio, and files
*
* @param params - Conversion parameters
* @param params.message - The LangChain BaseMessage to convert. Must have contentBlocks property
* containing an array of standard content blocks (text, image, audio, file, etc.)
* @param params.model - Optional model name. Used to determine if special role mapping is needed
* (e.g., "system" -> "developer" for reasoning models like o1)
*
* @returns An OpenAI ChatCompletionMessageParam object formatted for the Chat Completions API.
* The structure varies by role:
* - Developer/System/Assistant: `{ role, content: TextBlock[] }`
* - Tool: `{ role: "tool", tool_call_id, content: TextBlock[] }`
* - Function: `{ role: "function", name, content: string }`
* - User: `{ role: "user", content: Array<TextPart | ImagePart | AudioPart | FilePart> }`
*
* @example
* Simple text message:
* ```typescript
* const message = new HumanMessage({
* content: [{ type: "text", text: "Hello!" }]
* });
* const param = convertStandardContentMessageToCompletionsMessage({ message });
* // Returns: { role: "user", content: [{ type: "text", text: "Hello!" }] }
* ```
*
* @example
* Multi-modal user message with image:
* ```typescript
* const message = new HumanMessage({
* content: [
* { type: "text", text: "What's in this image?" },
* { type: "image", url: "https://example.com/image.jpg" }
* ]
* });
* const param = convertStandardContentMessageToCompletionsMessage({ message });
* // Returns: {
* // role: "user",
* // content: [
* // { type: "text", text: "What's in this image?" },
* // { type: "image_url", image_url: { url: "https://example.com/image.jpg" } }
* // ]
* // }
* ```
*/
export const convertStandardContentMessageToCompletionsMessage: Converter<{ message: BaseMessage; model?: string }, OpenAIClient.Chat.Completions.ChatCompletionMessageParam> = ({
message,
model,
}) => {
let role = messageToOpenAIRole(message);
if (role === "system" && isReasoningModel(model)) {
role = "developer";
}
if (role === "developer") {
return {
role: "developer",
content: message.contentBlocks.filter((block) => block.type === "text"),
};
} else if (role === "system") {
return {
role: "system",
content: message.contentBlocks.filter((block) => block.type === "text"),
};
} else if (role === "assistant") {
return {
role: "assistant",
content: message.contentBlocks.filter((block) => block.type === "text"),
};
} else if (role === "tool" && ToolMessage.isInstance(message)) {
return {
role: "tool",
tool_call_id: message.tool_call_id,
content: message.contentBlocks.filter((block) => block.type === "text"),
};
} else if (role === "function") {
return {
role: "function",
name: message.name ?? "",
content: message.contentBlocks.filter((block) => block.type === "text").join(""),
};
}
// Default to user message handling
function* iterateUserContent(blocks: ContentBlock.Standard[]) {
for (const block of blocks) {
if (block.type === "text") {
yield {
type: "text" as const,
text: block.text,
};
}
const data = convertStandardContentBlockToCompletionsContentPart(block);
if (data) {
yield data;
}
}
}
return {
role: "user",
content: Array.from(iterateUserContent(message.contentBlocks)),
};
};
/**
* Converts an array of LangChain BaseMessages to OpenAI Chat Completions API message parameters.
*
* This converter transforms LangChain's internal message representation into the format required
* by OpenAI's Chat Completions API. It handles various message types, roles, content formats,
* tool calls, function calls, audio messages, and special model-specific requirements.
*
* @remarks
* The converter performs several key transformations:
* - Maps LangChain message types to OpenAI roles (user, assistant, system, tool, function, developer)
* - Converts standard content blocks (v1 format) using a specialized converter
* - Handles multimodal content including text, images, audio, and data blocks
* - Preserves tool calls and function calls with proper formatting
* - Applies model-specific role mappings (e.g., "system" → "developer" for reasoning models)
* - Splits audio messages into separate message parameters when needed
*
* @param params - Conversion parameters
* @param params.messages - Array of LangChain BaseMessages to convert. Can include any message
* type: HumanMessage, AIMessage, SystemMessage, ToolMessage, FunctionMessage, etc.
* @param params.model - Optional model name used to determine if special role mapping is needed.
* For reasoning models (o1, o3, etc.), "system" role is converted to "developer" role.
*
* @returns Array of ChatCompletionMessageParam objects formatted for OpenAI's Chat Completions API.
* Some messages may be split into multiple parameters (e.g., audio messages).
*
* @example
* Basic message conversion:
* ```typescript
* const messages = [
* new HumanMessage("What's the weather like?"),
* new AIMessage("Let me check that for you.")
* ];
*
* const params = convertMessagesToCompletionsMessageParams({
* messages,
* model: "gpt-4"
* });
* // Returns:
* // [
* // { role: "user", content: "What's the weather like?" },
* // { role: "assistant", content: "Let me check that for you." }
* // ]
* ```
*
* @example
* Message with tool calls:
* ```typescript
* const messages = [
* new AIMessage({
* content: "",
* tool_calls: [{
* id: "call_123",
* name: "get_weather",
* args: { location: "San Francisco" }
* }]
* })
* ];
*
* const params = convertMessagesToCompletionsMessageParams({ messages });
* // Returns:
* // [{
* // role: "assistant",
* // content: "",
* // tool_calls: [{
* // id: "call_123",
* // type: "function",
* // function: { name: "get_weather", arguments: '{"location":"San Francisco"}' }
* // }]
* // }]
* ```
*/
export const convertMessagesToCompletionsMessageParams: Converter<{ messages: BaseMessage[]; model?: string }, OpenAIClient.Chat.Completions.ChatCompletionMessageParam[]> = ({ messages, model }) => {
return messages.flatMap((message) => {
if ("output_version" in message.response_metadata && message.response_metadata?.output_version === "v1") {
return convertStandardContentMessageToCompletionsMessage({ message });
}
let role = messageToOpenAIRole(message);
if (role === "system" && isReasoningModel(model)) {
role = "developer";
}
const content =
typeof message.content === "string"
? message.content
: message.content.map((m) => {
if (isDataContentBlock(m)) {
return convertToProviderContentBlock(m, completionsApiContentBlockConverter);
}
return m;
});
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const completionParam: Record<string, any> = {
role,
content,
};
if (message.name != null) {
completionParam.name = message.name;
}
if (message.additional_kwargs.function_call != null) {
completionParam.function_call = message.additional_kwargs.function_call;
completionParam.content = "";
}
if (message.additional_kwargs?.reasoning_content && model?.includes("deepseek")) {
completionParam.reasoning_content = message.additional_kwargs.reasoning_content;
}
if (AIMessage.isInstance(message) && !!message.tool_calls?.length) {
completionParam.tool_calls = message.tool_calls.map(convertLangChainToolCallToOpenAI);
completionParam.content = "";
} else {
if (message.additional_kwargs.tool_calls != null) {
completionParam.tool_calls = message.additional_kwargs.tool_calls;
}
if (ToolMessage.isInstance(message) && message.tool_call_id != null) {
completionParam.tool_call_id = message.tool_call_id;
}
}
if (message.additional_kwargs.audio && typeof message.additional_kwargs.audio === "object" && "id" in message.additional_kwargs.audio) {
const audioMessage = {
role: "assistant",
audio: {
id: message.additional_kwargs.audio.id,
},
};
return [completionParam, audioMessage] as OpenAIClient.Chat.Completions.ChatCompletionMessageParam[];
}
return completionParam as OpenAIClient.Chat.Completions.ChatCompletionMessageParam;
});
};