@openai/agents-core
Version:
The OpenAI Agents SDK is a lightweight yet powerful framework for building multi-agent workflows.
1,234 lines • 67.3 kB
JavaScript
import { consumeAgentToolRunResult, } from "./agent.mjs";
import { ModelBehaviorError, ToolCallError, UserError } from "./errors.mjs";
import { getTransferMessage } from "./handoff.mjs";
import { RunHandoffCallItem, RunHandoffOutputItem, RunMessageOutputItem, RunReasoningItem, RunToolApprovalItem, RunToolCallItem, RunToolCallOutputItem, } from "./items.mjs";
import logger from "./logger.mjs";
import { getLastTextFromOutputMessage } from "./utils/messages.mjs";
import { withFunctionSpan, withHandoffSpan } from "./tracing/createSpans.mjs";
import { getSchemaAndParserFromInputType } from "./utils/tools.mjs";
import { encodeUint8ArrayToBase64 } from "./utils/base64.mjs";
import { isArrayBufferView, isNodeBuffer, isSerializedBufferSnapshot, toSmartString, } from "./utils/smartString.mjs";
import { safeExecute } from "./utils/safeExecute.mjs";
import { addErrorToCurrentSpan } from "./tracing/context.mjs";
import { RunItemStreamEvent } from "./events.mjs";
import { z } from 'zod';
import { isZodObject } from "./utils/index.mjs";
function isApprovalItemLike(value) {
if (!value || typeof value !== 'object') {
return false;
}
if (!('rawItem' in value)) {
return false;
}
const rawItem = value.rawItem;
if (!rawItem || typeof rawItem !== 'object') {
return false;
}
const itemType = rawItem.type;
return itemType === 'function_call' || itemType === 'hosted_tool_call';
}
function getApprovalIdentity(approval) {
const rawItem = approval.rawItem;
if (!rawItem) {
return undefined;
}
if (rawItem.type === 'function_call' && rawItem.callId) {
return `function_call:${rawItem.callId}`;
}
if ('callId' in rawItem && rawItem.callId) {
return `${rawItem.type}:${rawItem.callId}`;
}
const id = 'id' in rawItem ? rawItem.id : undefined;
if (id) {
return `${rawItem.type}:${id}`;
}
const providerData = typeof rawItem.providerData === 'object' && rawItem.providerData
? rawItem.providerData
: undefined;
if (providerData?.id) {
return `${rawItem.type}:provider:${providerData.id}`;
}
const agentName = 'agent' in approval && approval.agent ? approval.agent.name : '';
try {
return `${agentName}:${rawItem.type}:${JSON.stringify(rawItem)}`;
}
catch {
return `${agentName}:${rawItem.type}`;
}
}
/**
* @internal
* Walks a raw model response and classifies each item so the runner can schedule follow-up work.
* Returns both the serializable RunItems (for history/streaming) and the actionable tool metadata.
*/
export function processModelResponse(modelResponse, agent, tools, handoffs) {
const items = [];
const runHandoffs = [];
const runFunctions = [];
const runComputerActions = [];
const runMCPApprovalRequests = [];
const toolsUsed = [];
const handoffMap = new Map(handoffs.map((h) => [h.toolName, h]));
// Resolve tools upfront so we can look up the concrete handler in O(1) while iterating outputs.
const functionMap = new Map(tools.filter((t) => t.type === 'function').map((t) => [t.name, t]));
const computerTool = tools.find((t) => t.type === 'computer');
const mcpToolMap = new Map(tools
.filter((t) => t.type === 'hosted_tool' && t.providerData?.type === 'mcp')
.map((t) => t)
.map((t) => [t.providerData.server_label, t]));
for (const output of modelResponse.output) {
if (output.type === 'message') {
if (output.role === 'assistant') {
items.push(new RunMessageOutputItem(output, agent));
}
}
else if (output.type === 'hosted_tool_call') {
items.push(new RunToolCallItem(output, agent));
const toolName = output.name;
toolsUsed.push(toolName);
if (output.providerData?.type === 'mcp_approval_request' ||
output.name === 'mcp_approval_request') {
// Hosted remote MCP server's approval process
const providerData = output.providerData;
const mcpServerLabel = providerData.server_label;
const mcpServerTool = mcpToolMap.get(mcpServerLabel);
if (typeof mcpServerTool === 'undefined') {
const message = `MCP server (${mcpServerLabel}) not found in Agent (${agent.name})`;
addErrorToCurrentSpan({
message,
data: { mcp_server_label: mcpServerLabel },
});
throw new ModelBehaviorError(message);
}
// Do this approval later:
// We support both onApproval callback (like the Python SDK does) and HITL patterns.
const approvalItem = new RunToolApprovalItem({
type: 'hosted_tool_call',
// We must use this name to align with the name sent from the servers
name: providerData.name,
id: providerData.id,
status: 'in_progress',
providerData,
}, agent);
runMCPApprovalRequests.push({
requestItem: approvalItem,
mcpTool: mcpServerTool,
});
if (!mcpServerTool.providerData.on_approval) {
// When onApproval function exists, it confirms the approval right after this.
// Thus, this approval item must be appended only for the next turn interruption patterns.
items.push(approvalItem);
}
}
}
else if (output.type === 'reasoning') {
items.push(new RunReasoningItem(output, agent));
}
else if (output.type === 'computer_call') {
items.push(new RunToolCallItem(output, agent));
toolsUsed.push('computer_use');
if (!computerTool) {
addErrorToCurrentSpan({
message: 'Model produced computer action without a computer tool.',
data: {
agent_name: agent.name,
},
});
throw new ModelBehaviorError('Model produced computer action without a computer tool.');
}
runComputerActions.push({
toolCall: output,
computer: computerTool,
});
}
if (output.type !== 'function_call') {
continue;
}
toolsUsed.push(output.name);
const handoff = handoffMap.get(output.name);
if (handoff) {
items.push(new RunHandoffCallItem(output, agent));
runHandoffs.push({
toolCall: output,
handoff: handoff,
});
}
else {
const functionTool = functionMap.get(output.name);
if (!functionTool) {
addErrorToCurrentSpan({
message: `Tool ${output.name} not found in agent ${agent.name}.`,
data: {
tool_name: output.name,
agent_name: agent.name,
},
});
throw new ModelBehaviorError(`Tool ${output.name} not found in agent ${agent.name}.`);
}
items.push(new RunToolCallItem(output, agent));
runFunctions.push({
toolCall: output,
tool: functionTool,
});
}
}
return {
newItems: items,
handoffs: runHandoffs,
functions: runFunctions,
computerActions: runComputerActions,
mcpApprovalRequests: runMCPApprovalRequests,
toolsUsed: toolsUsed,
hasToolsOrApprovalsToRun() {
return (runHandoffs.length > 0 ||
runFunctions.length > 0 ||
runMCPApprovalRequests.length > 0 ||
runComputerActions.length > 0);
},
};
}
export const nextStepSchema = z.discriminatedUnion('type', [
z.object({
type: z.literal('next_step_handoff'),
newAgent: z.any(),
}),
z.object({
type: z.literal('next_step_final_output'),
output: z.string(),
}),
z.object({
type: z.literal('next_step_run_again'),
}),
z.object({
type: z.literal('next_step_interruption'),
data: z.record(z.string(), z.any()),
}),
]);
/**
* Internal convenience wrapper that groups the outcome of a single agent turn. It lets the caller
* update the RunState in one shot and decide which step to execute next.
*/
class SingleStepResult {
originalInput;
modelResponse;
preStepItems;
newStepItems;
nextStep;
constructor(
/**
* The input items (i.e., the items before run() was called). May be mutated by handoff input filters.
*/
originalInput,
/**
* The model response for the current step
*/
modelResponse,
/**
* The items before the current step was executed
*/
preStepItems,
/**
* The items after the current step was executed
*/
newStepItems,
/**
* The next step to execute
*/
nextStep) {
this.originalInput = originalInput;
this.modelResponse = modelResponse;
this.preStepItems = preStepItems;
this.newStepItems = newStepItems;
this.nextStep = nextStep;
}
/**
* The items generated during the agent run (i.e. everything generated after originalInput)
*/
get generatedItems() {
return this.preStepItems.concat(this.newStepItems);
}
}
/**
* @internal
* Resets the tool choice when the agent is configured to prefer a fresh tool selection after
* any tool usage. This prevents the provider from reusing stale tool hints across turns.
*/
export function maybeResetToolChoice(agent, toolUseTracker, modelSettings) {
if (agent.resetToolChoice && toolUseTracker.hasUsedTools(agent)) {
return { ...modelSettings, toolChoice: undefined };
}
return modelSettings;
}
/**
* @internal
* Continues a turn that was previously interrupted waiting for tool approval. Executes the now
* approved tools and returns the resulting step transition.
*/
export async function resolveInterruptedTurn(agent, originalInput, originalPreStepItems, newResponse, processedResponse, runner, state) {
// call_ids for function tools
const functionCallIds = originalPreStepItems
.filter((item) => item instanceof RunToolApprovalItem &&
'callId' in item.rawItem &&
item.rawItem.type === 'function_call')
.map((item) => item.rawItem.callId);
// We already persisted the turn once when the approval interrupt was raised, so the
// counter reflects the approval items as "flushed". When we resume the same turn we need
// to rewind it so the eventual tool output for this call is still written to the session.
const pendingApprovalItems = state
.getInterruptions()
.filter(isApprovalItemLike);
if (pendingApprovalItems.length > 0) {
const pendingApprovalIdentities = new Set();
for (const approval of pendingApprovalItems) {
const identity = getApprovalIdentity(approval);
if (identity) {
pendingApprovalIdentities.add(identity);
}
}
if (pendingApprovalIdentities.size > 0) {
let rewindCount = 0;
for (let index = originalPreStepItems.length - 1; index >= 0; index--) {
const item = originalPreStepItems[index];
if (!(item instanceof RunToolApprovalItem)) {
continue;
}
const identity = getApprovalIdentity(item);
if (!identity) {
continue;
}
if (!pendingApprovalIdentities.has(identity)) {
continue;
}
rewindCount++;
pendingApprovalIdentities.delete(identity);
if (pendingApprovalIdentities.size === 0) {
break;
}
}
// Persisting the approval request already advanced the counter once, so undo the increment
// to make sure we write the final tool output back to the session when the turn resumes.
if (rewindCount > 0) {
state._currentTurnPersistedItemCount = Math.max(0, state._currentTurnPersistedItemCount - rewindCount);
}
}
}
// Run function tools that require approval after they get their approval results
const functionToolRuns = processedResponse.functions.filter((run) => {
return functionCallIds.includes(run.toolCall.callId);
});
const functionResults = await executeFunctionToolCalls(agent, functionToolRuns, runner, state);
// There is no built-in HITL approval surface for computer tools today, so every pending action
// is executed immediately when the turn resumes.
const computerResults = processedResponse.computerActions.length > 0
? await executeComputerActions(agent, processedResponse.computerActions, runner, state._context)
: [];
// When resuming we receive the original RunItem references; suppress duplicates so history and streaming do not double-emit the same items.
const originalPreStepItemSet = new Set(originalPreStepItems);
const newItems = [];
const newItemsSet = new Set();
const appendIfNew = (item) => {
if (originalPreStepItemSet.has(item) || newItemsSet.has(item)) {
return;
}
newItems.push(item);
newItemsSet.add(item);
};
for (const result of functionResults) {
appendIfNew(result.runItem);
}
for (const result of computerResults) {
appendIfNew(result);
}
// Run MCP tools that require approval after they get their approval results
const mcpApprovalRuns = processedResponse.mcpApprovalRequests.filter((run) => {
return (run.requestItem.type === 'tool_approval_item' &&
run.requestItem.rawItem.type === 'hosted_tool_call' &&
run.requestItem.rawItem.providerData?.type === 'mcp_approval_request');
});
// Hosted MCP approvals may still be waiting on a human decision when the turn resumes.
const pendingHostedMCPApprovals = new Set();
const pendingHostedMCPApprovalIds = new Set();
// Keep track of approvals we still need to surface next turn so HITL flows can resume cleanly.
for (const run of mcpApprovalRuns) {
// the approval_request_id "mcpr_123..."
const approvalRequestId = run.requestItem.rawItem.id;
const approved = state._context.isToolApproved({
// Since this item name must be the same with the one sent from Responses API server
toolName: run.requestItem.rawItem.name,
callId: approvalRequestId,
});
if (typeof approved !== 'undefined') {
const providerData = {
approve: approved,
approval_request_id: approvalRequestId,
reason: undefined,
};
// Tell Responses API server the approval result in the next turn
const responseItem = new RunToolCallItem({
type: 'hosted_tool_call',
name: 'mcp_approval_response',
providerData,
}, agent);
appendIfNew(responseItem);
}
else {
pendingHostedMCPApprovals.add(run.requestItem);
pendingHostedMCPApprovalIds.add(approvalRequestId);
functionResults.push({
type: 'hosted_mcp_tool_approval',
tool: run.mcpTool,
runItem: run.requestItem,
});
appendIfNew(run.requestItem);
}
}
// Server-managed conversations rely on preStepItems to re-surface pending approvals.
// Keep unresolved hosted MCP approvals in place so HITL flows still have something to approve next turn.
// Drop resolved approval placeholders so they are not replayed on the next turn, but keep
// pending approvals in place to signal the outstanding work to the UI and session store.
const preStepItems = originalPreStepItems.filter((item) => {
if (!(item instanceof RunToolApprovalItem)) {
return true;
}
if (item.rawItem.type === 'hosted_tool_call' &&
item.rawItem.providerData?.type === 'mcp_approval_request') {
if (pendingHostedMCPApprovals.has(item)) {
return true;
}
const approvalRequestId = item.rawItem.id;
if (approvalRequestId) {
return pendingHostedMCPApprovalIds.has(approvalRequestId);
}
return false;
}
return false;
});
const completedStep = await maybeCompleteTurnFromToolResults({
agent,
runner,
state,
functionResults,
originalInput,
newResponse,
preStepItems,
newItems,
});
if (completedStep) {
return completedStep;
}
// we only ran new tools and side effects. We need to run the rest of the agent
return new SingleStepResult(originalInput, newResponse, preStepItems, newItems, { type: 'next_step_run_again' });
}
/**
* @internal
* Executes every follow-up action the model requested (function tools, computer actions, MCP flows),
* appends their outputs to the run history, and determines the next step for the agent loop.
*/
export async function resolveTurnAfterModelResponse(agent, originalInput, originalPreStepItems, newResponse, processedResponse, runner, state) {
// Reuse the same array reference so we can compare object identity when deciding whether to
// append new items, ensuring we never double-stream existing RunItems.
const preStepItems = originalPreStepItems;
const seenItems = new Set(originalPreStepItems);
const newItems = [];
const appendIfNew = (item) => {
if (seenItems.has(item)) {
return;
}
newItems.push(item);
seenItems.add(item);
};
for (const item of processedResponse.newItems) {
appendIfNew(item);
}
// Run function tools and computer actions in parallel; neither depends on the other's side effects.
const [functionResults, computerResults] = await Promise.all([
executeFunctionToolCalls(agent, processedResponse.functions, runner, state),
executeComputerActions(agent, processedResponse.computerActions, runner, state._context),
]);
for (const result of functionResults) {
appendIfNew(result.runItem);
}
for (const item of computerResults) {
appendIfNew(item);
}
// run hosted MCP approval requests
if (processedResponse.mcpApprovalRequests.length > 0) {
for (const approvalRequest of processedResponse.mcpApprovalRequests) {
const toolData = approvalRequest.mcpTool
.providerData;
const requestData = approvalRequest.requestItem.rawItem
.providerData;
if (toolData.on_approval) {
// synchronously handle the approval process here
const approvalResult = await toolData.on_approval(state._context, approvalRequest.requestItem);
const approvalResponseData = {
approve: approvalResult.approve,
approval_request_id: requestData.id,
reason: approvalResult.reason,
};
newItems.push(new RunToolCallItem({
type: 'hosted_tool_call',
name: 'mcp_approval_response',
providerData: approvalResponseData,
}, agent));
}
else {
// receive a user's approval on the next turn
newItems.push(approvalRequest.requestItem);
const approvalItem = {
type: 'hosted_mcp_tool_approval',
tool: approvalRequest.mcpTool,
runItem: new RunToolApprovalItem({
type: 'hosted_tool_call',
name: requestData.name,
id: requestData.id,
arguments: requestData.arguments,
status: 'in_progress',
providerData: requestData,
}, agent),
};
functionResults.push(approvalItem);
// newItems.push(approvalItem.runItem);
}
}
}
// process handoffs
if (processedResponse.handoffs.length > 0) {
return await executeHandoffCalls(agent, originalInput, preStepItems, newItems, newResponse, processedResponse.handoffs, runner, state._context);
}
const completedStep = await maybeCompleteTurnFromToolResults({
agent,
runner,
state,
functionResults,
originalInput,
newResponse,
preStepItems,
newItems,
});
if (completedStep) {
return completedStep;
}
// If the model issued any tool calls or handoffs in this turn,
// we must NOT treat any assistant message in the same turn as the final output.
// We should run the loop again so the model can see the tool results and respond.
const hadToolCallsOrActions = (processedResponse.functions?.length ?? 0) > 0 ||
(processedResponse.computerActions?.length ?? 0) > 0 ||
(processedResponse.mcpApprovalRequests?.length ?? 0) > 0 ||
(processedResponse.handoffs?.length ?? 0) > 0;
if (hadToolCallsOrActions) {
return new SingleStepResult(originalInput, newResponse, preStepItems, newItems, { type: 'next_step_run_again' });
}
// No tool calls/actions in this turn; safe to consider a plain assistant message as final.
const messageItems = newItems.filter((item) => item instanceof RunMessageOutputItem);
// we will use the last content output as the final output
const potentialFinalOutput = messageItems.length > 0
? getLastTextFromOutputMessage(messageItems[messageItems.length - 1].rawItem)
: undefined;
// if there is no output we just run again
if (typeof potentialFinalOutput === 'undefined') {
return new SingleStepResult(originalInput, newResponse, preStepItems, newItems, { type: 'next_step_run_again' });
}
// Keep looping if any tool output placeholders still require an approval follow-up.
const hasPendingToolsOrApprovals = functionResults.some((result) => result.runItem instanceof RunToolApprovalItem);
if (!hasPendingToolsOrApprovals) {
if (agent.outputType === 'text') {
return new SingleStepResult(originalInput, newResponse, preStepItems, newItems, {
type: 'next_step_final_output',
output: potentialFinalOutput,
});
}
if (agent.outputType !== 'text' && potentialFinalOutput) {
// Structured output schema => always leads to a final output if we have text.
const { parser } = getSchemaAndParserFromInputType(agent.outputType, 'final_output');
const [error] = await safeExecute(() => parser(potentialFinalOutput));
if (error) {
addErrorToCurrentSpan({
message: 'Invalid output type',
data: {
error: String(error),
},
});
throw new ModelBehaviorError('Invalid output type');
}
return new SingleStepResult(originalInput, newResponse, preStepItems, newItems, { type: 'next_step_final_output', output: potentialFinalOutput });
}
}
return new SingleStepResult(originalInput, newResponse, preStepItems, newItems, { type: 'next_step_run_again' });
}
// Consolidates the logic that determines whether tool results yielded a final answer,
// triggered an interruption, or require the agent loop to continue running.
async function maybeCompleteTurnFromToolResults({ agent, runner, state, functionResults, originalInput, newResponse, preStepItems, newItems, }) {
const toolOutcome = await checkForFinalOutputFromTools(agent, functionResults, state);
if (toolOutcome.isFinalOutput) {
runner.emit('agent_end', state._context, agent, toolOutcome.finalOutput);
agent.emit('agent_end', state._context, toolOutcome.finalOutput);
return new SingleStepResult(originalInput, newResponse, preStepItems, newItems, {
type: 'next_step_final_output',
output: toolOutcome.finalOutput,
});
}
if (toolOutcome.isInterrupted) {
return new SingleStepResult(originalInput, newResponse, preStepItems, newItems, {
type: 'next_step_interruption',
data: {
interruptions: toolOutcome.interruptions,
},
});
}
return null;
}
/**
* @internal
* Normalizes tool outputs once so downstream code works with fully structured protocol items.
* Doing this here keeps API surface stable even when providers add new shapes.
*/
export function getToolCallOutputItem(toolCall, output) {
const maybeStructuredOutputs = normalizeStructuredToolOutputs(output);
if (maybeStructuredOutputs) {
const structuredItems = maybeStructuredOutputs.map(convertStructuredToolOutputToInputItem);
return {
type: 'function_call_result',
name: toolCall.name,
callId: toolCall.callId,
status: 'completed',
output: structuredItems,
};
}
return {
type: 'function_call_result',
name: toolCall.name,
callId: toolCall.callId,
status: 'completed',
output: {
type: 'text',
text: toSmartString(output),
},
};
}
function normalizeFileValue(value) {
const directFile = value.file;
if (typeof directFile === 'string' && directFile.length > 0) {
return directFile;
}
const normalizedObject = normalizeFileObjectCandidate(directFile);
if (normalizedObject) {
return normalizedObject;
}
const legacyValue = normalizeLegacyFileValue(value);
if (legacyValue) {
return legacyValue;
}
return null;
}
function normalizeFileObjectCandidate(value) {
if (!isRecord(value)) {
return null;
}
if ('data' in value && value.data !== undefined) {
const dataValue = value.data;
const hasStringData = typeof dataValue === 'string' && dataValue.length > 0;
const hasBinaryData = dataValue instanceof Uint8Array && dataValue.length > 0;
if (!hasStringData && !hasBinaryData) {
return null;
}
if (!isNonEmptyString(value.mediaType) ||
!isNonEmptyString(value.filename)) {
return null;
}
return {
data: typeof dataValue === 'string' ? dataValue : new Uint8Array(dataValue),
mediaType: value.mediaType,
filename: value.filename,
};
}
if (isNonEmptyString(value.url)) {
const result = { url: value.url };
if (isNonEmptyString(value.filename)) {
result.filename = value.filename;
}
return result;
}
const referencedId = (isNonEmptyString(value.id) && value.id) ||
(isNonEmptyString(value.fileId) && value.fileId);
if (referencedId) {
const result = { id: referencedId };
if (isNonEmptyString(value.filename)) {
result.filename = value.filename;
}
return result;
}
return null;
}
function normalizeLegacyFileValue(value) {
const filename = typeof value.filename === 'string' && value.filename.length > 0
? value.filename
: undefined;
const mediaType = typeof value.mediaType === 'string' && value.mediaType.length > 0
? value.mediaType
: undefined;
if (typeof value.fileData === 'string' && value.fileData.length > 0) {
if (!mediaType || !filename) {
return null;
}
return { data: value.fileData, mediaType, filename };
}
if (value.fileData instanceof Uint8Array && value.fileData.length > 0) {
if (!mediaType || !filename) {
return null;
}
return { data: new Uint8Array(value.fileData), mediaType, filename };
}
if (typeof value.fileUrl === 'string' && value.fileUrl.length > 0) {
const result = { url: value.fileUrl };
if (filename) {
result.filename = filename;
}
return result;
}
if (typeof value.fileId === 'string' && value.fileId.length > 0) {
const result = { id: value.fileId };
if (filename) {
result.filename = filename;
}
return result;
}
return null;
}
function isRecord(value) {
return typeof value === 'object' && value !== null;
}
function isNonEmptyString(value) {
return typeof value === 'string' && value.length > 0;
}
function toInlineImageString(data, mediaType) {
if (typeof data === 'string') {
if (mediaType && !data.startsWith('data:')) {
return asDataUrl(data, mediaType);
}
return data;
}
const base64 = encodeUint8ArrayToBase64(data);
return asDataUrl(base64, mediaType);
}
function asDataUrl(base64, mediaType) {
return mediaType ? `data:${mediaType};base64,${base64}` : base64;
}
/**
* @internal
* Runs every function tool call requested by the model and returns their outputs alongside
* the `RunItem` instances that should be appended to history.
*/
export async function executeFunctionToolCalls(agent, toolRuns, runner, state) {
async function runSingleTool(toolRun) {
let parsedArgs = toolRun.toolCall.arguments;
if (toolRun.tool.parameters) {
if (isZodObject(toolRun.tool.parameters)) {
parsedArgs = toolRun.tool.parameters.parse(parsedArgs);
}
else {
parsedArgs = JSON.parse(parsedArgs);
}
}
// Some tools require a human or policy check before execution; defer until approval is recorded.
const needsApproval = await toolRun.tool.needsApproval(state._context, parsedArgs, toolRun.toolCall.callId);
if (needsApproval) {
const approval = state._context.isToolApproved({
toolName: toolRun.tool.name,
callId: toolRun.toolCall.callId,
});
if (approval === false) {
// rejected
return withFunctionSpan(async (span) => {
const response = 'Tool execution was not approved.';
span.setError({
message: response,
data: {
tool_name: toolRun.tool.name,
error: `Tool execution for ${toolRun.toolCall.callId} was manually rejected by user.`,
},
});
span.spanData.output = response;
return {
type: 'function_output',
tool: toolRun.tool,
output: response,
runItem: new RunToolCallOutputItem(getToolCallOutputItem(toolRun.toolCall, response), agent, response),
};
}, {
data: {
name: toolRun.tool.name,
},
});
}
if (approval !== true) {
// this approval process needs to be done in the next turn
return {
type: 'function_approval',
tool: toolRun.tool,
runItem: new RunToolApprovalItem(toolRun.toolCall, agent),
};
}
}
return withFunctionSpan(async (span) => {
if (runner.config.traceIncludeSensitiveData) {
span.spanData.input = toolRun.toolCall.arguments;
}
try {
runner.emit('agent_tool_start', state._context, agent, toolRun.tool, {
toolCall: toolRun.toolCall,
});
agent.emit('agent_tool_start', state._context, toolRun.tool, {
toolCall: toolRun.toolCall,
});
const toolOutput = await toolRun.tool.invoke(state._context, toolRun.toolCall.arguments, { toolCall: toolRun.toolCall });
// Use string data for tracing and event emitter
const stringResult = toSmartString(toolOutput);
runner.emit('agent_tool_end', state._context, agent, toolRun.tool, stringResult, { toolCall: toolRun.toolCall });
agent.emit('agent_tool_end', state._context, toolRun.tool, stringResult, { toolCall: toolRun.toolCall });
if (runner.config.traceIncludeSensitiveData) {
span.spanData.output = stringResult;
}
const functionResult = {
type: 'function_output',
tool: toolRun.tool,
output: toolOutput,
runItem: new RunToolCallOutputItem(getToolCallOutputItem(toolRun.toolCall, toolOutput), agent, toolOutput),
};
const nestedRunResult = consumeAgentToolRunResult(toolRun.toolCall);
if (nestedRunResult) {
functionResult.agentRunResult = nestedRunResult;
const nestedInterruptions = nestedRunResult.interruptions;
if (nestedInterruptions.length > 0) {
functionResult.interruptions = nestedInterruptions;
}
}
return functionResult;
}
catch (error) {
span.setError({
message: 'Error running tool',
data: {
tool_name: toolRun.tool.name,
error: String(error),
},
});
throw error;
}
}, {
data: {
name: toolRun.tool.name,
},
});
}
try {
const results = await Promise.all(toolRuns.map(runSingleTool));
return results;
}
catch (e) {
throw new ToolCallError(`Failed to run function tools: ${e}`, e, state);
}
}
/**
* @internal
*/
// Internal helper: dispatch a computer action and return a screenshot (sync/async)
async function _runComputerActionAndScreenshot(computer, toolCall) {
const action = toolCall.action;
let screenshot;
// Dispatch based on action type string (assume action.type exists)
switch (action.type) {
case 'click':
await computer.click(action.x, action.y, action.button);
break;
case 'double_click':
await computer.doubleClick(action.x, action.y);
break;
case 'drag':
await computer.drag(action.path.map((p) => [p.x, p.y]));
break;
case 'keypress':
await computer.keypress(action.keys);
break;
case 'move':
await computer.move(action.x, action.y);
break;
case 'screenshot':
screenshot = await computer.screenshot();
break;
case 'scroll':
await computer.scroll(action.x, action.y, action.scroll_x, action.scroll_y);
break;
case 'type':
await computer.type(action.text);
break;
case 'wait':
await computer.wait();
break;
default:
action; // ensures that we handle every action we know of
// Unknown action, just take screenshot
break;
}
if (typeof screenshot !== 'undefined') {
return screenshot;
}
// Always return screenshot as base64 string
if (typeof computer.screenshot === 'function') {
screenshot = await computer.screenshot();
if (typeof screenshot !== 'undefined') {
return screenshot;
}
}
throw new Error('Computer does not implement screenshot()');
}
/**
* @internal
* Executes any computer-use actions emitted by the model and returns the resulting items so the
* run history reflects the computer session.
*/
export async function executeComputerActions(agent, actions, runner, runContext, customLogger = undefined) {
const _logger = customLogger ?? logger;
const results = [];
for (const action of actions) {
const computer = action.computer.computer;
const toolCall = action.toolCall;
// Hooks: on_tool_start (global + agent)
runner.emit('agent_tool_start', runContext, agent, action.computer, {
toolCall,
});
if (typeof agent.emit === 'function') {
agent.emit('agent_tool_start', runContext, action.computer, { toolCall });
}
// Run the action and get screenshot
let output;
try {
output = await _runComputerActionAndScreenshot(computer, toolCall);
}
catch (err) {
_logger.error('Failed to execute computer action:', err);
output = '';
}
// Hooks: on_tool_end (global + agent)
runner.emit('agent_tool_end', runContext, agent, action.computer, output, {
toolCall,
});
if (typeof agent.emit === 'function') {
agent.emit('agent_tool_end', runContext, action.computer, output, {
toolCall,
});
}
// Return the screenshot as a data URL when available; fall back to an empty string on failures.
const imageUrl = output ? `data:image/png;base64,${output}` : '';
const rawItem = {
type: 'computer_call_result',
callId: toolCall.callId,
output: { type: 'computer_screenshot', data: imageUrl },
};
results.push(new RunToolCallOutputItem(rawItem, agent, imageUrl));
}
return results;
}
/**
* @internal
* Drives handoff calls by invoking the downstream agent and capturing any generated items so
* the current agent can continue with the new context.
*/
export async function executeHandoffCalls(agent, originalInput, preStepItems, newStepItems, newResponse, runHandoffs, runner, runContext) {
newStepItems = [...newStepItems];
if (runHandoffs.length === 0) {
logger.warn('Incorrectly called executeHandoffCalls with no handoffs. This should not happen. Moving on.');
return new SingleStepResult(originalInput, newResponse, preStepItems, newStepItems, { type: 'next_step_run_again' });
}
if (runHandoffs.length > 1) {
// multiple handoffs. Ignoring all but the first one by adding reject responses for those
const outputMessage = 'Multiple handoffs detected, ignoring this one.';
for (let i = 1; i < runHandoffs.length; i++) {
newStepItems.push(new RunToolCallOutputItem(getToolCallOutputItem(runHandoffs[i].toolCall, outputMessage), agent, outputMessage));
}
}
const actualHandoff = runHandoffs[0];
return withHandoffSpan(async (handoffSpan) => {
const handoff = actualHandoff.handoff;
const newAgent = await handoff.onInvokeHandoff(runContext, actualHandoff.toolCall.arguments);
handoffSpan.spanData.to_agent = newAgent.name;
if (runHandoffs.length > 1) {
const requestedAgents = runHandoffs.map((h) => h.handoff.agentName);
handoffSpan.setError({
message: 'Multiple handoffs requested',
data: {
requested_agents: requestedAgents,
},
});
}
newStepItems.push(new RunHandoffOutputItem(getToolCallOutputItem(actualHandoff.toolCall, getTransferMessage(newAgent)), agent, newAgent));
runner.emit('agent_handoff', runContext, agent, newAgent);
agent.emit('agent_handoff', runContext, newAgent);
const inputFilter = handoff.inputFilter ?? runner.config.handoffInputFilter;
if (inputFilter) {
logger.debug('Filtering inputs for handoff');
if (typeof inputFilter !== 'function') {
handoffSpan.setError({
message: 'Invalid input filter',
data: {
details: 'not callable',
},
});
}
const handoffInputData = {
inputHistory: Array.isArray(originalInput)
? [...originalInput]
: originalInput,
preHandoffItems: [...preStepItems],
newItems: [...newStepItems],
runContext,
};
const filtered = inputFilter(handoffInputData);
originalInput = filtered.inputHistory;
preStepItems = filtered.preHandoffItems;
newStepItems = filtered.newItems;
}
return new SingleStepResult(originalInput, newResponse, preStepItems, newStepItems, { type: 'next_step_handoff', newAgent });
}, {
data: {
from_agent: agent.name,
},
});
}
const NOT_FINAL_OUTPUT = {
isFinalOutput: false,
isInterrupted: undefined,
};
/**
* @internal
* Determines whether tool executions produced a final agent output, triggered an interruption,
* or whether the agent loop should continue collecting more responses.
*/
export async function checkForFinalOutputFromTools(agent, toolResults, state) {
if (toolResults.length === 0) {
return NOT_FINAL_OUTPUT;
}
const interruptions = [];
for (const result of toolResults) {
if (result.runItem instanceof RunToolApprovalItem) {
interruptions.push(result.runItem);
}
if (result.type === 'function_output') {
if (Array.isArray(result.interruptions)) {
interruptions.push(...result.interruptions);
}
else if (result.agentRunResult) {
const nestedInterruptions = result.agentRunResult.interruptions;
if (nestedInterruptions.length > 0) {
interruptions.push(...nestedInterruptions);
}
}
}
}
if (interruptions.length > 0) {
return {
isFinalOutput: false,
isInterrupted: true,
interruptions,
};
}
if (agent.toolUseBehavior === 'run_llm_again') {
return NOT_FINAL_OUTPUT;
}
const firstToolResult = toolResults[0];
if (agent.toolUseBehavior === 'stop_on_first_tool') {
if (firstToolResult?.type === 'function_output') {
const stringOutput = toSmartString(firstToolResult.output);
return {
isFinalOutput: true,
isInterrupted: undefined,
finalOutput: stringOutput,
};
}
return NOT_FINAL_OUTPUT;
}
const toolUseBehavior = agent.toolUseBehavior;
if (typeof toolUseBehavior === 'object') {
const stoppingTool = toolResults.find((r) => toolUseBehavior.stopAtToolNames.includes(r.tool.name));
if (stoppingTool?.type === 'function_output') {
const stringOutput = toSmartString(stoppingTool.output);
return {
isFinalOutput: true,
isInterrupted: undefined,
finalOutput: stringOutput,
};
}
return NOT_FINAL_OUTPUT;
}
if (typeof toolUseBehavior === 'function') {
return toolUseBehavior(state._context, toolResults);
}
throw new UserError(`Invalid toolUseBehavior: ${toolUseBehavior}`, state);
}
function getRunItemStreamEventName(item) {
if (item instanceof RunMessageOutputItem) {
return 'message_output_created';
}
if (item instanceof RunHandoffCallItem) {
return 'handoff_requested';
}
if (item instanceof RunHandoffOutputItem) {
return 'handoff_occurred';
}
if (item instanceof RunToolCallItem) {
return 'tool_called';
}
if (item instanceof RunToolCallOutputItem) {
return 'tool_output';
}
if (item instanceof RunReasoningItem) {
return 'reasoning_item_created';
}
if (item instanceof RunToolApprovalItem) {
return 'tool_approval_requested';
}
return undefined;
}
function enqueueRunItemStreamEvent(result, item) {
const itemName = getRunItemStreamEventName(item);
if (!itemName) {
logger.warn('Unknown item type: ', item);
return;
}
result._addItem(new RunItemStreamEvent(itemName, item));
}
export function streamStepItemsToRunResult(result, items) {
// Preserve the order in which items were generated by enqueueing each one
// immediately on the streamed result.
for (const item of items) {
enqueueRunItemStreamEvent(result, item);
}
}
export function addStepToRunResult(result, step, options) {
// skipItems contains run items that were already streamed so we avoid
// enqueueing duplicate events for the same instance.
const skippedItems = options?.skipItems;
for (const item of step.newStepItems) {
if (skippedItems?.has(item)) {
continue;
}
enqueueRunItemStreamEvent(result, item);
}
}
export class AgentToolUseTracker {
#agentToTools = new Map();
addToolUse(agent, toolNames) {
this.#agentToTools.set(agent, toolNames);
}
hasUsedTools(agent) {
return this.#agentToTools.has(agent);
}
toJSON() {
return Object.fromEntries(Array.from(this.#agentToTools.entries()).map(([agent, toolNames]) => {
return [agent.name, toolNames];
}));
}
}
/**
* @internal
* Convert a user-provided input into a list of input items.
*/
export function toInputItemList(input) {
if (typeof input === 'string') {
return [
{
type: 'message',
role: 'user',
content: input,
},
];
}
return [...input];
}
/**
* @internal
* Extract model output items from run items, excluding tool approval items.
*/
export function extractOutputItemsFromRunItems(items) {
return items
.filter((item) => item.type !== 'tool_approval_item')
.map((item) => item.rawItem);
}
function normalizeItemsForSessionPersistence(items) {
// Persisted sessions must avoid raw binary so we convert every item into a JSON-safe shape before writing to storage.
return items.map((item) => sanitizeValueForSession(stripTransientCallIds(item)));
}
function sanitizeValueForSession(value, context = {}) {
if (value === null || value === undefined) {
return value;
}
// Convert supported binary payloads into ArrayBuffer views before serialization.
const binary = toUint8ArrayIfBinary(value);
if (binary) {
return toDataUrlFromBytes(binary, context.mediaType);
}
if (Array.isArray(value)) {
return value.map((entry) => sanitizeValueForSession(entry, context));
}
if (!isPlainObject(value)) {
return value;
}
const record = value;
const result = {};
const mediaType = typeof record.mediaType === 'string' && record.mediaType.length > 0
? record.mediaType
: context.mediaType;
for (const [key, entry] of Object.entries(record)) {
// Propagate explicit media type only when walking into binary payload containers.
const nextContext = key === 'data' || key === 'fileData' ? { mediaType } : context;
result[key] = sanitizeValueForSession(entry, nextContext);
}
return result;
}
function toUint8ArrayIfBinary(value) {
// Normalize the diverse binary containers we may receive into a shared Uint8Array view.
if (value instanceof ArrayBuffer) {
return new Uint8Array(value);
}
if (isArrayBufferView(value)) {
const view = value;
return new Uint8Array(view.buffer, view.byteOffset, view.byteLength);
}
if (isNodeBuffer(value)) {
const view = value;
return new Uint8Array(view.buffer, view.byteOffset, view.byteLength);
}
if (isSerializedBufferSnapshot(value)) {
const snapshot = value;
return Uint8Array.from(snapshot.data);
}
return undefined;
}
function toDataUrlFromBytes(bytes, mediaType) {
// Convert binary payloads into a durable data URL so session files remain self-contained.
const base64 = encodeUint8ArrayToBase64(bytes);
// Note that OpenAI Responses API never accepts application/octet-stream as a media type,
// so we fall back to text/plain; that said, tools are supposed to return a valid media type when this utility is used.
const type = mediaType && !mediaType.startsWith('data:') ? mediaType : 'text/plain';
return `data:${type};base64,${base64}`;
}
function isPlainObject(value) {
if (typeof value !== 'object' || value === null) {
return false;
}
const proto = Object.getPrototypeOf(value);
return proto === Object.prototype || proto === null;
}
function stripTransientCallIds(value) {
if (value === null || value === undefined) {
return value;
}
if (Array.isArray(value)) {
return value.map((entry) => stripTransientCallIds(entry));
}
if (!isPlainObject(value)) {
return value;
}
const record = value;
const result = {};
const isProtocolItem = typeof record.type === 'string