@mastra/core
Version:
1,435 lines (1,430 loc) • 236 kB
JavaScript
import { CachingPubSub } from '../../chunk-BOHPE4XX.js';
import { SaveQueueManager, createStep, mergeProviderOptions, PrepareStepProcessor, composeStepInput, buildLlmPromptArgs, applyAutoResumeSystemMessage, injectBackgroundTaskPrompt, execute, mergeLlmCallHeaders, buildMemoryHeaders, findProviderToolByName, createWorkflow, pruneAgentLoopSnapshot, runScorer, Agent, inferProviderExecuted, runStreamCompletionScorers, formatStreamCompletionFeedback, resolveGoalStore, readObjective, resolveEffectiveGoalSettings, createGoalScorer, GOAL_SCORER_ID, GOAL_SCORE_WAITING, writeObjective } from '../../chunk-OE4IEL7C.js';
import { PUBSUB_SYMBOL } from '../../chunk-2QXNHEDL.js';
import { TripWire, MastraModelOutput, isSupportedLanguageModel, ProcessorRunner, safeClose, safeEnqueue, createProcessorSendSignal } from '../../chunk-ZWZWJC3R.js';
import { resolveModelConfig } from '../../chunk-AR6WPTSV.js';
import { EventEmitterPubSub } from '../../chunk-2S76KRB5.js';
import { InMemoryServerCache } from '../../chunk-366PLPDH.js';
import { deepMerge } from '../../chunk-R5KSSZYP.js';
import { getNeedsApprovalFn } from '../../chunk-O5QBSZXE.js';
import { toStandardSchema } from '../../chunk-6SRTDZ7S.js';
import { createObservabilityContext } from '../../chunk-32GHUCZC.js';
import { getOrCreateSpan, EntityType, getStepAvailableToolNames } from '../../chunk-DEQCJWZZ.js';
import { resolveBackgroundConfig, createBackgroundTask } from '../../chunk-O5MLHBJQ.js';
import { MessageList, normalizeToolPayloadTransformPolicy, mastraDBMessageToSignal, transformToolPayloadForTargets, withToolPayloadTransformMetadata } from '../../chunk-WEN4YCAX.js';
import { RequestContext, MASTRA_VERSIONS_KEY, mergeVersionOverrides, MASTRA_THREAD_ID_KEY, MASTRA_RESOURCE_ID_KEY } from '../../chunk-W4DLMY73.js';
import { TTLCache } from '@isaacs/ttlcache';
import { ReadableStream as ReadableStream$1 } from 'stream/web';
import { z } from 'zod';
// src/agent/durable/constants.ts
var AGENT_STREAM_TOPIC = (runId) => `agent.stream.${runId}`;
var AgentStreamEventTypes = {
/** Chunk of streaming data (text, tool call, etc.) */
CHUNK: "chunk",
/** Start of a new step in the agentic loop */
STEP_START: "step-start",
/** End of a step in the agentic loop */
STEP_FINISH: "step-finish",
/** Agent execution completed successfully */
FINISH: "finish",
/** Error occurred during execution */
ERROR: "error",
/** Workflow suspended (e.g., for tool approval) */
SUSPENDED: "suspended",
/** Execution aborted by abortSignal */
ABORT: "abort",
/** Single agentic-loop iteration completed (observability hook) */
ITERATION_COMPLETE: "iteration-complete"
};
var DurableAgentDefaults = {
/** Default maximum number of agentic loop iterations */
MAX_STEPS: 5,
/**
* Default tool call concurrency.
* NOTE: Currently unused — durable workflows run tool calls sequentially
* (concurrency: 1) because tool approval and suspension require sequential
* execution. The serialized toolCallConcurrency option is preserved in
* workflow input for future use when dynamic foreach concurrency is supported.
*/
TOOL_CALL_CONCURRENCY: 10
};
var DurableStepIds = {
/** LLM execution step */
LLM_EXECUTION: "durable-llm-execution",
/** Tool call step */
TOOL_CALL: "durable-tool-call",
/** LLM mapping step (combines results) */
LLM_MAPPING: "durable-llm-mapping",
/** Agentic execution workflow (one iteration) */
AGENTIC_EXECUTION: "durable-agentic-execution",
/** Full agentic loop workflow */
AGENTIC_LOOP: "durable-agentic-loop",
/** Scorer execution step */
SCORER_EXECUTION: "durable-scorer-execution",
/** isTaskComplete evaluation step */
IS_TASK_COMPLETE: "durable-is-task-complete"
};
// src/agent/durable/durable-stream-until-idle.ts
var TERMINAL_BG_CHUNKS = /* @__PURE__ */ new Set([
"background-task-completed",
"background-task-failed",
"background-task-cancelled",
// Suspended is non-terminal for the bg task itself (it can be resumed
// later via `manager.resume`), but it IS terminal-for-this-iteration of
// the streamUntilIdle wrapper: the agent should react to the suspend in
// a follow-up turn so the user is told the task is parked. Without
// this, the wrapper waits indefinitely for completed/failed/cancelled
// and the stream times out.
"background-task-suspended"
]);
async function resolveScope(agent, mergedOptions) {
const requestContext = mergedOptions?.requestContext ?? new RequestContext();
const memory = await agent.getMemory();
if (!memory) return null;
const threadIdFromContext = requestContext.get(MASTRA_THREAD_ID_KEY);
const resourceIdFromContext = requestContext.get(MASTRA_RESOURCE_ID_KEY);
const threadIdFromArgs = typeof mergedOptions?.memory?.thread === "string" ? mergedOptions.memory.thread : mergedOptions?.memory?.thread?.id;
const threadId = threadIdFromContext ?? threadIdFromArgs;
const resourceId = resourceIdFromContext ?? mergedOptions?.memory?.resource;
const scopeKey = threadId || resourceId ? `${threadId ?? ""}|${resourceId ?? ""}` : null;
return { threadId, resourceId, scopeKey };
}
function buildContinuationDirective(batch) {
const entries = batch.map((chunk) => {
const payload = chunk.payload ?? {};
return {
toolCallId: payload.toolCallId,
toolName: payload.toolName,
isSuspended: !!payload.suspendedAt
};
}).filter((e) => !!e.toolCallId);
const idList = entries.filter((e) => !e.isSuspended).map((e) => e.toolName ? `${e.toolCallId} (${e.toolName})` : e.toolCallId).join(", ");
const suspendedIdList = entries.filter((e) => e.isSuspended).map((e) => `${e.toolCallId} (${e.toolName})`).join(", ");
return `Background task(s) you previously dispatched have completed. Process ONLY these tool-call IDs (their results are now in the conversation): ${idList}. IMPORTANT: Do NOT process any tool-call IDs that were not in the list, and do NOT call the same tool again \u2014 the result is already available. Use these result(s) to answer the user's original question.IMPORTANT: The following tool-call IDs are suspended: ${suspendedIdList}. Do not attempt to resume them; let the user know they are waiting for explicit resume input.`;
}
function buildContinuationOpts(baseContinuationOpts, callerContext, batch) {
const directive = buildContinuationDirective(batch);
return {
...baseContinuationOpts,
context: [...callerContext ?? [], { role: "user", content: directive }]
};
}
function acquireStreamSlot(activeStreams, scopeKey, closer) {
if (!scopeKey) return;
const priorClose = activeStreams.get(scopeKey);
priorClose?.();
activeStreams.set(scopeKey, closer);
}
function releaseStreamSlot(activeStreams, scopeKey, closer) {
if (!scopeKey) return;
if (activeStreams.get(scopeKey) === closer) {
activeStreams.delete(scopeKey);
}
}
async function runWithIdleWrapper(agent, streamOptions, deps, firstTurn) {
const { maxIdleMs: _maxIdleMs, ...restStreamOptions } = streamOptions ?? {};
const defaultOptions = await agent.getDefaultOptions({
requestContext: streamOptions?.requestContext
});
const mergedOptions = deepMerge(
defaultOptions,
restStreamOptions ?? {}
);
const scope = await resolveScope(agent, mergedOptions);
if (!deps.bgManager || !scope) {
return firstTurn(restStreamOptions);
}
const { threadId, resourceId, scopeKey } = scope;
const maxIdleMs = _maxIdleMs ?? 5 * 6e4;
const baseContinuationOpts = {
...restStreamOptions ?? {},
onFinish: void 0,
_skipBgTaskWait: true
};
const initialStreamOpts = {
...restStreamOptions ?? {},
_skipBgTaskWait: true
};
const runningTaskIds = /* @__PURE__ */ new Set();
const pendingCompletions = [];
const processedTerminalKeys = /* @__PURE__ */ new Set();
const innerCleanups = [];
const innerAborts = [];
let isProcessing = false;
let closed = false;
let idleTimer;
let outerController;
const outerAbort = new AbortController();
let firstRunId;
const forceClose = () => {
if (closed) return;
closed = true;
if (idleTimer) {
clearTimeout(idleTimer);
idleTimer = void 0;
}
outerAbort.abort();
try {
outerController.close();
} catch {
}
for (const fn of innerCleanups) {
try {
fn();
} catch {
}
}
releaseStreamSlot(deps.activeStreams, scopeKey, forceClose);
};
const tryClose = () => {
if (closed) return;
if (isProcessing) return;
if (runningTaskIds.size > 0) return;
if (pendingCompletions.length > 0) return;
forceClose();
};
const clearIdleTimer = () => {
if (idleTimer) {
clearTimeout(idleTimer);
idleTimer = void 0;
}
};
const updateIdleTimer = () => {
if (closed) return;
clearIdleTimer();
if (isProcessing) return;
if (runningTaskIds.size === 0) return;
if (pendingCompletions.length > 0) return;
idleTimer = setTimeout(forceClose, maxIdleMs);
};
const pipeInner = async (inner) => {
const reader = inner.getReader();
try {
while (true) {
if (outerAbort.signal.aborted) break;
const { done, value } = await reader.read();
if (done) break;
clearIdleTimer();
try {
outerController.enqueue(value);
} catch {
break;
}
if (value && typeof value === "object" && value.type === "background-task-started") {
const taskId = value.payload?.taskId;
if (taskId) runningTaskIds.add(taskId);
}
}
} finally {
reader.releaseLock();
}
};
const processIfIdle = async () => {
if (isProcessing || closed || pendingCompletions.length === 0) return;
isProcessing = true;
try {
const batch = pendingCompletions.splice(0, pendingCompletions.length);
for (const chunk of batch) {
const tid = chunk.payload?.taskId;
const ctype = chunk.type;
if (tid && ctype) processedTerminalKeys.add(`${tid}:${ctype}`);
}
const continuationOpts = buildContinuationOpts(baseContinuationOpts, restStreamOptions?.context, batch);
const inner = await agent.stream([], continuationOpts);
innerCleanups.push(inner.cleanup);
if (typeof inner.abort === "function") innerAborts.push(inner.abort);
await pipeInner(inner.fullStream);
} catch (err) {
try {
outerController.error(err);
} catch {
}
forceClose();
return;
} finally {
isProcessing = false;
if (pendingCompletions.length > 0) {
void processIfIdle();
} else {
tryClose();
updateIdleTimer();
}
}
};
acquireStreamSlot(deps.activeStreams, scopeKey, forceClose);
streamOptions?.abortSignal?.addEventListener("abort", forceClose);
const combinedStream = new ReadableStream({
start(controller) {
outerController = controller;
},
cancel() {
closed = true;
outerAbort.abort();
clearIdleTimer();
}
});
const bgStream = deps.bgManager.stream({
agentId: agent.id,
threadId,
resourceId,
abortSignal: outerAbort.signal
});
const bgReader = bgStream.getReader();
void (async () => {
try {
while (true) {
if (outerAbort.signal.aborted) break;
const { done, value } = await bgReader.read();
if (done) break;
const chunk = value;
if (!chunk || typeof chunk !== "object" || typeof chunk.type !== "string") continue;
const taskId = chunk.payload?.taskId;
const terminalKey = taskId && TERMINAL_BG_CHUNKS.has(chunk.type) ? `${taskId}:${chunk.type}` : void 0;
if (terminalKey && processedTerminalKeys.has(terminalKey)) {
continue;
}
updateIdleTimer();
try {
outerController.enqueue(chunk);
} catch {
break;
}
if (!taskId) continue;
if (chunk.type === "background-task-running") {
runningTaskIds.add(taskId);
} else if (TERMINAL_BG_CHUNKS.has(chunk.type)) {
runningTaskIds.delete(taskId);
pendingCompletions.push(chunk);
void processIfIdle();
}
}
} catch {
} finally {
bgReader.releaseLock();
}
})();
isProcessing = true;
clearIdleTimer();
let first;
try {
first = await firstTurn(initialStreamOpts);
} catch (err) {
forceClose();
throw err;
}
firstRunId = first.runId;
innerCleanups.push(first.cleanup);
if (typeof first.abort === "function") innerAborts.push(first.abort);
void (async () => {
try {
await pipeInner(first.fullStream);
} catch (err) {
try {
outerController.error(err);
} catch {
}
}
isProcessing = false;
if (pendingCompletions.length > 0) {
void processIfIdle();
} else {
tryClose();
updateIdleTimer();
}
})();
return {
output: new Proxy(first.output, {
get(target, prop) {
if (prop === "fullStream") return combinedStream;
const value = Reflect.get(target, prop, target);
return typeof value === "function" ? value.bind(target) : value;
}
}),
get fullStream() {
return combinedStream;
},
runId: firstRunId,
threadId,
resourceId,
cleanup: forceClose,
abort: (reason) => {
for (const innerAbort of innerAborts) {
try {
innerAbort(reason);
} catch {
}
}
forceClose();
}
};
}
async function runDurableStreamUntilIdle(agent, messages, streamOptions, deps) {
return runWithIdleWrapper(
agent,
streamOptions,
deps,
(opts) => agent.stream(messages, opts)
);
}
async function runResumeDurableStreamUntilIdle(agent, runId, resumeData, streamOptions, deps) {
return runWithIdleWrapper(
agent,
streamOptions,
deps,
(opts) => agent.resume(runId, resumeData, opts)
);
}
// src/agent/durable/utils/serialize-state.ts
function serializeToolMetadata(name, tool) {
let inputSchema = { type: "object" };
if (tool.parameters) {
if ("type" in tool.parameters && typeof tool.parameters.type === "string") {
inputSchema = tool.parameters;
} else if ("jsonSchema" in tool.parameters) {
inputSchema = tool.parameters.jsonSchema;
} else if ("_def" in tool.parameters) {
inputSchema = { type: "object" };
}
}
return {
id: "id" in tool && typeof tool.id === "string" ? tool.id : name,
name,
description: tool.description,
inputSchema,
requireApproval: tool.requireApproval,
hasSuspendSchema: tool.hasSuspendSchema
};
}
function serializeToolsMetadata(tools) {
return Object.entries(tools).map(([name, tool]) => serializeToolMetadata(name, tool));
}
function serializeModelConfig(model) {
return {
provider: model.provider,
modelId: model.modelId,
specificationVersion: model.specificationVersion,
// Store the original config string for runtime resolution (e.g., 'openai/gpt-4o')
originalConfig: `${model.provider}/${model.modelId}`
// Note: We don't serialize model settings here - they come from execution options
};
}
function serializeModelListEntry(entry) {
const model = entry.model;
return {
id: entry.id,
config: {
provider: model.provider,
modelId: model.modelId,
specificationVersion: model.specificationVersion,
originalConfig: `${model.provider}/${model.modelId}`,
providerOptions: entry.providerOptions
},
maxRetries: entry.maxRetries,
enabled: entry.enabled
};
}
function serializeModelList(models) {
return models.filter((m) => m.enabled !== false).map(serializeModelListEntry);
}
function serializeScorersConfig(scorers) {
const result = {};
for (const [key, entry] of Object.entries(scorers)) {
const scorerName = typeof entry.scorer === "string" ? entry.scorer : entry.scorer.name;
const scorerEntry = {
scorerName
};
if (entry.sampling) {
scorerEntry.sampling = entry.sampling;
}
result[key] = scorerEntry;
}
return result;
}
function serializeDurableState(params) {
return {
memoryConfig: params.memoryConfig,
threadId: params.threadId,
resourceId: params.resourceId,
threadExists: params.threadExists,
savePerStep: params.savePerStep,
observationalMemory: params.observationalMemory
};
}
function serializeModelSettings(settings) {
if (!settings || typeof settings !== "object") return void 0;
const source = settings;
const out = {};
const pickNumber = (key) => {
const value = source[key];
if (typeof value === "number" && Number.isFinite(value)) {
out[key] = value;
}
};
pickNumber("maxOutputTokens");
pickNumber("temperature");
pickNumber("topP");
pickNumber("topK");
pickNumber("presencePenalty");
pickNumber("frequencyPenalty");
pickNumber("seed");
pickNumber("maxRetries");
if (Array.isArray(source.stopSequences) && source.stopSequences.every((v) => typeof v === "string")) {
out.stopSequences = source.stopSequences;
}
return Object.keys(out).length > 0 ? out : void 0;
}
function serializeDurableOptions(options) {
let serializedToolChoice;
if (options.toolChoice) {
if (typeof options.toolChoice === "string") {
serializedToolChoice = options.toolChoice;
} else if (typeof options.toolChoice === "object" && "type" in options.toolChoice) {
if (options.toolChoice.type === "tool" && "toolName" in options.toolChoice) {
serializedToolChoice = {
type: "tool",
toolName: options.toolChoice.toolName
};
}
}
}
return {
maxSteps: options.maxSteps,
toolChoice: serializedToolChoice,
activeTools: options.activeTools,
modelSettings: serializeModelSettings(options.modelSettings),
requireToolApproval: options.requireToolApproval,
toolCallConcurrency: options.toolCallConcurrency,
autoResumeSuspendedTools: options.autoResumeSuspendedTools,
maxProcessorRetries: options.maxProcessorRetries,
includeRawChunks: options.includeRawChunks,
returnScorerData: options.returnScorerData,
hasErrorProcessors: options.hasErrorProcessors,
providerOptions: options.providerOptions,
structuredOutput: options.structuredOutput,
skipBgTaskWait: options.skipBgTaskWait,
disableBackgroundTasks: options.disableBackgroundTasks,
tracingOptions: options.tracingOptions,
actor: options.actor,
instructionsOverride: options.instructionsOverride,
systemMessage: options.systemMessage,
transform: options.transform,
isTaskComplete: options.isTaskComplete
};
}
function createWorkflowInput(params) {
return {
__workflowKind: "durable-agent",
runId: params.runId,
agentId: params.agentId,
agentName: params.agentName,
messageListState: params.messageList.serialize(),
toolsMetadata: serializeToolsMetadata(params.tools),
modelConfig: serializeModelConfig(params.model),
modelList: params.modelList ? serializeModelList(params.modelList) : void 0,
scorers: params.scorers ? serializeScorersConfig(params.scorers) : void 0,
options: serializeDurableOptions(params.options),
state: serializeDurableState(params.state),
messageId: params.messageId,
agentSpanData: params.agentSpanData,
modelSpanData: params.modelSpanData,
requestContextEntries: params.requestContextEntries
};
}
function serializeError(error) {
if (error instanceof Error) {
return {
name: error.name,
message: error.message,
stack: error.stack
};
}
return {
name: "Error",
message: String(error)
};
}
// src/agent/durable/preparation.ts
function snapshotRequestContextEntries(requestContext) {
if (!requestContext) return void 0;
const out = {};
let any = false;
for (const [key, value] of requestContext.entries()) {
try {
const cloned = JSON.parse(JSON.stringify(value));
out[key] = cloned;
any = true;
} catch {
}
}
return any ? out : void 0;
}
function convertInstructionsToString(instructions) {
if (!instructions) return "";
if (typeof instructions === "string") return instructions;
if (Array.isArray(instructions)) {
return instructions.map((msg) => typeof msg === "string" ? msg : typeof msg.content === "string" ? msg.content : "").filter(Boolean).join("\n\n");
}
return typeof instructions.content === "string" ? instructions.content : "";
}
function getInitialSignalEchoes(messageList) {
const inputMessageIds = messageList.makeMessageSourceChecker().input;
return messageList.get.all.db().filter((message) => message.role === "signal" && inputMessageIds.has(message.id)).map(mastraDBMessageToSignal);
}
async function prepareForDurableExecution(options) {
const {
agent,
messages,
options: rawExecOptions,
runId: providedRunId,
requestContext: providedRequestContext,
logger,
mastra,
methodType = "stream"
} = options;
const typedAgent = agent;
const runId = providedRunId ?? crypto.randomUUID();
const messageId = crypto.randomUUID();
const requestContext = providedRequestContext ?? new RequestContext();
const requestContextEntriesSnapshot = snapshotRequestContextEntries(requestContext);
const defaultOptions = await typedAgent.getDefaultOptions({ requestContext });
const execOptions = deepMerge(
defaultOptions ?? {},
rawExecOptions ?? {}
);
const requestVersions = requestContext.get(MASTRA_VERSIONS_KEY);
let mergedVersions = mergeVersionOverrides(mastra?.getVersionOverrides?.(), requestVersions);
if (execOptions?.versions) {
mergedVersions = mergeVersionOverrides(mergedVersions, execOptions.versions);
}
if (mergedVersions) {
requestContext.set(MASTRA_VERSIONS_KEY, mergedVersions);
}
const thread = typeof execOptions?.memory?.thread === "string" ? { id: execOptions.memory.thread } : execOptions?.memory?.thread;
const threadId = thread?.id;
const resourceId = execOptions?.memory?.resource;
let threadObject;
let threadExists = false;
const messageList = new MessageList({
threadId,
resourceId
});
const instructions = execOptions?.instructions || await typedAgent.getInstructions({ requestContext });
if (instructions) {
if (typeof instructions === "string") {
messageList.addSystem(instructions);
} else if (Array.isArray(instructions)) {
for (const inst of instructions) {
messageList.addSystem(inst);
}
} else {
messageList.addSystem(instructions);
}
}
const workspace = await typedAgent.getWorkspace({ requestContext });
if (workspace) {
const hasFs = typeof workspace.hasFilesystemConfig === "function" ? workspace.hasFilesystemConfig() : !!workspace.filesystem;
const hasSb = typeof workspace.hasSandboxConfig === "function" ? workspace.hasSandboxConfig() : !!workspace.sandbox;
if (hasFs || hasSb) {
const wsInstructions = typeof workspace.getInstructionsAsync === "function" ? await workspace.getInstructionsAsync({ requestContext }) : workspace.getInstructions({ requestContext });
if (wsInstructions) {
messageList.addSystem({ role: "system", content: wsInstructions });
}
}
}
if (execOptions?.context) {
messageList.add(execOptions.context, "context");
}
if (execOptions?.system) {
const sys = execOptions.system;
if (typeof sys === "string") {
messageList.addSystem(sys);
} else if (Array.isArray(sys)) {
for (const s of sys) {
messageList.addSystem(s);
}
} else {
messageList.addSystem(sys);
}
}
messageList.add(messages, "input");
const memory = await typedAgent.getMemory({ requestContext });
const memoryConfig = execOptions?.memory?.options;
if (memory && threadId && resourceId) {
const existingThread = await memory.getThreadById({ threadId });
threadObject = existingThread ?? await memory.createThread({
threadId,
metadata: thread?.metadata,
title: thread?.title,
memoryConfig,
resourceId,
saveThread: true
});
threadExists = true;
requestContext.set("MastraMemory", { thread: threadObject, resourceId, memoryConfig });
} else {
requestContext.delete("MastraMemory");
}
const processorStates = /* @__PURE__ */ new Map();
let inputProcessors = [];
let llmRequestInputProcessors = [];
let outputProcessors = [];
let errorProcessors = [];
try {
inputProcessors = await typedAgent.listInputProcessors(requestContext);
llmRequestInputProcessors = await typedAgent.__listLLMRequestProcessors(requestContext);
outputProcessors = execOptions?.outputProcessors ? execOptions.outputProcessors : await typedAgent.listOutputProcessors(requestContext);
errorProcessors = await typedAgent.listErrorProcessors(requestContext);
} catch (error) {
logger?.warn?.(`[DurableAgent] Error resolving processors: ${error}`);
}
const rawConfig = typeof agent.toRawConfig === "function" ? agent.toRawConfig() : void 0;
const resolvedVersionId = rawConfig?.resolvedVersionId;
const agentTracingPolicy = typeof agent.getTracingPolicy === "function" ? agent.getTracingPolicy() : void 0;
const agentSpan = getOrCreateSpan({
type: "agent_run" /* AGENT_RUN */,
name: `agent run: '${agent.id}'`,
entityType: EntityType.AGENT,
entityId: agent.id,
entityName: agent.name,
input: messages,
attributes: {
conversationId: threadId,
instructions: convertInstructionsToString(instructions),
// @deprecated — use entityVersionId (top-level span context field) instead.
// Kept for backward compatibility during migration.
...resolvedVersionId ? { resolvedVersionId } : {}
},
metadata: {
runId,
resourceId,
threadId,
...resolvedVersionId ? { entityVersionId: resolvedVersionId } : {}
},
tracingPolicy: agentTracingPolicy,
tracingContext: execOptions?.tracingContext,
tracingOptions: execOptions?.tracingOptions,
requestContext,
mastra
});
let tripwireData;
if (inputProcessors.length > 0) {
try {
const { ProcessorRunner: ProcessorRunner2 } = await import('../../runner-EU664FG6.js');
const runner = new ProcessorRunner2({
inputProcessors,
outputProcessors,
errorProcessors,
logger,
agentName: agent.name,
processorStates
});
await runner.runInputProcessors(
messageList,
createObservabilityContext({ currentSpan: agentSpan }),
requestContext,
0
);
} catch (error) {
if (error instanceof TripWire) {
tripwireData = {
reason: error.message,
retry: error.options?.retry,
metadata: error.options?.metadata,
processorId: error.processorId
};
logger?.warn?.("Input processor tripwire triggered", {
agent: agent.name,
reason: error.message,
processorId: error.processorId,
retry: error.options?.retry
});
} else {
logger?.warn?.(`[DurableAgent] Error running input processors: ${error}`);
}
}
}
let tools = {};
try {
tools = await typedAgent.getToolsForExecution({
toolsets: execOptions?.toolsets,
clientTools: execOptions?.clientTools,
threadId,
resourceId,
runId,
requestContext,
memoryConfig: execOptions?.memory?.options,
autoResumeSuspendedTools: execOptions?.autoResumeSuspendedTools,
hooks: execOptions?.hooks,
delegation: execOptions?.delegation,
methodType
});
} catch (error) {
logger?.warn?.(`[DurableAgent] Error converting tools: ${error}`);
}
const model = await typedAgent.getModel({ requestContext });
if (!model) {
throw new Error("Agent model not available");
}
const modelList = await typedAgent.getModelList(requestContext);
const overrideScorers = execOptions?.scorers;
let scorers;
if (overrideScorers) {
scorers = overrideScorers;
} else {
try {
const agentScorers = await typedAgent.listScorers({ requestContext });
if (agentScorers && Object.keys(agentScorers).length > 0) {
scorers = agentScorers;
}
} catch (error) {
logger?.debug?.(`[DurableAgent] Error getting scorers: ${error}`);
}
}
const saveQueueManager = memory ? new SaveQueueManager({
logger,
memory
}) : void 0;
let serializedStructuredOutput;
if (execOptions?.structuredOutput) {
const so = execOptions.structuredOutput;
if (so.schema) {
serializedStructuredOutput = {
jsonPromptInjection: so.jsonPromptInjection,
useAgent: so.useAgent
};
if (typeof so.schema === "object" && "type" in so.schema) {
serializedStructuredOutput.schema = so.schema;
} else if (typeof so.schema === "object" && "jsonSchema" in so.schema) {
serializedStructuredOutput.schema = so.schema.jsonSchema;
}
}
}
const backgroundTasksConfig = typedAgent.getBackgroundTasksConfig?.();
const backgroundTaskManager = execOptions?.disableBackgroundTasks ? void 0 : mastra?.backgroundTaskManager;
const toolPayloadTransform = normalizeToolPayloadTransformPolicy(execOptions?.transform) ?? typedAgent.getToolPayloadTransform?.() ?? normalizeToolPayloadTransformPolicy(
mastra?.getToolPayloadTransform?.() ?? mastra?.getToolPayloadProjection?.()
);
const savePerStep = execOptions?.savePerStep;
const observationalMemory = !!memoryConfig?.observationalMemory;
const modelSpan = agentSpan?.createChildSpan({
type: "model_generation" /* MODEL_GENERATION */,
name: `llm: '${model.modelId}'`,
attributes: {
model: model.modelId,
provider: model.provider,
streaming: true
},
metadata: {
runId,
threadId,
resourceId
},
requestContext
});
const workflowInput = createWorkflowInput({
runId,
agentId: agent.id,
agentName: agent.name,
messageList,
tools,
model,
modelList: modelList ?? void 0,
scorers,
options: {
maxSteps: execOptions?.maxSteps,
toolChoice: execOptions?.toolChoice,
activeTools: execOptions?.activeTools,
modelSettings: execOptions?.modelSettings,
// Function-form approval policies are closures that can't ride on the
// serialized workflow input — the live closure is parked on the run
// registry below. This boolean shadow is the cross-process fallback:
// function policies degrade to "require approval for every tool call"
// when the registry slot is unavailable (e.g. Inngest after a worker
// restart), which is the safe default.
requireToolApproval: typeof execOptions?.requireToolApproval === "function" ? true : execOptions?.requireToolApproval,
toolCallConcurrency: execOptions?.toolCallConcurrency,
autoResumeSuspendedTools: execOptions?.autoResumeSuspendedTools,
maxProcessorRetries: execOptions?.maxProcessorRetries,
includeRawChunks: execOptions?.includeRawChunks,
returnScorerData: execOptions?.returnScorerData,
hasErrorProcessors: errorProcessors.length > 0,
providerOptions: execOptions?.providerOptions,
structuredOutput: serializedStructuredOutput,
skipBgTaskWait: execOptions?._skipBgTaskWait,
disableBackgroundTasks: execOptions?.disableBackgroundTasks,
tracingOptions: execOptions?.tracingOptions,
actor: execOptions?.actor,
instructionsOverride: execOptions?.instructions,
systemMessage: execOptions?.system,
transform: toolPayloadTransform?.targets ? { targets: toolPayloadTransform.targets } : void 0,
isTaskComplete: execOptions?.isTaskComplete ? {
scorerNames: execOptions.isTaskComplete.scorers?.map((s) => s.name).filter((n) => !!n),
strategy: execOptions.isTaskComplete.strategy,
timeout: execOptions.isTaskComplete.timeout,
parallel: execOptions.isTaskComplete.parallel,
suppressFeedback: execOptions.isTaskComplete.suppressFeedback
} : void 0
},
state: {
memoryConfig,
threadId,
resourceId,
threadExists,
savePerStep,
observationalMemory
},
messageId,
agentSpanData: agentSpan?.exportSpan(),
modelSpanData: modelSpan?.exportSpan(),
requestContextEntries: requestContextEntriesSnapshot
});
const registryEntry = {
tools,
saveQueueManager,
memory,
model,
modelList: modelList ? modelList.map((entry) => ({
id: entry.id,
model: entry.model,
maxRetries: entry.maxRetries ?? 0,
enabled: entry.enabled ?? true,
headers: entry.headers
})) : void 0,
workspace,
requestContext,
inputProcessors,
llmRequestInputProcessors,
outputProcessors,
errorProcessors,
processorStates,
backgroundTaskManager,
backgroundTasksConfig,
agentSpan,
modelSpan,
// Park the stopWhen predicate(s) on the registry so the durable agentic
// loop can evaluate them on each iteration. The predicate is a closure and
// cannot ride on the serialized workflow input; in-process engines read it
// back via globalRunRegistry, cross-process engines degrade to maxSteps.
stopWhen: execOptions?.stopWhen,
onIterationComplete: execOptions?.onIterationComplete,
prepareStep: execOptions?.prepareStep,
toolPayloadTransform,
isTaskComplete: execOptions?.isTaskComplete,
// Park the per-call requireToolApproval policy on the registry so the
// durable tool-call step can evaluate function-form policies with the
// real (toolName, args) on each call. The boolean shadow on the
// serialized workflow input is the cross-process fallback.
requireToolApproval: execOptions?.requireToolApproval,
// Signal drain — the closure reads from AgentThreadStreamRuntime's queues.
// Non-serializable; cross-process engines lose it and signals go undelivered.
drainPendingSignals: (scope) => typedAgent.__getDrainPendingSignals()(runId, scope),
// Signal messages already in the messageList at run start (from persisted
// history). Echoed as data-signal parts on the first LLM step so the client
// sees them without refetching. Spliced once, never re-emitted.
initialSignalEchoes: getInitialSignalEchoes(messageList),
// Agent-level goal config (judge resolver, tools resolver, scorer).
// Non-serializable — cross-process engines skip goal evaluation.
goal: agent.__getGoalConfig(),
// Tripwire from processInput (initial input processing). When an input
// processor calls abort() during runInputProcessors, we store the tripwire
// data here so the first llm-execution step can emit a tripwire chunk and
// bail immediately without calling the model.
tripwire: tripwireData,
// Call-time headers from modelSettings.headers. Kept off the serialized
// workflow input so they never reach durable storage; the durable
// llm-execution step reads them from this registry slot instead.
callTimeHeaders: extractCallTimeHeaders(execOptions?.modelSettings),
// Call-time structured output config with the live schema. The schema is
// non-serializable (Zod / standard-schema instance), so it lives on the
// in-process registry. The durable stream adapter reads it to pipe LLM
// text through `createObjectStreamTransformer`, producing `object-result`
// chunks. Cross-process engines lose this slot and structured output
// degrades to raw text.
structuredOutput: execOptions?.structuredOutput?.schema ? {
...execOptions.structuredOutput,
schema: toStandardSchema(execOptions.structuredOutput.schema)
} : void 0,
cleanup: () => {
}
};
return {
runId,
messageId,
workflowInput,
registryEntry,
messageList,
threadId,
resourceId
};
}
function extractCallTimeHeaders(modelSettings) {
const raw = modelSettings?.headers;
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return void 0;
const headers = {};
for (const [key, value] of Object.entries(raw)) {
if (typeof value === "string") headers[key] = value;
}
return Object.keys(headers).length > 0 ? headers : void 0;
}
var globalRunRegistry = new TTLCache({
max: 1e3,
ttl: 10 * 60 * 1e3,
updateAgeOnGet: true,
dispose: (entry) => {
entry?.cleanup?.();
},
noDisposeOnSet: true
});
function endRunSpansWithError(runId, error) {
try {
const entry = globalRunRegistry.get(runId);
(entry?.resumeModelSpan ?? entry?.modelSpan)?.error({ error, endSpan: true });
(entry?.resumeAgentSpan ?? entry?.agentSpan)?.error({ error, endSpan: true });
} catch {
}
}
var RunRegistry = class {
#entries = /* @__PURE__ */ new Map();
/**
* Register non-serializable state for a run
* @param runId - The unique run identifier
* @param entry - The registry entry containing tools, saveQueueManager, etc.
*/
register(runId, entry) {
this.cleanup(runId);
this.#entries.set(runId, entry);
}
/**
* Get the registry entry for a run
* @param runId - The unique run identifier
* @returns The registry entry or undefined if not found
*/
get(runId) {
return this.#entries.get(runId);
}
/**
* Get tools for a specific run
* @param runId - The unique run identifier
* @returns The tools record or an empty object if not found
*/
getTools(runId) {
return this.#entries.get(runId)?.tools ?? {};
}
/**
* Get SaveQueueManager for a specific run
* @param runId - The unique run identifier
* @returns The SaveQueueManager or undefined if not found
*/
getSaveQueueManager(runId) {
return this.#entries.get(runId)?.saveQueueManager;
}
/**
* Get the language model for a specific run
* @param runId - The unique run identifier
* @returns The MastraLanguageModel or undefined if not found
*/
getModel(runId) {
return this.#entries.get(runId)?.model;
}
/**
* Check if a run is registered
* @param runId - The unique run identifier
* @returns True if the run is registered
*/
has(runId) {
return this.#entries.has(runId);
}
/**
* Cleanup and remove a run's entry from the registry
* @param runId - The unique run identifier
*/
cleanup(runId) {
const entry = this.#entries.get(runId);
if (entry) {
entry.cleanup?.();
this.#entries.delete(runId);
}
}
/**
* Get the number of active runs in the registry
*/
get size() {
return this.#entries.size;
}
/**
* Get all active run IDs
*/
get runIds() {
return Array.from(this.#entries.keys());
}
/**
* Clear all entries from the registry
* Calls cleanup on each entry before removing
*/
clear() {
for (const runId of this.#entries.keys()) {
this.cleanup(runId);
}
}
};
var ExtendedRunRegistry = class extends RunRegistry {
#messageLists = /* @__PURE__ */ new Map();
#memoryInfo = /* @__PURE__ */ new Map();
/**
* Register non-serializable state for a run including MessageList
*/
registerWithMessageList(runId, entry, messageList, memoryInfo) {
this.register(runId, entry);
this.#messageLists.set(runId, messageList);
if (memoryInfo) {
this.#memoryInfo.set(runId, memoryInfo);
}
}
/**
* Get MessageList for a specific run
*/
getMessageList(runId) {
return this.#messageLists.get(runId);
}
/**
* Get memory info for a specific run
*/
getMemoryInfo(runId) {
return this.#memoryInfo.get(runId);
}
/**
* Override cleanup to also remove MessageList and memory info
*/
cleanup(runId) {
super.cleanup(runId);
this.#messageLists.delete(runId);
this.#memoryInfo.delete(runId);
}
/**
* Override clear to also clear MessageLists and memory info
*/
clear() {
super.clear();
this.#messageLists.clear();
this.#memoryInfo.clear();
}
};
function normalizeUsage(raw) {
if (!raw) {
return { inputTokens: 0, outputTokens: 0, totalTokens: 0 };
}
const inputTokens = raw.inputTokens ?? raw.promptTokens ?? 0;
const outputTokens = raw.outputTokens ?? raw.completionTokens ?? 0;
const totalTokens = raw.totalTokens ?? inputTokens + outputTokens;
return { inputTokens, outputTokens, totalTokens };
}
function createDurableAgentStream(options) {
const {
pubsub,
runId,
messageId,
model,
threadId,
resourceId,
offset,
onChunk,
onStepFinish,
onFinish,
onStreamFinished,
onError,
onSuspended,
onAbort,
onIterationComplete,
logger,
closeOnSuspend = false,
structuredOutput,
outputProcessors,
messageList: externalMessageList
} = options;
const logError = (message, error) => {
if (logger) {
logger.error(message, error);
} else {
console.error(message, error);
}
};
const messageList = externalMessageList ?? new MessageList({
threadId,
resourceId
});
let isSubscribed = false;
let cancelled = false;
let controller = null;
let resolveReady;
let rejectReady;
const ready = new Promise((resolve, reject) => {
resolveReady = resolve;
rejectReady = reject;
});
let lastErrorMessage;
const handleEvent = async (event) => {
if (!controller) return;
const streamEvent = event;
try {
switch (streamEvent.type) {
case AgentStreamEventTypes.CHUNK: {
const chunk = streamEvent.data;
if (chunk.type === "error") {
const errPayload = chunk.payload;
lastErrorMessage = errPayload?.error?.message || errPayload?.message || "LLM execution error";
}
safeEnqueue(controller, chunk);
await onChunk?.(chunk);
break;
}
case AgentStreamEventTypes.STEP_START: {
const chunk = streamEvent.data;
if (chunk && "type" in chunk) {
safeEnqueue(controller, chunk);
}
break;
}
case AgentStreamEventTypes.STEP_FINISH: {
const data = streamEvent.data;
await onStepFinish?.(data);
break;
}
case AgentStreamEventTypes.FINISH: {
const data = streamEvent.data;
const finishChunk = {
type: "finish",
payload: {
output: data.output,
stepResult: data.stepResult
}
};
safeEnqueue(controller, finishChunk);
safeClose(controller);
if (onFinish) {
try {
const steps = data.output?.steps ?? [];
const allToolResults = steps.flatMap((s) => s?.toolResults ?? []);
const allToolCalls = steps.flatMap((s) => s?.toolCalls ?? []);
await onFinish({
text: data.output?.text ?? "",
steps,
toolResults: allToolResults,
toolCalls: allToolCalls,
dynamicToolCalls: [],
dynamicToolResults: [],
staticToolCalls: [],
staticToolResults: [],
files: [],
sources: [],
reasoning: [],
content: [],
finishReason: data.stepResult?.reason ?? "stop",
usage: normalizeUsage(data.output?.usage),
totalUsage: normalizeUsage(data.output?.usage),
warnings: data.stepResult?.warnings ?? [],
request: { body: void 0 },
response: {},
reasoningText: void 0,
providerMetadata: void 0
});
} catch (callbackError) {
logError(`[DurableAgentStream] onFinish callback error:`, callbackError);
}
}
if (onError && data.stepResult?.reason === "error") {
try {
await onError({ error: new Error(lastErrorMessage || "LLM execution error") });
} catch (callbackError) {
logError(`[DurableAgentStream] onError (from FINISH) callback error:`, callbackError);
}
}
try {
await onStreamFinished?.();
} catch (callbackError) {
logError(`[DurableAgentStream] onStreamFinished callback error:`, callbackError);
}
break;
}
case AgentStreamEventTypes.ERROR: {
const data = streamEvent.data;
const error = new Error(data.error.message);
error.name = data.error.name;
if (data.error.stack) {
error.stack = data.error.stack;
}
safeEnqueue(controller, {
type: "error",
payload: { error }
});
safeClose(controller);
try {
await onError?.({ error });
} catch (callbackError) {
logError(`[DurableAgentStream] onError callback error:`, callbackError);
}
break;
}
case AgentStreamEventTypes.SUSPENDED: {
const data = streamEvent.data;
await onSuspended?.(data);
if (closeOnSuspend) {
safeClose(controller);
}
break;
}
case AgentStreamEventTypes.ABORT: {
const data = streamEvent.data;
try {
await onAbort?.(data);
} catch (callbackError) {
logError(`[DurableAgentStream] onAbort callback error:`, callbackError);
}
safeClose(controller);
break;
}
case AgentStreamEventTypes.ITERATION_COMPLETE: {
const data = streamEvent.data;
try {
await onIterationComplete?.(data);
} catch (callbackError) {
logError(`[DurableAgentStream] onIterationComplete callback error:`, callbackError);
}
break;
}
}
} catch (error) {
logError(`[DurableAgentStream] Error handling event ${streamEvent.type}:`, error);
}
};
const stream = new ReadableStream$1({
start(ctrl) {
controller = ctrl;
const topic = AGENT_STREAM_TOPIC(runId);
const subscribePromise = offset !== void 0 ? pubsub.subscribeFromOffset(topic, offset, handleEvent) : pubsub.subscribeWithReplay(topic, handleEvent);
subscribePromise.then(() => {
if (cancelled) {
void pubsub.unsubscribe(topic, handleEvent).catch((error) => {
logError(`[DurableAgentStream] Failed to unsubscribe from ${topic}:`, error);
});
resolveReady();
return;
}
isSubscribed = true;
resolveReady();
}).catch((error) => {
logError(`[DurableAgentStream] Failed to subscribe to ${topic}:`, error);
rejectReady(error);
ctrl.error(error);
});
},
cancel() {
cleanup();
}
});
const cleanup = () => {
cancelled = true;
if (isSubscribed) {
isSubscribed = false;
const topic = AGENT_STREAM_TOPIC(runId);
void pubsub.unsubscribe(topic, handleEvent).catch((error) => {
logError(`[DurableAgentStream] Failed to unsubscribe from ${topic}:`, error);
});
}
controller = null;
};
const output = new MastraModelOutput({
model,
stream,
messageList,
messageId,
options: {
runId,
onStepFinish,
// For durable agents there is only one MastraModelOutput for the whole run.
// isLLMExecutionStep must be true so output processors run per-chunk
// (processOutputStream / processPart path) rather than the batch
// runOutputProcessors path which only fires at finish. It also gates
// createObjectStreamTransformer for structured output.
// resolveFinalPromises forces text/finishReason promise resolution at
// step-finish despite isLLMExecutionStep being true — durable agents have
// no outer MastraModelOutput to resolve them.
structuredOutput,
isLLMExecutionStep: true,
resolveFinalPromises: true,
outputProcessors
}
});
return {
output,
cleanup,
ready
};
}
async function emitChunkEvent(pubsub, runId, chunk) {
const topic = AGENT_STREAM_TOPIC(runId);
await pubsub.publish(topic, {
type: AgentStreamEventTypes.CHUNK,
runId,
data: chunk
});
}
async function emitStepStartEvent(pubsub, runId, data) {
await pubsub.publish(AGENT_STREAM_TOPIC(runId), {
type: AgentStreamEventTypes.STEP_START,
runId,
data: { type: "step-start", ...data }
});
}
async function emitStepFinishEvent(pubsub, runId, data) {
await pubsub.publish(AGENT_STREAM_TOPIC(runId), {
type: AgentStreamEventTypes.STEP_FINISH,
runId,
data
});
}
async function emitFinishEvent(pubsub, runId, data) {
await pubsub.publish(AGENT_STREAM_TOPIC(runId), {
type: AgentStreamEventTypes.FINISH,
runId,
data
});
}
async function emitErrorEvent(pubsub, runId, error) {
await pubsub.publish(AGENT_STREAM_TOPIC(runId), {
type: AgentStreamEventTypes.ERROR,
runId,
data: {
error: {
name: error.name,
message: error.message
// stack intentionally omitted — avoid leaking internals through external pubsub
}
}
});
}
async function emitSuspendedEvent(pubsub, runId, data) {
await pubsub.publish(AGENT_STREAM_TOPIC(runId), {
type: AgentStreamEventTypes.SUSPENDED,
runId,
data
});
}
async function emitAbortEvent(pubsub, runId, data) {
await pubsub.publish(AGENT_STREAM_TOPIC(runId), {
type: AgentStreamEventTypes.ABORT,
runId,
data
});
}
async function emitIterationCompleteEvent(pubsub, runId, data) {
await pubsub.publish(AGENT_STREAM_TOPIC(runId), {
type: AgentStreamEventTypes.ITERATION_COMPLETE,
runId,
data
});
}
// src/agent/durable/workflows/shared/execute-tool-calls.ts
async function executeDurableToolCalls(ctx) {
const toolResults = [];
for (const toolCall of ctx.toolCalls) {
if (toolCall.providerEx