UNPKG

@mastra/core

Version:
1 lines 172 kB
{"version":3,"file":"storage-C4FD5U8Z.cjs","names":["readPositiveIntEnv","EventEmitterPubSub","#getPubSub","NoopLeaseProvider","isLeaseProvider","#getLeaseProvider","#hasLiveThreadLease","#resolveLeaseProvider","#id","#stopLeaseRenewal","#getState","#transferThreadLease","#startLeaseRenewal","#acquireOrTransferThreadLease","#statesByPubSub","#isApprovalSuspendedRun","#isSuspendedRun","#threadKey","#isThreadBlockingRun","#publishAndWait","#threadTopic","#markRunSuspending","#getSourceId","#getThreadTarget","#releaseThreadLease","#publish","#resetState","#persistSignal","#nextStreamIdentity","#withBroadcastStream","#persistAndBroadcastIdleSignal","#broadcastPersistedSignal","#clearSuspendedRun","#sweepStaleSuspendedRecords","#watchThreadRunCompletion","#cleanupPreparedRun","#hasPendingThreadWork","#drainPendingSignals","#serializeSignal","#drainPendingContinuations","#drainPendingIdleSignals","#startContinuation","getErrorFromUnknown","#waitForRemoteRunToFinish","createSignal","#createMessageSignalInput","createMessageSignal","#generateSignalMessageId","parseMemoryRequestContext","applyStateSignal","resolveDeliveryAttributes","createSignal","StorageDomain","#notifications"],"sources":["../src/agent/thread-stream-runtime.ts","../src/notifications/delivery-policy.ts","../src/notifications/signals.ts","../src/notifications/dispatcher.ts","../src/notifications/storage.ts"],"sourcesContent":["import { randomUUID } from 'node:crypto';\n\nimport { getErrorFromUnknown } from '../error';\nimport { EventEmitterPubSub } from '../events/event-emitter';\nimport { isLeaseProvider, NoopLeaseProvider } from '../events/pubsub';\nimport type { LeaseProvider, PubSub } from '../events/pubsub';\nimport type { EventCallback } from '../events/types';\nimport { parseMemoryRequestContext } from '../memory/types';\nimport type { RequestContext } from '../request-context';\nimport { MASTRA_RESOURCE_ID_KEY, MASTRA_THREAD_ID_KEY } from '../request-context';\nimport type { MastraModelOutput } from '../stream/base/output';\nimport { readPositiveIntEnv } from '../utils';\nimport type { Agent } from './agent';\nimport type { AgentExecutionOptions } from './agent.types';\nimport type { MessageListInput } from './message-list';\nimport { createMessageSignal, createSignal, resolveDeliveryAttributes } from './signals';\nimport type { AgentMessageInput, AgentStateSignalInput, CreatedAgentSignal } from './signals';\nimport { applyStateSignal } from './state-signals';\nimport type {\n AgentSignal,\n AgentSubscribeToThreadOptions,\n AgentThreadSubscription,\n QueueAgentMessageOptions,\n QueueAgentMessageResult,\n SendAgentMessageOptions,\n SendAgentMessageResult,\n SendAgentSignalOptions,\n SendAgentSignalAccepted,\n SendAgentSignalResult,\n SendAgentStateSignalOptions,\n SendAgentStateSignalResult,\n} from './types';\n\nconst AGENT_THREAD_KEY_SEPARATOR = '\\u0000';\nconst AGENT_THREAD_STREAM_TOPIC_PREFIX = 'agent.thread-stream';\n/**\n * Lease TTL for the cross-process thread lease acquired in the idle-wake\n * path. Kept short so a crashed owner process frees the thread quickly; a\n * background timer renews it while the run is still running. Overridable via\n * `MASTRA_AGENT_THREAD_LEASE_TTL_MS` (production keeps the 15s default).\n */\nconst AGENT_THREAD_LEASE_TTL_MS = readPositiveIntEnv('MASTRA_AGENT_THREAD_LEASE_TTL_MS', 15_000);\n/**\n * Interval at which the owner process renews its lease. Defaults to TTL/3,\n * leaving room for two missed renewals (network blip, GC pause) before the\n * lease expires. Overridable via `MASTRA_AGENT_THREAD_LEASE_RENEW_INTERVAL_MS`.\n */\nconst AGENT_THREAD_LEASE_RENEW_INTERVAL_MS = readPositiveIntEnv(\n 'MASTRA_AGENT_THREAD_LEASE_RENEW_INTERVAL_MS',\n Math.floor(AGENT_THREAD_LEASE_TTL_MS / 3),\n);\n/**\n * TTL for a suspended run's warm in-memory state — the parked thread-run record\n * (swept by #sweepStaleSuspendedRecords). The Mastra internal-workflow registry\n * reads the same `MASTRA_SUSPENDED_RUN_TTL_MS` so both expire on one bound. A\n * suspended run is kept warm so a same-instance resume can reattach and the thread\n * stays blocked; once it lapses the state is evicted and resume falls back to the\n * durable snapshot. Multi-instance deployments (resume rarely lands on the origin)\n * can shed it sooner; 30 minute default.\n */\nconst AGENT_SUSPENDED_RUN_TTL_MS = readPositiveIntEnv('MASTRA_SUSPENDED_RUN_TTL_MS', 30 * 60 * 1000);\n\nexport let defaultAgentThreadPubSub: PubSub = new EventEmitterPubSub();\n\nfunction withThreadMemory(memory: unknown, resourceId: string, threadId: string) {\n return {\n ...((memory && typeof memory === 'object' ? memory : {}) as Record<string, unknown>),\n resource: (memory as { resource?: string } | undefined)?.resource ?? resourceId,\n thread: (memory as { thread?: string } | undefined)?.thread ?? threadId,\n };\n}\n\ntype AgentThreadRunLifecycle = 'running' | 'suspending' | 'suspended' | 'completed' | 'failed' | 'aborted';\n\ntype AgentThreadRunSuspension = {\n toolCallId?: string;\n toolName?: string;\n kind: 'approval' | 'generic-tool';\n};\n\ntype AgentThreadRunRecord<OUTPUT = unknown> = {\n agent: Agent<any, any, any, any>;\n output: MastraModelOutput<OUTPUT>;\n runId: string;\n streamId: string;\n streamSeq: number;\n lifecycle: AgentThreadRunLifecycle;\n suspension?: AgentThreadRunSuspension;\n /** When the record was parked as suspended (ms epoch); drives the TTL sweep. */\n suspendedAt?: number;\n threadId: string;\n resourceId?: string;\n streamOptions: AgentExecutionOptions<OUTPUT>;\n createSubscriberStream?: () => ReadableStream<unknown>;\n};\n\ntype PreparedThreadRun = {\n abortController: AbortController;\n cleanup: () => void;\n};\n\ntype PendingIdleSignal<OUTPUT = unknown> = {\n agent: Agent<any, any, any, any>;\n signal: CreatedAgentSignal;\n runId: string;\n resourceId: string;\n threadId: string;\n streamOptions?: AgentExecutionOptions<OUTPUT>;\n};\n\ntype PendingContinuation<OUTPUT = unknown> = {\n agent: Agent<any, any, any, any>;\n messages: MessageListInput;\n runId: string;\n resourceId: string;\n threadId: string;\n streamOptions?: AgentExecutionOptions<OUTPUT>;\n};\n\ntype AgentThreadRuntimeState = {\n threadRunsById: Map<string, AgentThreadRunRecord<any>>;\n threadRunsByStreamId: Map<string, AgentThreadRunRecord<any>>;\n threadKeysByRunId: Map<string, string>;\n remoteThreadKeysByRunId: Map<string, string>;\n activeThreadRunIds: Map<string, string>;\n activeThreadStreamIds: Map<string, string>;\n streamSeqByRunId: Map<string, number>;\n approvalSuspendedRunIds: Set<string>;\n suspendedRunIds: Set<string>;\n suspensionMetadataByRunId: Map<string, AgentThreadRunSuspension>;\n pendingSignalsByThread: Map<string, CreatedAgentSignal[]>;\n // Signals queued for a run that is starting but has not made its first model\n // request yet. The first LLM step drains these and folds them into that\n // request; `pendingSignalsByThread` follow-ups instead become their own turn.\n preRunSignalsByThread: Map<string, CreatedAgentSignal[]>;\n pendingIdleSignalsByThread: Map<string, PendingIdleSignal<any>[]>;\n pendingContinuationsByThread: Map<string, PendingContinuation<any>[]>;\n watchedThreadStreamIds: Set<string>;\n preparedRunsById: Map<string, PreparedThreadRun>;\n abortedRunIds: Set<string>;\n /**\n * Active lease-renewal timers keyed by runId. Set when the owner\n * process wins the cross-process lease, cleared on release. Stored\n * here (not on a Map<key,timer>) so a run's renewal timer survives even\n * if `activeThreadRunIds` is rotated by a follow-up signal.\n */\n leaseRenewalTimers: Map<string, ReturnType<typeof setInterval>>;\n};\n\nexport type AgentThreadState = 'active' | 'idle';\n\ntype SerializableAgentSignal = AgentSignal & Pick<CreatedAgentSignal, 'id' | 'createdAt'>;\n\ntype AgentThreadStreamRuntimeEvent =\n | { type: 'run-registered'; runId: string; streamId: string; streamSeq: number }\n | { type: 'stream-part'; runId: string; streamId: string; part: unknown; sourceId: string }\n | { type: 'run-completed'; runId: string; streamId?: string }\n | { type: 'run-suspended'; runId: string; streamId?: string }\n | { type: 'run-abort-requested'; runId: string; streamId: string }\n | { type: 'run-aborted'; runId: string; streamId?: string }\n | { type: 'run-failed'; runId: string; streamId?: string; error: string }\n | { type: 'signal-enqueued'; runId: string; signal: SerializableAgentSignal; sourceId: string; preRun?: boolean };\n\nfunction createRuntimeState(): AgentThreadRuntimeState {\n return {\n threadRunsById: new Map(),\n threadRunsByStreamId: new Map(),\n threadKeysByRunId: new Map(),\n remoteThreadKeysByRunId: new Map(),\n activeThreadRunIds: new Map(),\n activeThreadStreamIds: new Map(),\n streamSeqByRunId: new Map(),\n approvalSuspendedRunIds: new Set(),\n suspendedRunIds: new Set(),\n suspensionMetadataByRunId: new Map(),\n pendingSignalsByThread: new Map(),\n preRunSignalsByThread: new Map(),\n pendingIdleSignalsByThread: new Map(),\n pendingContinuationsByThread: new Map(),\n watchedThreadStreamIds: new Set(),\n preparedRunsById: new Map(),\n abortedRunIds: new Set(),\n leaseRenewalTimers: new Map(),\n };\n}\n\nexport class AgentThreadStreamRuntime {\n #id?: string;\n #statesByPubSub = new WeakMap<PubSub, AgentThreadRuntimeState>();\n\n #getPubSub(pubsub?: PubSub): PubSub {\n return pubsub ?? defaultAgentThreadPubSub;\n }\n\n /**\n * Resolve the {@link LeaseProvider} for the configured pubsub. Leasing is\n * a separate capability from event delivery: a backend only implements it\n * when it can genuinely coordinate a distributed lock (Redis via SET-NX,\n * in-memory for single-process). We feature-detect once here so all lease\n * call sites can use the resolved provider unconditionally.\n *\n * `CachingPubSub` exposes its inner's lease provider via `getLeaseProvider`\n * (caching is transparent to leasing). Otherwise we duck-type the pubsub\n * directly. Backends that cannot lease fall back to {@link NoopLeaseProvider}\n * (always-win / no-op), preserving single-process behavior.\n */\n #getLeaseProvider(pubsub?: PubSub): LeaseProvider {\n const resolved = this.#getPubSub(pubsub);\n const unwrap = (resolved as { getLeaseProvider?: () => LeaseProvider | undefined }).getLeaseProvider;\n if (typeof unwrap === 'function') {\n const inner = unwrap.call(resolved);\n return inner ?? NoopLeaseProvider;\n }\n return isLeaseProvider(resolved) ? resolved : NoopLeaseProvider;\n }\n\n #resolveLeaseProvider(pubsub?: PubSub): { provider: LeaseProvider; isFallback: boolean } {\n const provider = this.#getLeaseProvider(pubsub);\n return { provider, isFallback: provider === NoopLeaseProvider };\n }\n\n async #hasLiveThreadLease(pubsub: PubSub, key: string, runId: string): Promise<boolean> {\n const { provider, isFallback } = this.#resolveLeaseProvider(pubsub);\n if (isFallback) return true;\n return provider\n .getLeaseOwner(key)\n .then(owner => owner === runId)\n .catch(() => false);\n }\n\n #getSourceId(): string {\n this.#id ??= randomUUID();\n return this.#id;\n }\n\n /**\n * Fire-and-forget release of the cross-process thread lease held by\n * this owner. Safe to call when no lease was ever acquired — the\n * pubsub's `releaseLease` is a no-op for non-owners (Lua-guarded\n * GET+DEL on Redis), and the default in-memory implementation is\n * identical. Also stops the renewal timer if one is running for\n * this run.\n */\n #releaseThreadLease(pubsub: PubSub | undefined, key: string, runId: string): void {\n const resolved = this.#getPubSub(pubsub);\n this.#stopLeaseRenewal(resolved, runId);\n void this.#getLeaseProvider(resolved)\n .releaseLease(key, runId)\n .catch(() => {});\n }\n\n /**\n * Start a background timer that renews the cross-process lease at\n * TTL/3 intervals while the run is still going. If the lease is lost\n * (e.g. expired due to clock skew or pubsub outage) the renewal\n * stops itself — there's nothing useful we can do from the runner\n * side beyond log; the original owner will keep running until the run\n * itself errors or completes.\n */\n #startLeaseRenewal(pubsub: PubSub, key: string, runId: string): void {\n const state = this.#getState(pubsub);\n if (state.leaseRenewalTimers.has(runId)) return;\n const leaseProvider = this.#getLeaseProvider(pubsub);\n const timer = setInterval(() => {\n void leaseProvider\n .renewLease(key, runId, AGENT_THREAD_LEASE_TTL_MS)\n .then(renewed => {\n if (!renewed) {\n // If renewLease reports the lease is gone, stop renewing; the current stream may still finish,\n // but another process can now claim the thread until this run completes or errors.\n this.#stopLeaseRenewal(pubsub, runId);\n }\n })\n .catch(() => {});\n }, AGENT_THREAD_LEASE_RENEW_INTERVAL_MS);\n // Don't keep the process alive solely to renew a lease.\n if (typeof timer === 'object' && timer && typeof (timer as any).unref === 'function') {\n (timer as any).unref();\n }\n state.leaseRenewalTimers.set(runId, timer);\n }\n\n #stopLeaseRenewal(pubsub: PubSub, runId: string): void {\n const state = this.#getState(pubsub);\n const timer = state.leaseRenewalTimers.get(runId);\n if (!timer) return;\n clearInterval(timer);\n state.leaseRenewalTimers.delete(runId);\n }\n\n /**\n * Hand the cross-process thread lease from a finishing run (`fromRunId`)\n * to the run that will drain queued follow-up work next (`toRunId`),\n * without the lease key ever going empty.\n *\n * The previous owner releases its renewal timer and the new owner starts\n * its own; the lease key is re-stamped by `transferLease` (with a full fresh\n * TTL). On atomic backends (Redis, in-memory) a racing process cannot win a\n * freed key between a release and a re-acquire. Backends that can't transfer\n * atomically implement `transferLease` as release+acquire internally and own\n * that race cost. Returns `true` if the new owner now holds the lease.\n */\n async #transferThreadLease(\n pubsub: PubSub | undefined,\n key: string,\n fromRunId: string,\n toRunId: string,\n ): Promise<boolean> {\n const resolved = this.#getPubSub(pubsub);\n const leaseProvider = this.#getLeaseProvider(resolved);\n // `transferLease` is a required `LeaseProvider` method. Atomic backends\n // (Redis, in-memory) swap the key gap-free; backends that can't be atomic\n // implement it as release+acquire internally and own that race cost.\n const held = await leaseProvider\n .transferLease(key, fromRunId, toRunId, AGENT_THREAD_LEASE_TTL_MS)\n .catch(() => false);\n // Move the renewal timer to the new owner regardless: the old timer is\n // owner-guarded and would only no-op now, and the new owner needs its\n // own keep-alive for long drains.\n this.#stopLeaseRenewal(resolved, fromRunId);\n if (held) {\n this.#startLeaseRenewal(resolved, key, toRunId);\n }\n return held;\n }\n\n /**\n * Ensure this process owns the cross-process lease for `toRunId` before it\n * starts a run, regardless of whether it already held the lease.\n *\n * - When `fromRunId` is provided (draining after a run this process owned),\n * atomically transfer the held lease to `toRunId` — gap-free, no empty key.\n * - When `fromRunId` is absent, or the transfer reports the old owner no\n * longer holds the lease, fall back to a fresh `acquireLease`. This covers\n * a *different* process that observed the owner finish via pub/sub and now\n * wants to wake the thread: it never held the lease, so it must win one.\n *\n * On success the renewal timer is started for `toRunId`. On failure the\n * returned `owner` is the current holder so the caller can forward work to it.\n */\n async #acquireOrTransferThreadLease(\n pubsub: PubSub | undefined,\n key: string,\n toRunId: string,\n fromRunId?: string,\n ): Promise<{ acquired: boolean; owner?: string }> {\n const resolved = this.#getPubSub(pubsub);\n if (fromRunId) {\n const transferred = await this.#transferThreadLease(pubsub, key, fromRunId, toRunId);\n if (transferred) return { acquired: true, owner: toRunId };\n // Old owner lost the lease before the handoff — fall through to acquire.\n }\n const leaseProvider = this.#getLeaseProvider(resolved);\n const result = await leaseProvider\n .acquireLease(key, toRunId, AGENT_THREAD_LEASE_TTL_MS)\n .catch(() => ({ acquired: false as boolean, owner: undefined as string | undefined }));\n if (result.acquired) {\n this.#startLeaseRenewal(resolved, key, toRunId);\n return { acquired: true, owner: toRunId };\n }\n return { acquired: false, owner: result.owner };\n }\n\n /**\n * Whether the thread has any queued follow-up work that a finishing run's\n * completion handler would drain next: pending follow-up signals (including\n * any pre-run leftover that will be folded in), queued continuations, or\n * queued idle signals.\n */\n #hasPendingThreadWork(state: AgentThreadRuntimeState, key: string): boolean {\n return (\n (state.pendingSignalsByThread.get(key)?.length ?? 0) > 0 ||\n (state.preRunSignalsByThread.get(key)?.length ?? 0) > 0 ||\n (state.pendingContinuationsByThread.get(key)?.length ?? 0) > 0 ||\n (state.pendingIdleSignalsByThread.get(key)?.length ?? 0) > 0\n );\n }\n\n #getState(pubsub?: PubSub): AgentThreadRuntimeState {\n const resolvedPubSub = this.#getPubSub(pubsub);\n let state = this.#statesByPubSub.get(resolvedPubSub);\n if (!state) {\n state = createRuntimeState();\n this.#statesByPubSub.set(resolvedPubSub, state);\n }\n return state;\n }\n\n #threadKey(resourceId: string | undefined, threadId: string): string {\n return [resourceId ?? '', threadId].join(AGENT_THREAD_KEY_SEPARATOR);\n }\n\n #threadTopic(key: string): string {\n return `${AGENT_THREAD_STREAM_TOPIC_PREFIX}.${encodeURIComponent(key)}`;\n }\n\n #isApprovalSuspendedRun(state: AgentThreadRuntimeState, runId: string) {\n return state.approvalSuspendedRunIds.has(runId);\n }\n\n #isSuspendedRun(state: AgentThreadRuntimeState, runId: string) {\n return state.suspendedRunIds.has(runId) || this.#isApprovalSuspendedRun(state, runId);\n }\n\n #isThreadBlockingRun(state: AgentThreadRuntimeState, record: AgentThreadRunRecord<any>) {\n return (\n record.output.status === 'running' ||\n record.output.status === 'suspended' ||\n record.lifecycle === 'suspending' ||\n record.lifecycle === 'suspended' ||\n !!record.suspension ||\n this.#isSuspendedRun(state, record.runId)\n );\n }\n\n #serializeSignal(signal: CreatedAgentSignal): SerializableAgentSignal {\n return signal;\n }\n\n #nextStreamIdentity(state: AgentThreadRuntimeState, runId: string) {\n const streamSeq = (state.streamSeqByRunId.get(runId) ?? 0) + 1;\n state.streamSeqByRunId.set(runId, streamSeq);\n return { streamId: randomUUID(), streamSeq };\n }\n\n #markRunSuspending(\n state: AgentThreadRuntimeState,\n runId: string,\n streamId: string,\n suspension: AgentThreadRunSuspension,\n ) {\n state.suspendedRunIds.add(runId);\n state.suspensionMetadataByRunId.set(runId, suspension);\n const record = state.threadRunsByStreamId.get(streamId) ?? state.threadRunsById.get(runId);\n if (record) {\n record.lifecycle = 'suspending';\n record.suspension = suspension;\n }\n if (suspension.kind === 'approval') {\n state.approvalSuspendedRunIds.add(runId);\n }\n }\n\n #clearSuspendedRun(state: AgentThreadRuntimeState, runId: string) {\n state.suspendedRunIds.delete(runId);\n state.suspensionMetadataByRunId.delete(runId);\n state.approvalSuspendedRunIds.delete(runId);\n }\n\n #generateSignalMessageId(\n agent: Agent<any, any, any, any>,\n target: { threadId?: string; resourceId?: string },\n ): string {\n return (\n agent.getMastraInstance?.()?.generateId({\n idType: 'message',\n source: 'agent',\n entityId: agent.id,\n threadId: target.threadId,\n resourceId: target.resourceId,\n }) ?? randomUUID()\n );\n }\n\n #createMessageSignalInput(message: AgentMessageInput): AgentSignal {\n const normalizedMessage = typeof message === 'string' || Array.isArray(message) ? { contents: message } : message;\n return {\n ...normalizedMessage,\n type: 'user',\n tagName: 'user',\n };\n }\n\n getThreadState(options: { resourceId?: string; threadId: string }, pubsub?: PubSub): AgentThreadState {\n const state = this.#getState(pubsub);\n const key = this.#threadKey(options.resourceId, options.threadId);\n const activeRunId = state.activeThreadRunIds.get(key);\n if (!activeRunId) return 'idle';\n\n const activeRecord = state.threadRunsById.get(activeRunId);\n if (activeRecord && !this.#isThreadBlockingRun(state, activeRecord)) {\n state.activeThreadRunIds.delete(key);\n return 'idle';\n }\n\n return 'active';\n }\n\n #publish(pubsub: PubSub | undefined, key: string, event: AgentThreadStreamRuntimeEvent) {\n void this.#publishAndWait(pubsub, key, event).catch(() => {});\n }\n\n async #publishAndWait(pubsub: PubSub | undefined, key: string, event: AgentThreadStreamRuntimeEvent) {\n await this.#getPubSub(pubsub).publish(this.#threadTopic(key), {\n type: event.type,\n runId: event.runId,\n data: event,\n });\n }\n\n #withBroadcastStream<OUTPUT>(\n output: MastraModelOutput<OUTPUT>,\n pubsub: PubSub | undefined,\n key: string,\n streamId: string,\n ) {\n const runtime = this;\n\n const parts: unknown[] = [];\n const waiters = new Set<() => void>();\n let started = false;\n let done = false;\n let error: unknown;\n\n const wake = () => {\n const pending = [...waiters];\n waiters.clear();\n for (const waiter of pending) waiter();\n };\n\n const emitPart = async (part: unknown) => {\n if (part && typeof part === 'object' && 'type' in part) {\n const typedPart = part as { type?: string; payload?: { toolCallId?: string; toolName?: string } };\n if (typedPart.type === 'tool-call-approval' || typedPart.type === 'tool-call-suspended') {\n runtime.#markRunSuspending(runtime.#getState(pubsub), output.runId, streamId, {\n toolCallId: typedPart.payload?.toolCallId,\n toolName: typedPart.payload?.toolName,\n kind: typedPart.type === 'tool-call-approval' ? 'approval' : 'generic-tool',\n });\n }\n }\n parts.push(part);\n await runtime.#publishAndWait(pubsub, key, {\n type: 'stream-part',\n runId: output.runId,\n streamId,\n part,\n sourceId: runtime.#getSourceId(),\n });\n wake();\n };\n\n const start = () => {\n if (started) return;\n started = true;\n void (async () => {\n try {\n const source = output.fullStream as ReadableStream<unknown> | undefined;\n if (!source) return;\n\n if (typeof source.getReader === 'function') {\n const reader = source.getReader();\n try {\n while (true) {\n const { value: part, done: streamDone } = await reader.read();\n if (streamDone) break;\n await emitPart(part);\n }\n } finally {\n reader.releaseLock();\n }\n } else {\n for await (const part of source as any) {\n await emitPart(part);\n }\n }\n } catch (caught) {\n error = caught;\n } finally {\n done = true;\n wake();\n }\n })();\n };\n\n const createStream = () => {\n let index = 0;\n let closed = false;\n let waiter: (() => void) | undefined;\n return new ReadableStream({\n async pull(controller) {\n start();\n while (!closed) {\n if (index < parts.length) {\n controller.enqueue(parts[index++]);\n return;\n }\n if (error) {\n controller.error(error);\n return;\n }\n if (done) {\n controller.close();\n return;\n }\n await new Promise<void>(resolve => {\n waiter = resolve;\n waiters.add(resolve);\n });\n if (waiter) {\n waiters.delete(waiter);\n waiter = undefined;\n }\n }\n },\n cancel() {\n closed = true;\n if (waiter) {\n waiters.delete(waiter);\n waiter();\n waiter = undefined;\n }\n },\n });\n };\n\n return { output, createSubscriberStream: createStream, startBroadcast: start };\n }\n\n #getThreadTarget(options?: { memory?: AgentExecutionOptions<any>['memory']; requestContext?: RequestContext }) {\n const thread = options?.memory?.thread;\n const threadId =\n (options?.requestContext?.get(MASTRA_THREAD_ID_KEY) as string | undefined) ||\n (typeof thread === 'string' ? thread : thread?.id);\n const resourceId =\n (options?.requestContext?.get(MASTRA_RESOURCE_ID_KEY) as string | undefined) || options?.memory?.resource;\n\n return { threadId, resourceId };\n }\n\n prepareRunOptions<OUTPUT>(options: AgentExecutionOptions<OUTPUT>, pubsub?: PubSub): AgentExecutionOptions<OUTPUT> {\n const { threadId } = this.#getThreadTarget(options);\n if (!threadId || !options.runId) return options;\n\n const state = this.#getState(pubsub);\n const abortController = new AbortController();\n const upstreamAbortSignal = options.abortSignal;\n const abort = () => abortController.abort();\n if (upstreamAbortSignal?.aborted) {\n abort();\n } else {\n upstreamAbortSignal?.addEventListener('abort', abort, { once: true });\n }\n\n state.preparedRunsById.set(options.runId, {\n abortController,\n cleanup: () => upstreamAbortSignal?.removeEventListener('abort', abort),\n });\n\n if (state.abortedRunIds.has(options.runId)) {\n abort();\n }\n\n return {\n ...options,\n abortSignal: abortController.signal,\n };\n }\n\n abortRun(runId: string, pubsub?: PubSub): boolean {\n const state = this.#getState(pubsub);\n const preparedRun = state.preparedRunsById.get(runId);\n if (!preparedRun) {\n state.abortedRunIds.add(runId);\n return false;\n }\n\n preparedRun.abortController.abort();\n state.abortedRunIds.add(runId);\n\n const key = state.threadKeysByRunId.get(runId);\n if (key) {\n const streamId = state.activeThreadRunIds.get(key) === runId ? state.activeThreadStreamIds.get(key) : undefined;\n this.#releaseThreadLease(pubsub, key, runId);\n this.#publish(pubsub, key, { type: 'run-aborted', runId, streamId });\n }\n\n return true;\n }\n\n getActiveThreadRunId(options: AgentSubscribeToThreadOptions, pubsub?: PubSub): string | undefined {\n const state = this.#getState(pubsub);\n const key = this.#threadKey(options.resourceId, options.threadId);\n const activeRunId = state.activeThreadRunIds.get(key);\n if (!activeRunId) return undefined;\n\n const record = state.threadRunsById.get(activeRunId);\n if (record && !this.#isThreadBlockingRun(state, record)) return undefined;\n\n return activeRunId;\n }\n\n getResumableThreadRun(\n options: AgentSubscribeToThreadOptions & { runId: string; toolCallId?: string },\n pubsub?: PubSub,\n ): { runId: string; toolCallId?: string } | undefined {\n const state = this.#getState(pubsub);\n const key = this.#threadKey(options.resourceId, options.threadId);\n const record = state.threadRunsById.get(options.runId);\n const isSuspended = this.#isSuspendedRun(state, options.runId);\n if (!record || state.threadKeysByRunId.get(options.runId) !== key || !isSuspended) {\n return undefined;\n }\n\n const suspension = record.suspension ?? state.suspensionMetadataByRunId.get(options.runId);\n if (options.toolCallId && suspension?.toolCallId && suspension.toolCallId !== options.toolCallId) {\n return undefined;\n }\n\n return { runId: options.runId, toolCallId: options.toolCallId ?? suspension?.toolCallId };\n }\n\n abortThread(options: AgentSubscribeToThreadOptions, pubsub?: PubSub): boolean {\n const resolvedPubSub = this.#getPubSub(pubsub);\n const state = this.#getState(resolvedPubSub);\n const key = this.#threadKey(options.resourceId, options.threadId);\n const runId = this.getActiveThreadRunId(options, resolvedPubSub);\n if (!runId) return false;\n if (state.preparedRunsById.has(runId)) return this.abortRun(runId, resolvedPubSub);\n if (state.threadKeysByRunId.get(runId) === key) {\n // Reserved locally (a sendSignal wake that has not prepared its run yet):\n // record the abort intent in abortedRunIds so prepareRunOptions aborts the\n // run the moment it starts, instead of letting it run to completion.\n this.abortRun(runId, resolvedPubSub);\n return true;\n }\n if (state.remoteThreadKeysByRunId.get(runId) !== key) return false;\n const streamId = state.activeThreadStreamIds.get(key);\n if (!streamId) return false;\n this.#publish(resolvedPubSub, key, { type: 'run-abort-requested', runId, streamId });\n return true;\n }\n\n /** @internal */\n resetForTests() {\n for (const pubsub of [defaultAgentThreadPubSub]) {\n this.#resetState(pubsub);\n void (pubsub as { close?: () => Promise<void> }).close?.();\n }\n defaultAgentThreadPubSub = new EventEmitterPubSub();\n }\n\n #resetState(pubsub: PubSub) {\n const state = this.#statesByPubSub.get(pubsub);\n if (!state) return;\n\n state.preparedRunsById.forEach(preparedRun => {\n preparedRun.abortController.abort();\n preparedRun.cleanup();\n });\n state.leaseRenewalTimers.forEach(timer => clearInterval(timer));\n state.leaseRenewalTimers.clear();\n state.threadRunsById.clear();\n state.threadRunsByStreamId.clear();\n state.threadKeysByRunId.clear();\n state.remoteThreadKeysByRunId.clear();\n state.activeThreadRunIds.clear();\n state.approvalSuspendedRunIds.clear();\n state.suspendedRunIds.clear();\n state.suspensionMetadataByRunId.clear();\n state.pendingSignalsByThread.clear();\n state.preRunSignalsByThread.clear();\n state.pendingIdleSignalsByThread.clear();\n state.pendingContinuationsByThread.clear();\n state.activeThreadStreamIds.clear();\n state.streamSeqByRunId.clear();\n state.watchedThreadStreamIds.clear();\n state.preparedRunsById.clear();\n state.abortedRunIds.clear();\n }\n\n #cleanupPreparedRun(state: AgentThreadRuntimeState, runId: string) {\n state.preparedRunsById.get(runId)?.cleanup();\n state.preparedRunsById.delete(runId);\n state.abortedRunIds.delete(runId);\n }\n\n async #persistSignal(\n agent: Agent<any, any, any, any>,\n signal: CreatedAgentSignal,\n resourceId: string,\n threadId: string,\n requestContext?: RequestContext,\n ) {\n // Transient signals are delivery-only: never write them to storage, even when the\n // active-behavior asked to persist. Honored here (not just in the memory layer) so it holds\n // for any memory implementation, including ones without a signal-aware save filter.\n if (signal.transient) return;\n const memory = await agent.getMemory({ requestContext });\n if (!memory) return;\n await memory.saveMessages({\n messages: [signal.toDBMessage({ resourceId, threadId })],\n });\n }\n\n #broadcastPersistedSignal(\n state: AgentThreadRuntimeState,\n pubsub: PubSub | undefined,\n key: string,\n runId: string,\n signal: CreatedAgentSignal,\n resourceId: string,\n threadId: string,\n ) {\n let finish!: () => void;\n const finished = new Promise<void>(resolve => {\n finish = resolve;\n });\n const parts: any[] = [\n { type: 'start', runId },\n { ...signal.toDataPart(), runId },\n {\n type: 'finish',\n runId,\n payload: {\n stepResult: { reason: 'stop' },\n output: {\n usage: { inputTokens: 0, outputTokens: 0, totalTokens: 0 },\n },\n },\n },\n ];\n const output = {\n runId,\n status: 'running',\n fullStream: new ReadableStream({\n start(controller) {\n for (const part of parts) controller.enqueue(part);\n controller.close();\n finish();\n },\n }),\n _waitUntilFinished: () => finished,\n } as MastraModelOutput<any>;\n const { streamId, streamSeq } = this.#nextStreamIdentity(state, runId);\n const {\n output: outputForSubscribers,\n createSubscriberStream,\n startBroadcast,\n } = this.#withBroadcastStream(output, pubsub, key, streamId);\n const record: AgentThreadRunRecord<any> = {\n agent: { id: `persisted-signal:${signal.id}` } as Agent<any, any, any, any>,\n output: outputForSubscribers,\n runId,\n streamId,\n streamSeq,\n lifecycle: 'running',\n threadId,\n resourceId,\n streamOptions: {},\n createSubscriberStream,\n };\n\n state.threadRunsById.set(runId, record);\n state.threadRunsByStreamId.set(streamId, record);\n state.threadKeysByRunId.set(runId, key);\n state.activeThreadStreamIds.set(key, streamId);\n const registered = this.#publishAndWait(pubsub, key, { type: 'run-registered', runId, streamId, streamSeq });\n void registered.then(startBroadcast, startBroadcast);\n void outputForSubscribers._waitUntilFinished().finally(() => {\n setTimeout(() => {\n state.threadRunsByStreamId.delete(streamId);\n if (state.threadRunsById.get(runId) === record) {\n state.threadRunsById.delete(runId);\n state.threadKeysByRunId.delete(runId);\n }\n if (state.activeThreadRunIds.get(key) === runId && state.activeThreadStreamIds.get(key) === streamId) {\n state.activeThreadRunIds.delete(key);\n state.activeThreadStreamIds.delete(key);\n }\n this.#releaseThreadLease(pubsub, key, runId);\n this.#publish(pubsub, key, { type: 'run-completed', runId, streamId });\n }, 0);\n });\n }\n\n async #persistAndBroadcastIdleSignal(\n state: AgentThreadRuntimeState,\n pubsub: PubSub | undefined,\n key: string,\n runId: string,\n agent: Agent<any, any, any, any>,\n signal: CreatedAgentSignal,\n resourceId: string,\n threadId: string,\n requestContext?: RequestContext,\n ) {\n if (signal.transient) return;\n\n await this.#persistSignal(agent, signal, resourceId, threadId, requestContext);\n this.#broadcastPersistedSignal(state, pubsub, key, runId, signal, resourceId, threadId);\n }\n\n /**\n * Evict SUSPENDED records parked longer than {@link AGENT_SUSPENDED_RUN_TTL_MS}.\n * Called lazily on each registration so cleanup is proportional to activity and\n * zero-cost when idle — mirrors the internal-workflow registry sweep. Bounds the\n * records left behind by abandoned suspends and by resumes that land on a\n * different instance (which never clean the origin instance's record).\n *\n * When the expiring record is still the run's current record — an abandoned\n * suspend, not one superseded by a same-instance resume — the teardown mirrors\n * #watchThreadRunCompletion's terminal path: it clears run-level state, releases\n * the cross-process lease, and publishes `run-completed` so remote subscribers\n * stop treating the thread as blocked and drain any queued follow-up work. A\n * superseded older stream just has its stream entry dropped; the resumed run\n * keeps its lease, suspended marker, and active slot.\n */\n #sweepStaleSuspendedRecords(state: AgentThreadRuntimeState, pubsub: PubSub | undefined) {\n const now = Date.now();\n for (const [streamId, record] of state.threadRunsByStreamId) {\n if (record.lifecycle !== 'suspended' || record.suspendedAt === undefined) continue;\n if (now - record.suspendedAt <= AGENT_SUSPENDED_RUN_TTL_MS) continue;\n state.threadRunsByStreamId.delete(streamId);\n state.watchedThreadStreamIds.delete(streamId);\n // A same-instance resume re-registers the run under a newer streamId, so a\n // record that is no longer the run's current record is just the superseded\n // older stream: dropping its stream entry above is enough. Only the current\n // record (an abandoned suspend) gets the full run-level teardown below.\n if (state.threadRunsById.get(record.runId) !== record) continue;\n const staleKey = this.#threadKey(record.resourceId, record.threadId);\n state.threadRunsById.delete(record.runId);\n state.threadKeysByRunId.delete(record.runId);\n this.#clearSuspendedRun(state, record.runId);\n // Stop renewing and release the cross-process lease, otherwise the run's\n // lease-renewal timer keeps the thread owned forever on other instances.\n this.#releaseThreadLease(pubsub, staleKey, record.runId);\n if (\n state.activeThreadRunIds.get(staleKey) === record.runId &&\n state.activeThreadStreamIds.get(staleKey) === streamId\n ) {\n state.activeThreadRunIds.delete(staleKey);\n state.activeThreadStreamIds.delete(staleKey);\n }\n this.#publish(pubsub, staleKey, { type: 'run-completed', runId: record.runId, streamId });\n }\n }\n\n registerRun<OUTPUT>(\n agent: Agent<any, any, any, any>,\n output: MastraModelOutput<OUTPUT>,\n streamOptions: AgentExecutionOptions<OUTPUT>,\n pubsub?: PubSub,\n ): Promise<void> | undefined {\n const { threadId, resourceId } = this.#getThreadTarget(streamOptions);\n if (!threadId) return;\n\n const state = this.#getState(pubsub);\n this.#sweepStaleSuspendedRecords(state, pubsub);\n const key = this.#threadKey(resourceId, threadId);\n const { streamId, streamSeq } = this.#nextStreamIdentity(state, output.runId);\n const {\n output: outputForSubscribers,\n createSubscriberStream,\n startBroadcast,\n } = this.#withBroadcastStream(output, pubsub, key, streamId);\n const record: AgentThreadRunRecord<OUTPUT> = {\n agent,\n output: outputForSubscribers,\n runId: output.runId,\n streamId,\n streamSeq,\n lifecycle: 'running',\n threadId,\n resourceId,\n streamOptions: streamOptions as AgentThreadRunRecord<OUTPUT>['streamOptions'],\n createSubscriberStream,\n };\n\n this.#clearSuspendedRun(state, output.runId);\n state.threadRunsById.set(output.runId, record);\n state.threadRunsByStreamId.set(streamId, record);\n state.threadKeysByRunId.set(output.runId, key);\n state.activeThreadRunIds.set(key, output.runId);\n state.activeThreadStreamIds.set(key, streamId);\n const resolvedPubSub = this.#getPubSub(pubsub);\n const registered = (async () => {\n // Every thread-bound run must hold the cross-process lease while it is\n // live: the liveness checks (markActiveIfLive / #waitForRemoteRunToFinish)\n // treat a lease-less run as a ghost, so a plain `agent.stream()` run that\n // never acquired would let contending instances start competing runs\n // instead of serializing behind it. Acquire BEFORE publishing\n // `run-registered` so an observer that checks liveness on receipt finds\n // the lease held. Same-owner acquire is an idempotent TTL refresh, so\n // signal-woken runs that already hold the lease under this runId just\n // renew. Fail-open on loss or error (simultaneous-start race): proceed\n // and never roll back the local registration — matches pre-lease\n // semantics and sendSignal's documented fail-open rationale. A thrown\n // acquire (transient provider error) is treated as acquired so renewal\n // starts: if the acquire landed server-side but the response failed,\n // skipping renewal would let the lease expire mid-run; renewal\n // self-stops when we don't own the key.\n const lease = await this.#getLeaseProvider(resolvedPubSub)\n .acquireLease(key, output.runId, AGENT_THREAD_LEASE_TTL_MS)\n .catch(() => ({ acquired: true as boolean }));\n if (lease.acquired) this.#startLeaseRenewal(resolvedPubSub, key, output.runId);\n await this.#publishAndWait(pubsub, key, {\n type: 'run-registered',\n runId: output.runId,\n streamId,\n streamSeq,\n });\n })();\n // Always drive the run's stream to completion, even when no caller consumes\n // the returned output (e.g. a fire-and-forget schedule wake). The broadcast\n // tee buffers every part, so a later/external subscriber still replays the\n // full stream; without this pump the run never reaches a terminal state and\n // its active-run record + thread lease would never release, permanently\n // wedging the thread.\n void registered.then(startBroadcast, startBroadcast);\n this.#watchThreadRunCompletion(state, pubsub, key, record);\n return registered;\n }\n\n #watchThreadRunCompletion(\n state: AgentThreadRuntimeState,\n pubsub: PubSub | undefined,\n key: string,\n record: AgentThreadRunRecord<any>,\n ) {\n if (state.watchedThreadStreamIds.has(record.streamId)) return;\n state.watchedThreadStreamIds.add(record.streamId);\n\n void record.output._waitUntilFinished().finally(() => {\n state.watchedThreadStreamIds.delete(record.streamId);\n this.#cleanupPreparedRun(state, record.runId);\n\n if (record.output.status === 'suspended' && this.#isSuspendedRun(state, record.runId)) {\n record.lifecycle = 'suspended';\n // Leak fix: stamp when the run parked so the lazy TTL sweep\n // (#sweepStaleSuspendedRecords) can evict it. The record stays fully intact\n // for resume routing / thread-blocking / subscriber replay exactly as before\n // — it is simply no longer retained for the life of the process. Mirrors the\n // internal-workflow registry, which already bounds parked runs this way.\n record.suspendedAt = Date.now();\n this.#publish(pubsub, key, { type: 'run-suspended', runId: record.runId, streamId: record.streamId });\n return;\n }\n\n record.lifecycle = 'completed';\n this.#clearSuspendedRun(state, record.runId);\n state.threadRunsByStreamId.delete(record.streamId);\n if (state.threadRunsById.get(record.runId) === record) {\n state.threadRunsById.delete(record.runId);\n state.threadKeysByRunId.delete(record.runId);\n }\n\n if (\n state.activeThreadRunIds.get(key) === record.runId &&\n state.activeThreadStreamIds.get(key) === record.streamId\n ) {\n state.activeThreadRunIds.delete(key);\n state.activeThreadStreamIds.delete(key);\n }\n\n // If queued follow-up work exists, keep the cross-process lease held by\n // handing it to the next run instead of releasing it: releasing here\n // would briefly empty the lease key, letting a racing process win it and\n // start a competing run on this thread. The drain runs under the\n // transferred lease and releases it only once every queue is empty. If\n // there's no pending work, release as usual so other processes can wake\n // the thread.\n this.#publish(pubsub, key, { type: 'run-completed', runId: record.runId, streamId: record.streamId });\n if (this.#hasPendingThreadWork(state, key)) {\n void this.#drainPendingSignals(state, pubsub, key, record);\n } else {\n this.#releaseThreadLease(pubsub, key, record.runId);\n }\n });\n }\n\n async #drainPendingSignals(\n state: AgentThreadRuntimeState,\n pubsub: PubSub | undefined,\n key: string,\n previousRun: AgentThreadRunRecord<any>,\n ) {\n if (state.activeThreadRunIds.has(key)) {\n return;\n }\n\n // A run can finish before its first model request drained its pre-run\n // signals (e.g. it errored early). Don't strand them — fold them into the\n // follow-up queue so the next run still picks them up.\n const preRunLeftover = state.preRunSignalsByThread.get(key);\n if (preRunLeftover?.length) {\n state.preRunSignalsByThread.delete(key);\n state.pendingSignalsByThread.set(key, [...preRunLeftover, ...(state.pendingSignalsByThread.get(key) ?? [])]);\n }\n\n const queue = state.pendingSignalsByThread.get(key);\n const signal = queue?.shift();\n if (signal && queue) {\n if (queue.length === 0) {\n state.pendingSignalsByThread.delete(key);\n }\n\n // Hand the lease from the finished run to this drained run before\n // streaming, so the lease key never goes empty during the handoff. If the\n // old owner already lost the lease (e.g. a pubsub blip let the TTL lapse\n // and another process took over), forward the signal to the new winner\n // instead of starting a competing run here.\n const nextRunId = randomUUID();\n state.activeThreadRunIds.set(key, nextRunId);\n state.threadKeysByRunId.set(nextRunId, key);\n const owns = await this.#acquireOrTransferThreadLease(pubsub, key, nextRunId, previousRun.runId);\n if (!owns.acquired) {\n if (state.activeThreadRunIds.get(key) === nextRunId) {\n state.activeThreadRunIds.delete(key);\n }\n state.threadKeysByRunId.delete(nextRunId);\n // Early follow-ups were already published as retained signal-enqueued\n // events, so only discard this runtime's local pre-run copies.\n state.preRunSignalsByThread.delete(key);\n // Put the signal back at the head so a later drain (or the winner) runs\n // it, and forward it to the current lease owner.\n const restored = state.pendingSignalsByThread.get(key) ?? [];\n state.pendingSignalsByThread.set(key, [signal, ...restored]);\n if (owns.owner) {\n await this.#publishAndWait(pubsub, key, {\n type: 'signal-enqueued',\n runId: owns.owner,\n signal: this.#serializeSignal(signal),\n sourceId: this.#getSourceId(),\n }).catch(() => {});\n state.pendingSignalsByThread.get(key)?.shift();\n if ((state.pendingSignalsByThread.get(key)?.length ?? 0) === 0) {\n state.pendingSignalsByThread.delete(key);\n }\n }\n return;\n }\n\n const output = await previousRun.agent.stream(signal, {\n ...(previousRun.streamOptions as any),\n runId: nextRunId,\n memory: withThreadMemory(\n previousRun.streamOptions.memory,\n previousRun.resourceId ?? '',\n previousRun.threadId ?? '',\n ),\n });\n\n if (queue.length > 0) {\n const nextRecord = state.threadRunsById.get(output.runId);\n if (nextRecord) {\n this.#watchThreadRunCompletion(state, pubsub, key, nextRecord);\n }\n }\n return;\n }\n\n if (await this.#drainPendingContinuations(state, pubsub, key, previousRun.runId)) {\n return;\n }\n\n if (await this.#drainPendingIdleSignals(state, pubsub, key, previousRun.runId)) {\n return;\n }\n\n // Nothing left to drain: release the lease we kept held for the drain.\n this.#releaseThreadLease(pubsub, key, previousRun.runId);\n }\n\n async #drainPendingContinuations(\n state: AgentThreadRuntimeState,\n pubsub: PubSub | undefined,\n key: string,\n fromRunId?: string,\n ) {\n if (state.activeThreadRunIds.has(key)) {\n return false;\n }\n\n const queue = state.pendingContinuationsByThread.get(key);\n const pending = queue?.shift();\n if (!pending || !queue) {\n return false;\n }\n if (queue.length === 0) {\n state.pendingContinuationsByThread.delete(key);\n }\n\n // A continuation only ever drains in the process that owned the finished\n // run, so it always carries a `fromRunId` to hand the held lease to. If the\n // old owner already lost the lease, re-queue the continuation and let the\n // new lease owner take over rather than starting a competing run here.\n if (fromRunId) {\n state.activeThreadRunIds.set(key, pending.runId);\n state.threadKeysByRunId.set(pending.runId, key);\n const owns = await this.#acquireOrTransferThreadLease(pubsub, key, pending.runId, fromRunId);\n if (!owns.acquired) {\n if (state.activeThreadRunIds.get(key) === pending.runId) {\n state.activeThreadRunIds.delete(key);\n }\n state.threadKeysByRunId.delete(pending.runId);\n state.preRunSignalsByThread.delete(key);\n const restored = state.pendingContinuationsByThread.get(key) ?? [];\n state.pendingContinuationsByThread.set(key, [pending, ...restored]);\n return false;\n }\n }\n\n this.#startContinuation(state, pubsub, key, pending);\n return true;\n }\n\n #startContinuation(\n state: AgentThreadRuntimeState,\n pubsub: PubSub | undefined,\n key: string,\n pending: PendingContinuation<any>,\n ) {\n state.activeThreadRunIds.set(key, pending.runId);\n state.threadKeysByRunId.set(pending.runId, k