@mastra/core/dist/background-tasks-lifNqs9M.cjs.map

{"version":3,"file":"background-tasks-lifNqs9M.cjs","names":["#mastra","#doInit","#ensureExecutionWorkersStarted","z"],"sources":["../src/background-tasks/workflow-id.ts","../src/background-tasks/manager.ts","../src/background-tasks/create.ts","../src/background-tasks/resolve-config.ts","../src/background-tasks/schema-injection.ts","../src/background-tasks/system-prompt.ts"],"sourcesContent":["/**\n * Workflow id used by the bg-task workflow registered on Mastra.\n * Double-underscore prefix marks it as internal — same convention as\n * `__batch-scoring-traces`.\n *\n * Lives in its own file (separate from `./workflow`) so `manager.ts` can\n * reference the id without statically pulling in `../workflows/evented`,\n * which would create a circular import via `agent → background-tasks →\n * workflow → evented → workflows/index → agent`.\n */\nexport const BACKGROUND_TASK_WORKFLOW_ID = '__background-task';\n","import { randomUUID } from 'node:crypto';\nimport type { Mastra } from '..';\nimport type { PubSub } from '../events/pubsub';\nimport type { Event, EventCallback } from '../events/types';\nimport type {\n  BackgroundTask,\n  BackgroundTaskManagerConfig,\n  BackgroundTaskStatus,\n  EnqueueResult,\n  TaskContext,\n  TaskFilter,\n  TaskPayload,\n  TaskListResult,\n  ToolExecutor,\n  BackgroundTaskEvent,\n} from './types';\nimport { BACKGROUND_TASK_WORKFLOW_ID } from './workflow-id';\n\nconst TOPIC_DISPATCH = 'background-tasks';\nconst TOPIC_RESULT = 'background-tasks-result';\nconst WORKER_GROUP = 'background-task-workers';\n\nexport class BackgroundTaskManager {\n  private pubsub!: PubSub;\n  config: Required<\n    Pick<BackgroundTaskManagerConfig, 'globalConcurrency' | 'perAgentConcurrency' | 'backpressure' | 'defaultTimeoutMs'>\n  > &\n    BackgroundTaskManagerConfig;\n\n  #mastra?: Mastra;\n\n  // Per-task contexts — keyed by task ID, holds closures from the caller's stream.\n  /** @internal — read by the workflow-engine step bodies in workflow.ts */\n  taskContexts: Map<string, TaskContext> = new Map();\n\n  // Static executors keyed by tool name. Populated by `Mastra` for every\n  // registered tool, and by `BackgroundTaskWorker.#wireStaticTools` on\n  // standalone worker processes. Used as the fallback for cross-process\n  // dispatch where the producer's per-task closure (taskContexts) is not\n  // visible — a remote worker resolves the tool by name instead.\n  private staticExecutors: Map<string, ToolExecutor> = new Map();\n\n  // Track active AbortControllers for running tasks (for cancellation + timeout)\n  /** @internal — read by the workflow-engine step bodies in workflow.ts */\n  activeAbortControllers: Map<string, AbortController> = new Map();\n\n  // Pubsub callbacks (kept for unsubscribe)\n  private workerCallback?: EventCallback;\n  private resultCallback?: EventCallback;\n\n  private shuttingDown = false;\n\n  // Cleanup interval handle\n  private cleanupInterval?: ReturnType<typeof setInterval>;\n\n  // Tracks the in-flight `init(pubsub)` so consumers can await readiness.\n  // Mastra fires init as fire-and-forget in `#ensureBackgroundTaskManager`,\n  // so without this any caller that hits `enqueue`/`resume`/`cancel`\n  // before init completes races against worker subscription + workflow\n  // registration. Public methods that depend on init await this promise\n  // before doing work.\n  private initPromise?: Promise<void>;\n\n  constructor(config: BackgroundTaskManagerConfig = { enabled: false }) {\n    this.config = {\n      globalConcurrency: config.globalConcurrency ?? 10,\n      perAgentConcurrency: config.perAgentConcurrency ?? 5,\n      backpressure: config.backpressure ?? 'queue',\n      defaultTimeoutMs: config.defaultTimeoutMs ?? 300_000,\n      ...config,\n    };\n  }\n\n  __registerMastra(mastra: Mastra) {\n    this.#mastra = mastra;\n  }\n\n  async getStorage() {\n    const storage = this.#mastra?.getStorage();\n    if (!storage) {\n      throw new Error('Storage is not initialized');\n    }\n    const bgStore = await storage.getStore('backgroundTasks');\n    if (!bgStore) {\n      throw new Error('Background tasks storage is not available');\n    }\n    return bgStore;\n  }\n\n  async init(pubsub: PubSub): Promise<void> {\n    if (this.initPromise) return this.initPromise;\n    this.initPromise = this.#doInit(pubsub);\n    return this.initPromise;\n  }\n\n  async #doInit(pubsub: PubSub): Promise<void> {\n    this.pubsub = pubsub;\n\n    const isProducerOnly = this.config.mode === 'producer';\n\n    // Result listener: fan-out so all processes receive results.\n    // Both producer and worker modes need this — the producer uses it\n    // to receive completion/failure notifications for dispatched tasks.\n    this.resultCallback = async (event: Event, ack?: () => Promise<void>) => {\n      if (event.type === 'task.completed' || event.type === 'task.failed') {\n        await this.handleResult(event);\n      }\n      await ack?.();\n    };\n\n    if (!isProducerOnly) {\n      // Worker: subscribes with group so only one worker processes each task.\n      this.workerCallback = async (event: Event, ack?: () => Promise<void>) => {\n        if (event.type === 'task.dispatch' || event.type === 'task.restart') {\n          await this.handleDispatch(event);\n        } else if (event.type === 'task.resume') {\n          await this.handleResume(event);\n        } else if (event.type === 'task.cancel') {\n          this.handleCancel(event);\n        }\n        await ack?.();\n      };\n\n      // Register the workflow BEFORE subscribing the worker so that any\n      // dispatch event the worker picks up can immediately resolve the\n      // workflow on Mastra. Reversing this order races: a publish that\n      // arrives between `subscribe(TOPIC_DISPATCH)` and the workflow\n      // registration triggers `__getInternalWorkflow` to throw\n      // `Workflow with id __background-task not found`, the task stays at\n      // `running` forever, and the dispatch is silently dropped.\n      if (this.#mastra) {\n        // Dynamic import breaks the static cycle:\n        // agent → background-tasks → manager → workflow → workflows/evented →\n        // workflows/index → agent. Static import works at runtime but during\n        // module evaluation in test environments the cycle leaves `Workflow`\n        // undefined when `evented/workflow.ts` evaluates its `class extends`.\n        const { buildBackgroundTaskWorkflow } = await import('./workflow');\n        const workflow = buildBackgroundTaskWorkflow(this);\n        if (!this.#mastra.__hasInternalWorkflow(BACKGROUND_TASK_WORKFLOW_ID)) {\n          // The `__background-task` workflow is typed against `EventedEngineType`\n          // and a concrete input/output schema, while `__registerInternalWorkflow`\n          // accepts the looser default `Workflow` shape. The cast is purely a\n          // type-level bridge — the runtime value is a real Workflow.\n          this.#mastra.__registerInternalWorkflow(\n            workflow as unknown as Parameters<Mastra['__registerInternalWorkflow']>[0],\n          );\n        }\n      }\n\n      await this.pubsub.subscribe(TOPIC_DISPATCH, this.workerCallback, { group: WORKER_GROUP });\n    }\n\n    await this.pubsub.subscribe(TOPIC_RESULT, this.resultCallback);\n\n    if (!isProducerOnly) {\n      // Recover stale tasks from a previous process — only workers should\n      // attempt recovery since they own execution.\n      await this.recoverStaleTasks();\n    }\n\n    // Start periodic cleanup if configured\n    const cleanupConfig = this.config.cleanup;\n    if (cleanupConfig) {\n      const intervalMs = cleanupConfig.cleanupIntervalMs ?? 60_000;\n      this.cleanupInterval = setInterval(() => {\n        void this.cleanup();\n      }, intervalMs);\n    }\n  }\n\n  // --- Per-task context registration ---\n\n  /**\n   * Register per-task hooks (executor, stream emitter, result injector).\n   * Called internally by createBackgroundTask or directly for advanced usage.\n   */\n  registerTaskContext(taskId: string, context: TaskContext): void {\n    this.taskContexts.set(taskId, context);\n  }\n\n  /**\n   * Remove per-task hooks. Called after task reaches terminal state.\n   */\n  deregisterTaskContext(taskId: string): void {\n    this.taskContexts.delete(taskId);\n  }\n\n  /**\n   * Register a tool executor by tool name. Used for cross-process dispatch:\n   * when a worker in a different process picks up a `task.dispatch` event,\n   * it has no per-task closure (`taskContexts`) for that taskId, but it can\n   * resolve the executor by tool name via this registry.\n   */\n  registerStaticExecutor(toolName: string, executor: ToolExecutor): void {\n    if (this.staticExecutors.has(toolName)) {\n      this.#mastra?.getLogger?.()?.debug?.(`Overwriting existing static executor for tool \"${toolName}\"`);\n    }\n    this.staticExecutors.set(toolName, executor);\n  }\n\n  /**\n   * Symmetric to `registerStaticExecutor`. Called when a tool is removed\n   * from `Mastra`.\n   */\n  unregisterStaticExecutor(toolName: string): void {\n    this.staticExecutors.delete(toolName);\n  }\n\n  /**\n   * Look up an executor by tool name. Read by the workflow-step body in\n   * `workflow.ts:runAttemptStep` as a fallback when no per-task `TaskContext`\n   * is registered (cross-process path).\n   */\n  getStaticExecutor(toolName: string): ToolExecutor | undefined {\n    return this.staticExecutors.get(toolName);\n  }\n\n  // --- Core operations ---\n\n  /**\n   * Enqueue a task for background execution.\n   * Prefer `createBackgroundTask()` which returns a self-contained handle.\n   */\n  async enqueue(payload: TaskPayload, context?: TaskContext): Promise<EnqueueResult> {\n    if (this.shuttingDown) {\n      throw new Error('BackgroundTaskManager is shutting down, cannot enqueue new tasks');\n    }\n\n    // Mastra fires `init` as fire-and-forget. If a caller hits enqueue\n    // before init completes, the dispatch publish fires before the worker\n    // subscribes and the event is dropped (or, worse, lands on a worker\n    // whose Mastra hasn't yet registered the bg-task workflow → \"Workflow\n    // with id __background-task not found\"). Await readiness up front.\n    if (this.initPromise) await this.initPromise;\n\n    const task: BackgroundTask = {\n      id: this.#mastra?.generateId() ?? randomUUID(),\n      status: 'pending',\n      toolName: payload.toolName,\n      toolCallId: payload.toolCallId,\n      args: payload.args,\n      agentId: payload.agentId,\n      threadId: payload.threadId,\n      resourceId: payload.resourceId,\n      runId: payload.runId,\n      retryCount: 0,\n      maxRetries: payload.maxRetries ?? this.config.defaultRetries?.maxRetries ?? 0,\n      timeoutMs: payload.timeoutMs ?? this.config.defaultTimeoutMs,\n      createdAt: new Date(),\n    };\n\n    // Register per-task context if provided\n    if (context) {\n      this.registerTaskContext(task.id, context);\n    }\n\n    const storage = await this.getStorage();\n    await storage.createTask(task);\n\n    const canRun = await this.checkConcurrency(task.agentId);\n\n    if (canRun) {\n      await this.dispatch(task);\n      return { task };\n    }\n\n    // Backpressure\n    switch (this.config.backpressure) {\n      case 'reject':\n        this.deregisterTaskContext(task.id);\n        await storage.deleteTask(task.id);\n        throw new Error(`Concurrency limit reached, cannot enqueue task for tool \"${task.toolName}\"`);\n\n      case 'fallback-sync':\n        this.deregisterTaskContext(task.id);\n        await storage.deleteTask(task.id);\n        return { task, fallbackToSync: true };\n\n      case 'queue':\n      default:\n        // Task stays pending in storage, will be dispatched when a slot opens\n        return { task };\n    }\n  }\n\n  async cancel(taskId: string): Promise<void> {\n    if (this.initPromise) await this.initPromise;\n    const storage = await this.getStorage();\n    const task = await storage.getTask(taskId);\n    if (!task) {\n      throw new Error(`Task not found: ${taskId}`);\n    }\n\n    if (\n      task.status === 'completed' ||\n      task.status === 'failed' ||\n      task.status === 'cancelled' ||\n      task.status === 'timed_out'\n    ) {\n      return; // no-op for terminal states\n    }\n\n    if (task.status === 'pending') {\n      await storage.updateTask(taskId, { status: 'cancelled', completedAt: new Date() });\n      const cancelledTask = await storage.getTask(taskId);\n      if (cancelledTask) await this.publishLifecycleEvent('task.cancelled', cancelledTask);\n      this.deregisterTaskContext(taskId);\n      return;\n    }\n\n    if (task.status === 'suspended') {\n      // No active executor or AbortController to tear down — the task is\n      // sitting on a workflow snapshot. Flip storage, publish, and tell the\n      // workflow run to cancel so the snapshot is cleaned up too.\n      await storage.updateTask(taskId, { status: 'cancelled', completedAt: new Date() });\n      if (this.#mastra) {\n        try {\n          const workflow = this.#mastra.__getInternalWorkflow(BACKGROUND_TASK_WORKFLOW_ID);\n          const wrapper = await workflow.createRun({ runId: taskId });\n          await wrapper.cancel();\n        } catch (err) {\n          this.#mastra?.getLogger?.()?.warn(`background-task workflow cancel failed for ${taskId}:`, err as any);\n        }\n      }\n      const cancelledTask = await storage.getTask(taskId);\n      if (cancelledTask) await this.publishLifecycleEvent('task.cancelled', cancelledTask);\n      this.deregisterTaskContext(taskId);\n      return;\n    }\n\n    if (task.status === 'running') {\n      await storage.updateTask(taskId, { status: 'cancelled', completedAt: new Date() });\n\n      // Abort the running tool\n      const controller = this.activeAbortControllers.get(taskId);\n      if (controller) {\n        controller.abort(new Error('Task cancelled'));\n        this.activeAbortControllers.delete(taskId);\n      }\n\n      // Also cancel the workflow run so workflow storage reflects the\n      // cancellation (run status flips to 'canceled' and the workflow's\n      // abortSignal fires — redundant with the local AbortController above\n      // but keeps run history clean and propagates cross-process via the\n      // workflow.cancel pubsub event).\n      if (this.#mastra) {\n        try {\n          const workflow = this.#mastra.__getInternalWorkflow(BACKGROUND_TASK_WORKFLOW_ID);\n          const wrapper = await workflow.createRun({ runId: taskId });\n          await wrapper.cancel();\n        } catch (err) {\n          this.#mastra?.getLogger?.()?.warn(`background-task workflow cancel failed for ${taskId}:`, err as any);\n        }\n      }\n\n      const cancelledTask = await storage.getTask(taskId);\n      if (cancelledTask) await this.publishLifecycleEvent('task.cancelled', cancelledTask);\n      this.deregisterTaskContext(taskId);\n\n      // Also publish cancel on dispatch topic for distributed worker abort\n      await this.pubsub.publish(TOPIC_DISPATCH, {\n        type: 'task.cancel',\n        data: { taskId },\n        runId: taskId,\n      });\n    }\n  }\n\n  /**\n   * Resume a suspended task. The tool executor must be re-registered via\n   * `registerTaskContext(taskId, ...)` before calling this if the original\n   * registration is gone (e.g. process restart) — the manager doesn't\n   * rehydrate executor closures from storage.\n   *\n   * `resumeData` is forwarded to the tool's `execute` options on the\n   * resumed run.\n   */\n  async resume(taskId: string, resumeData?: unknown): Promise<BackgroundTask> {\n    if (!this.#mastra) {\n      throw new Error('Mastra is not registered with this manager');\n    }\n\n    if (this.initPromise) await this.initPromise;\n\n    const storage = await this.getStorage();\n    const task = await storage.getTask(taskId);\n    if (!task) {\n      throw new Error(`Task not found: ${taskId}`);\n    }\n    if (task.status !== 'suspended') {\n      throw new Error(`Cannot resume task in status '${task.status}' (expected 'suspended')`);\n    }\n\n    const canRun = await this.checkConcurrency(task.agentId);\n    if (!canRun) {\n      // Resume sits outside the queue/fallback-sync paths — there's no\n      // synchronous caller to fall back to, and silently leaving the task\n      // suspended hides the failure from the caller. Throw and let the\n      // caller retry once a slot frees.\n      throw new Error(`Concurrency limit reached, cannot resume task \"${taskId}\" — retry once a slot is available`);\n    }\n\n    // Resume publishes directly (not via dispatch()), so it needs its own\n    // lazy worker start for the library-mode process-restart case.\n    await this.#ensureExecutionWorkersStarted();\n\n    // Hand off to the worker subscriber. `task.resume` rides the same\n    // `TOPIC_DISPATCH` + `WORKER_GROUP` exactly-once channel as\n    // `task.dispatch`, so any worker (including a different process from\n    // the one that suspended the task) can pick it up.\n    await this.pubsub.publish(TOPIC_DISPATCH, {\n      type: 'task.resume',\n      data: { taskId, resumeData },\n      runId: taskId,\n    });\n\n    return task;\n  }\n\n  /**\n   * Restarts a previously running task. The tool executor is re-registered via\n   * `registerTaskContext(taskId, ...)` because the original\n   * registration is gone (e.g. process restart) — the manager doesn't\n   * rehydrate executor closures from storage.\n   *\n   */\n  async restart(taskId: string, context?: TaskContext): Promise<BackgroundTask> {\n    if (!this.#mastra) {\n      throw new Error('Mastra is not registered with this manager');\n    }\n\n    if (this.initPromise) await this.initPromise;\n\n    const storage = await this.getStorage();\n    const task = await storage.getTask(taskId);\n    if (!task) {\n      throw new Error(`Task not found: ${taskId}`);\n    }\n    if (task.status !== 'running') {\n      throw new Error(`Cannot restart task in status '${task.status}' (expected 'running')`);\n    }\n\n    if (context) {\n      this.registerTaskContext(task.id, context);\n    }\n\n    const canRun = await this.checkConcurrency(task.agentId);\n    if (!canRun) {\n      // Restart sits outside the queue/fallback-sync paths — there's no\n      // synchronous caller to fall back to, and silently leaving the task\n      // running hides the failure from the caller. Throw and let the\n      // caller retry once a slot frees.\n      throw new Error(`Concurrency limit reached, cannot restart task \"${taskId}\" — retry once a slot is available`);\n    }\n\n    await this.dispatch(task, true);\n\n    return task;\n  }\n\n  async getTask(taskId: string): Promise<BackgroundTask | null> {\n    const storage = await this.getStorage();\n    return storage.getTask(taskId);\n  }\n\n  async listTasks(filter: TaskFilter = {}): Promise<TaskListResult> {\n    const storage = await this.getStorage();\n    return storage.listTasks(filter);\n  }\n\n  /**\n   * Deletes old completed/failed/cancelled/timed_out task records from storage.\n   */\n  async cleanup(): Promise<void> {\n    const completedTtlMs = this.config.cleanup?.completedTtlMs ?? 3_600_000;\n    const failedTtlMs = this.config.cleanup?.failedTtlMs ?? 86_400_000;\n    const now = Date.now();\n\n    const storage = await this.getStorage();\n    await storage.deleteTasks({\n      status: ['completed'],\n      toDate: new Date(now - completedTtlMs),\n      dateFilterBy: 'completedAt',\n    });\n\n    await storage.deleteTasks({\n      status: ['failed', 'cancelled', 'timed_out'],\n      toDate: new Date(now - failedTtlMs),\n      dateFilterBy: 'completedAt',\n    });\n  }\n\n  /**\n   * Returns a promise that resolves when the next task from the given set\n   * reaches a terminal state.\n   */\n  async waitForNextTask(\n    taskIds: string[],\n    options?: {\n      timeoutMs?: number;\n      onProgress?: (elapsedMs: number) => void;\n      progressIntervalMs?: number;\n    },\n  ): Promise<BackgroundTask> {\n    const storage = await this.getStorage();\n\n    const isTerminal = (status: string) =>\n      status === 'completed' || status === 'failed' || status === 'cancelled' || status === 'timed_out';\n\n    for (const id of taskIds) {\n      const task = await storage.getTask(id);\n      if (task && isTerminal(task.status)) {\n        return task;\n      }\n    }\n\n    return new Promise((resolve, reject) => {\n      const startTime = Date.now();\n\n      const timeout = options?.timeoutMs\n        ? setTimeout(() => {\n            clearInterval(pollInterval);\n            if (progressInterval) clearInterval(progressInterval);\n            reject(new Error('Timed out waiting for background task'));\n          }, options.timeoutMs)\n        : undefined;\n\n      const progressInterval = options?.onProgress\n        ? setInterval(() => {\n            options.onProgress!(Date.now() - startTime);\n          }, options.progressIntervalMs ?? 3000)\n        : undefined;\n\n      const pollInterval = setInterval(async () => {\n        for (const id of taskIds) {\n          const task = await storage.getTask(id);\n          if (task && isTerminal(task.status)) {\n            clearInterval(pollInterval);\n            if (timeout) clearTimeout(timeout);\n            if (progressInterval) clearInterval(progressInterval);\n            resolve(task);\n            return;\n          }\n        }\n      }, 50);\n    });\n  }\n\n  /**\n   * Returns a ReadableStream of all background task lifecycle events,\n   * filtered by optional criteria. Intended to be piped directly to an SSE response.\n   *\n   * On connection, emits the current state of all non-terminal tasks as a snapshot,\n   * then subscribes to live pubsub events for subsequent updates.\n   *\n   * Events include:\n   * - `task.running` (status: 'running') — task picked up by a worker\n   * - `task.completed` (status: 'completed') — task finished successfully\n   * - `task.failed` (status: 'failed' or 'timed_out') — task errored or timed out\n   * - `task.cancelled` (status: 'cancelled') — task was cancelled\n   * - `task.suspended` (status: 'suspended') — task paused via `suspend()` from\n   *   inside its tool executor; resume with `manager.resume(taskId, data)`\n   * - `task.resumed` (status: 'running') — suspended task resumed\n   *\n   * The stream stays open until the caller's AbortSignal fires (client disconnect).\n   */\n  stream(options?: {\n    agentId?: string;\n    runId?: string;\n    threadId?: string;\n    resourceId?: string;\n    taskId?: string;\n    abortSignal?: AbortSignal;\n  }): ReadableStream<Record<string, unknown>> {\n    const manager = this;\n    const pubsub = this.pubsub;\n    const { agentId, runId, threadId, resourceId, abortSignal, taskId } = options ?? {};\n\n    const EVENT_STATUS_MAP: Record<string, BackgroundTaskStatus> = {\n      'task.running': 'running',\n      'task.output': 'running',\n      'task.completed': 'completed',\n      'task.failed': 'failed',\n      'task.cancelled': 'cancelled',\n      'task.suspended': 'suspended',\n      'task.resumed': 'running',\n    };\n\n    const CHUNK_EVENT_MAP: Record<string, string> = {\n      'task.running': 'background-task-running',\n      'task.output': 'background-task-output',\n      'task.completed': 'background-task-completed',\n      'task.failed': 'background-task-failed',\n      'task.cancelled': 'background-task-cancelled',\n      'task.suspended': 'background-task-suspended',\n      'task.resumed': 'background-task-resumed',\n    };\n\n    return new ReadableStream({\n      async start(controller) {\n        // 1. Subscribe to live events first (so we don't miss anything between snapshot and subscribe)\n        const handler = async (event: Event) => {\n          const status = EVENT_STATUS_MAP[event.type];\n          if (!status) return;\n\n          const data = event.data;\n          if (agentId && data.agentId !== agentId) return;\n          if (runId && data.runId !== runId) return;\n          if (threadId && data.threadId !== threadId) return;\n          if (resourceId && data.resourceId !== resourceId) return;\n          if (taskId && data.taskId !== taskId) return;\n\n          const payload: Record<string, unknown> = {\n            taskId: data.taskId,\n            toolName: data.toolName,\n            toolCallId: data.toolCallId,\n            agentId: data.agentId,\n            runId: data.runId,\n          };\n\n          switch (event.type) {\n            case 'task.running':\n              payload.startedAt = data.startedAt;\n              payload.args = data.args;\n              break;\n            case 'task.completed':\n              payload.completedAt = data.completedAt;\n              payload.result = data.result;\n              break;\n            case 'task.failed':\n              payload.completedAt = data.completedAt;\n              payload.error = data.error;\n              break;\n            case 'task.cancelled':\n              payload.completedAt = data.completedAt;\n              break;\n            case 'task.output':\n              payload.payload = data.chunk;\n              break;\n            case 'task.suspended':\n              payload.suspendPayload = data.suspendPayload;\n              payload.suspendedAt = data.suspendedAt;\n              payload.args = data.args;\n              break;\n            case 'task.resumed':\n              payload.startedAt = data.startedAt;\n              payload.args = data.args;\n              break;\n          }\n\n          try {\n            controller.enqueue({\n              type: CHUNK_EVENT_MAP[event.type],\n              payload,\n            });\n          } catch {\n            // Controller closed\n          }\n        };\n\n        void pubsub.subscribe(TOPIC_RESULT, handler);\n\n        abortSignal?.addEventListener('abort', () => {\n          void pubsub.unsubscribe(TOPIC_RESULT, handler);\n          try {\n            controller.close();\n          } catch {\n            // Already closed\n          }\n        });\n\n        // 2. Emit snapshot of existing in-flight tasks (running + suspended).\n        try {\n          const storage = await manager.getStorage();\n          if (taskId) {\n            const task = await storage.getTask(taskId);\n            if (task && task.status === 'running') {\n              controller.enqueue({\n                type: 'background-task-running',\n                payload: {\n                  taskId: task.id,\n                  toolName: task.toolName,\n                  toolCallId: task.toolCallId,\n                  agentId: task.agentId,\n                  runId: task.runId,\n                  startedAt: task.startedAt,\n                  args: task.args,\n                },\n              });\n            }\n          } else {\n            const { tasks: existing } = await storage.listTasks({\n              agentId,\n              runId,\n              threadId,\n              resourceId,\n              status: ['running'],\n            });\n\n            for (const task of existing) {\n              if (abortSignal?.aborted) break;\n              try {\n                controller.enqueue({\n                  type: 'background-task-running',\n                  payload: {\n                    taskId: task.id,\n                    toolName: task.toolName,\n                    toolCallId: task.toolCallId,\n                    agentId: task.agentId,\n                    runId: task.runId,\n                    startedAt: task.startedAt,\n                    args: task.args,\n                  },\n                });\n              } catch {\n                break;\n              }\n            }\n          }\n        } catch {\n          // Storage not available — continue with live events only\n        }\n      },\n    });\n  }\n\n  async shutdown(): Promise<void> {\n    this.shuttingDown = true;\n\n    if (this.cleanupInterval) {\n      clearInterval(this.cleanupInterval);\n      this.cleanupInterval = undefined;\n    }\n\n    if (this.workerCallback) {\n      await this.pubsub.unsubscribe(TOPIC_DISPATCH, this.workerCallback);\n    }\n    if (this.resultCallback) {\n      await this.pubsub.unsubscribe(TOPIC_RESULT, this.resultCallback);\n    }\n\n    this.taskContexts.clear();\n    await this.pubsub.flush();\n  }\n\n  // --- Internal ---\n\n  /**\n   * Lazily start Mastra's execution workers before publishing. In \"library\n   * mode\" nothing ever calls `mastra.startWorkers()`, so the evented\n   * `__background-task` workflow started by `handleDispatch` would publish\n   * to the `workflows` topic with no consumer and the task would sit at\n   * `running` forever (#19339). A no-op once workers are running; honors the\n   * `workers: false` / `MASTRA_WORKERS` opt-outs.\n   *\n   * Startup failures are logged but don't abort the publish — in distributed\n   * topologies a remote worker on the shared broker can still pick the task\n   * up, and throwing here would also abort `drainPending()` /\n   * `recoverStaleTasks()` loops.\n   */\n  async #ensureExecutionWorkersStarted(): Promise<void> {\n    if (!this.#mastra) return;\n    try {\n      await this.#mastra.__ensureExecutionWorkersStarted();\n    } catch (err) {\n      this.#mastra.getLogger?.()?.error('Failed to start execution workers for background task', err as any);\n    }\n  }\n\n  private async dispatch(task: BackgroundTask, isRestart?: boolean): Promise<void> {\n    await this.#ensureExecutionWorkersStarted();\n\n    // Publish `task.dispatch` on `TOPIC_DISPATCH` with `WORKER_GROUP`, so\n    // exactly one worker handles the task. `handleDispatch` flips the\n    // task to running and starts the per-task workflow run.\n    await this.pubsub.publish(TOPIC_DISPATCH, {\n      type: 'task.dispatch',\n      data: {\n        taskId: task.id,\n        toolName: task.toolName,\n        toolCallId: task.toolCallId,\n        args: task.args,\n        agentId: task.agentId,\n        threadId: task.threadId,\n        resourceId: task.resourceId,\n        timeoutMs: task.timeoutMs,\n        maxRetries: task.maxRetries,\n        runId: task.runId,\n        isRestart,\n      },\n      runId: task.id,\n    });\n  }\n\n  /**\n   * Handles a task.dispatch and task.restart events.\n   * Both events are similar, but the latter is used to restart a running task.\n   */\n  private async handleDispatch(event: Event): Promise<void> {\n    const { taskId, isRestart } = event.data;\n    const deliveryAttempt = event.deliveryAttempt ?? 1;\n\n    const storage = await this.getStorage();\n    const task = await storage.getTask(taskId);\n    if (!task || task.status === 'cancelled') {\n      this.deregisterTaskContext(taskId);\n      return;\n    }\n\n    if (isRestart && task.status !== 'running') {\n      // Either gone or already done/cancelled by another worker. Drop the\n      // event silently — the worker group ensures exactly-once delivery, but\n      // the task may have moved on between publish and pickup.\n      return;\n    }\n\n    await storage.updateTask(taskId, { status: 'running', startedAt: new Date(), retryCount: deliveryAttempt - 1 });\n\n    // Publish running lifecycle event (fan-out, for stream consumers)\n    const runningTask = await storage.getTask(taskId);\n    if (runningTask) await this.publishLifecycleEvent('task.running', runningTask);\n\n    // Fire-and-forget the workflow run; the workflow step body owns\n    // executor invocation, retries, and suspend/resume. The local\n    // execution hook still runs here so callers see `onExecution` fire.\n    if (this.#mastra) {\n      if (runningTask) void this.runLocalExecutionHook(runningTask);\n      const workflow = this.#mastra.__getInternalWorkflow(BACKGROUND_TASK_WORKFLOW_ID);\n      const prevWorkflowRun = isRestart ? await workflow.getWorkflowRunById(taskId) : undefined;\n      const shouldRestart = isRestart && prevWorkflowRun?.status === 'running';\n      const run = await workflow.createRun({ runId: taskId });\n      const runPromise = shouldRestart ? run.restart() : run.start({ inputData: { taskId } });\n      void runPromise\n        .then(result => {\n          if (result.status !== 'suspended') {\n            void workflow.deleteWorkflowRunById(taskId);\n          }\n        })\n        .catch(err => {\n          this.#mastra\n            ?.getLogger?.()\n            ?.error(`background-task workflow ${shouldRestart ? 'restart' : 'start'} failed for ${taskId}:`, err);\n        })\n        .finally(() => {\n          // Free the concurrency slot once the run terminates.\n          void this.drainPending();\n        });\n    }\n  }\n\n  /**\n   * Handles a task.resume event. Mirrors the workflow branch of handleDispatch\n   * but resumes an existing run from its suspended snapshot instead of starting\n   * a fresh one. Concurrency gating, suspended-status validation, and the\n   * `task.resumed` lifecycle publish all happen here so a different process\n   * than the one that suspended the task can drive the resume.\n   */\n  private async handleResume(event: Event): Promise<void> {\n    const { taskId, resumeData } = event.data;\n\n    const storage = await this.getStorage();\n    const task = await storage.getTask(taskId);\n    if (!task || task.status !== 'suspended') {\n      // Either gone or already resumed/cancelled by another worker. Drop the\n      // event silently — the worker group ensures exactly-once delivery, but\n      // the task may have moved on between publish and pickup.\n      return;\n    }\n\n    await storage.updateTask(taskId, {\n      status: 'running',\n      startedAt: new Date(),\n      suspendPayload: undefined,\n      suspendedAt: undefined,\n    });\n    const resumedTask = await storage.getTask(taskId);\n    if (resumedTask) {\n      await this.publishLifecycleEvent('task.resumed', resumedTask);\n    }\n\n    if (!this.#mastra) return;\n    const workflow = this.#mastra.__getInternalWorkflow(BACKGROUND_TASK_WORKFLOW_ID);\n    // `createRun({ runId })` reattaches to the existing snapshot when given a\n    // stable runId — we don't want a fresh run.\n    const run = await workflow.createRun({ runId: taskId });\n    void run\n      .resume({ resumeData })\n      .then(result => {\n        if (result.status !== 'suspended') {\n          void workflow.deleteWorkflowRunById(taskId);\n        }\n      })\n      .catch(err => {\n        this.#mastra?.getLogger?.()?.error(`background-task workflow resume failed for ${taskId}:`, err);\n      })\n      .finally(() => {\n        // Mirror dispatch's drain — resuming frees a slot when it terminates.\n        void this.drainPending();\n      });\n  }\n\n  /**\n   * Run per-task hooks (onChunk, onResult, onComplete/onFailed) locally in the\n   * worker path, before publishing the terminal lifecycle event. Ensures\n   * memory / stream state is consistent by the time any pubsub subscriber is\n   * notified. After running, the task context is deregistered so\n   * `handleResult` (which also fires from pubsub) becomes a no-op for this\n   * task in the same process.\n   *\n   * In distributed deployments where the worker runs in a different process\n   * from the dispatcher, `this.taskContexts` won't contain an entry for\n   * `task.id` — this method is a no-op there, and `handleResult` in the\n   * dispatching process runs the hooks instead.\n   */\n  /**\n   * Terminal-state hooks only. Called when a task reaches `'completed'` or\n   * `'failed'`. Suspend is non-terminal — see `runLocalSuspendHooks` for that\n   * path.\n   *\n   * @internal — also called by the workflow-engine step bodies in workflow.ts\n   */\n  async runLocalCompletionHooks(\n    task: BackgroundTask,\n    status: 'completed' | 'failed',\n    extras: { result?: unknown; error?: { message: string; stack?: string } },\n  ): Promise<void> {\n    const ctx = this.taskContexts.get(task.id);\n    if (!ctx) return;\n\n    try {\n      if (status === 'completed') {\n        ctx.onChunk?.({\n          type: 'background-task-completed',\n          payload: {\n            taskId: task.id,\n            toolName: task.toolName,\n            toolCallId: task.toolCallId,\n            runId: task.runId,\n            result: extras.result,\n            completedAt: task.completedAt!,\n            agentId: task.agentId,\n          },\n        });\n\n        await ctx.onResult?.({\n          runId: task.runId,\n          taskId: task.id,\n          toolCallId: task.toolCallId,\n          toolName: task.toolName,\n          agentId: task.agentId,\n          threadId: task.threadId,\n          resourceId: task.resourceId,\n          result: extras.result,\n          status: 'completed',\n          completedAt: task.completedAt!,\n          startedAt: task.startedAt!,\n        });\n\n        // Globals (this.config.onTaskComplete / onTaskFailed) fire from\n        // handleResult via pubsub so they run once per subscribing process\n        // — in distributed deployments that's the dispatching process, which\n        // is where observers/metrics are typically wired.\n        await ctx.onComplete?.(task);\n      } else {\n        ctx.onChunk?.({\n          type: 'background-task-failed',\n          payload: {\n            taskId: task.id,\n            toolName: task.toolName,\n            toolCallId: task.toolCallId,\n            runId: task.runId,\n            error: extras.error ?? { message: 'Unknown error' },\n            completedAt: task.completedAt!,\n            agentId: task.agentId,\n          },\n        });\n\n        await ctx.onResult?.({\n          runId: task.runId,\n          taskId: task.id,\n          toolCallId: task.toolCallId,\n          toolName: task.toolName,\n          agentId: task.agentId,\n          threadId: task.threadId,\n          resourceId: task.resourceId,\n          error: extras.error,\n          status: 'failed',\n          completedAt: task.completedAt!,\n          startedAt: task.startedAt!,\n        });\n\n        // See comment above — globals are handled exclusively by\n        // handleResult so they fire once per subscribing process.\n        await ctx.onFailed?.(task);\n      }\n    } finally {\n      this.deregisterTaskContext(task.id);\n    }\n  }\n\n  /**\n   * Per-task suspend hooks. Fires `ctx.onResult({ status: 'suspended', ... })`\n   * so the message list / memory pick up the suspension as the tool's\n   * current invocation state. Does NOT deregister the task context — resume\n   * needs the executor closure intact.\n   *\n   * @internal — called by the workflow-engine step bodies in workflow.ts\n   */\n  async runLocalSuspendHooks(task: BackgroundTask): Promise<void> {\n    const ctx = this.taskContexts.get(task.id);\n    if (!ctx) return;\n    await ctx.onExecution?.({\n      runId: task.runId,\n      taskId: task.id,\n      toolCallId: task.toolCallId,\n      toolName: task.toolName,\n      agentId: task.agentId,\n      threadId: task.threadId,\n      resourceId: task.resourceId,\n      startedAt: task.startedAt!,\n      suspendedAt: task.suspendedAt,\n    });\n  }\n\n  /** @internal — also called by the workflow-engine step bodies in workflow.ts */\n  async runLocalExecutionHook(task: BackgroundTask): Promise<void> {\n    const ctx = this.taskContexts.get(task.id);\n    if (!ctx) return;\n\n    try {\n      await ctx.onExecution?.({\n        runId: task.runId,\n        taskId: task.id,\n        toolCallId: task.toolCallId,\n        toolName: task.toolName,\n        agentId: task.agentId,\n        threadId: task.threadId,\n        resourceId: task.resourceId,\n        startedAt: task.startedAt!,\n      });\n    } catch {\n      //fail silently\n    }\n  }\n\n  private async handleResult(event: Event): Promise<void> {\n    const { taskId, toolName, toolCallId, threadId, resourceId, runId } = event.data;\n    const storage = await this.getStorage();\n    const task = await storage.getTask(taskId);\n\n    if (task?.completedAt) {\n      // Look up per-task hooks\n      const ctx = this.taskContexts.get(taskId);\n\n      if (event.type === 'task.completed') {\n        ctx?.onChunk?.({\n          type: 'background-task-completed',\n          payload: {\n            taskId,\n            toolName,\n            toolCallId,\n            runId,\n            result: event.data.result,\n            completedAt: task.completedAt,\n            agentId: task.agentId,\n          },\n        });\n\n        await ctx?.onResult?.({\n          runId,\n          taskId,\n          toolCallId,\n          toolName,\n          agentId: event.data.agentId,\n          threadId,\n          resourceId,\n          result: event.data.result,\n          status: 'completed',\n          completedAt: task.completedAt,\n          startedAt: task.startedAt!,\n        });\n\n        if (task) {\n          await Promise.all([ctx?.onComplete?.(task), this.config.onTaskComplete?.(task)]);\n        }\n      }\n\n      if (event.type === 'task.failed') {\n        ctx?.onChunk?.({\n          type: 'background-task-failed',\n          payload: {\n            taskId,\n            toolName,\n            toolCallId,\n            runId,\n            error: event.data.error,\n            completedAt: task.completedAt,\n            agentId: task.agentId,\n          },\n        });\n\n        await ctx?.onResult?.({\n          runId,\n          taskId,\n          toolCallId,\n          toolName,\n          agentId: event.data.agentId,\n          threadId,\n          resourceId,\n          error: event.data.error,\n          status: 'failed',\n          completedAt: task.completedAt,\n          startedAt: task.startedAt!,\n        });\n\n        if (task) {\n          await Promise.all([ctx?.onFailed?.(task), this.config.onTaskFailed?.(task)]);\n        }\n      }\n\n      // Clean up context after terminal result\n      this.deregisterTaskContext(taskId);\n    }\n  }\n\n  private handleCancel(event: Event): void {\n    const { taskId } = event.data;\n    const controller = this.activeAbortControllers.get(taskId);\n    if (controller) {\n      controller.abort(new Error('Task cancelled'));\n      this.activeAbortControllers.delete(taskId);\n    }\n    this.deregisterTaskContext(taskId);\n  }\n\n  /** @internal — also called by the workflow-engine step bodies in workflow.ts */\n  async publishLifecycleEvent(\n    type:\n      | 'task.running'\n      | 'task.completed'\n      | 'task.failed'\n      | 'task.cancelled'\n      | 'task.output'\n      | 'task.suspended'\n      | 'task.resumed',\n    task: BackgroundTaskEvent,\n  ): Promise<void> {\n    await this.pubsub.publish(TOPIC_RESULT, {\n      type,\n      data: {\n        taskId: task.id,\n        toolName: task.toolName,\n        toolCallId: task.toolCallId,\n        runId: task.runId,\n        agentId: task.agentId,\n        threadId: task.threadId,\n        resourceId: task.resourceId,\n        args: task.args,\n        result: task.result,\n        error: task.error,\n        chunk: task.chunk,\n        completedAt: task.completedAt,\n        startedAt: task.startedAt,\n        suspendPayload: task.suspendPayload,\n        suspendedAt: task.suspendedAt,\n      },\n      runId: task.id,\n    });\n  }\n\n  private async checkConcurrency(agentId: string): Promise<boolean> {\n    const storage = await this.getStorage();\n    const globalRunning = await storage.getRunningCount();\n    if (globalRunning >= this.config.globalConcurrency) {\n      return false;\n    }\n\n    const agentRunning = await storage.getRunningCountByAgent(agentId);\n    if (agentRunning >= this.config.perAgentConcurrency) {\n      return false;\n    }\n\n    return true;\n  }\n\n  private async drainPending(): Promise<void> {\n    const storage = await this.getStorage();\n    const { tasks: pending } = await storage.listTasks({\n      status: 'pending',\n      orderBy: 'createdAt',\n      orderDirection: 'asc',\n    });\n\n    for (const task of pending) {\n      if (await this.checkConcurrency(task.agentId)) {\n        await this.dispatch(task);\n      }\n    }\n  }\n\n  /**\n   * Recovers tasks left in 'running' or 'pending' state from a previous process.\n   */\n  private async recoverStaleTasks(): Promise<void> {\n    try {\n      const storage = await this.getStorage();\n      const { tasks: staleTasks } = await storage.listTasks({ status: 'running' });\n      for (const task of staleTasks) {\n        if (task.maxRetries > 0) {\n          await storage.updateTask(task.id, {\n            status: 'pending',\n            startedAt: undefined,\n          });\n        } else {\n          await storage.updateTask(task.id, {\n            status: 'failed',\n            error: { message: 'Worker process terminated before task completed' },\n            completedAt: new Date(),\n          });\n        }\n      }\n\n      const { tasks: pendingTasks } = await storage.listTasks({\n        status: 'pending',\n        orderBy: 'createdAt',\n        orderDirection: 'asc',\n      });\n      for (const task of pendingTasks) {\n        if (await this.checkConcurrency(task.agentId)) {\n          await this.dispatch(task);\n        }\n      }\n    } catch (error) {\n      const logger = this.#mastra?.getLogger();\n      if (logger) {\n        logger.error('Failed to recover stale background tasks', error);\n      }\n    }\n  }\n}\n","import type { BackgroundTaskManager } from './manager';\nimport type {\n  BackgroundTaskHandle,\n  CheckIfRunningPayload,\n  CheckIfSuspendedPayload,\n  CreateBackgroundTaskOptions,\n} from './types';\n\n/**\n * Creates a self-contained background task handle.\n *\n * Bundles the task payload with per-stream hooks (executor, onChunk, onResult)\n * so each dispatch is fully isolated — no shared mutable state on the manager.\n *\n * @example\n * ```ts\n * const bgTask = createBackgroundTask(manager, {\n *   toolName: 'research',\n *   toolCallId: 'call-1',\n *   args: { query: 'solana' },\n *   agentId: 'agent-1',\n *   runId: 'run-1',\n *   context: {\n *     executor: { execute: (args, opts) => tool.execute(args, opts) },\n *     onChunk: (chunk) => controller.enqueue(chunk),\n *     onResult: (params) => messageList.addToolResult(params),\n *   },\n * });\n *\n * const { task, fallbackToSync } = await bgTask.dispatch();\n * const completed = await bgTask.waitForCompletion();\n * await bgTask.cancel();\n * ```\n */\nexport function createBackgroundTask(\n  manager: BackgroundTaskManager,\n  options: CreateBackgroundTaskOptions,\n): BackgroundTaskHandle {\n  const { context, ...payload } = options;\n  let taskId: string | undefined;\n\n  return {\n    get task() {\n      if (!taskId) throw new Error('Task has not been dispatched yet');\n      // Synchronous access to task ID — full task data requires async getTask()\n      return { id: taskId } as any;\n    },\n\n    async dispatch() {\n      const result = await manager.enqueue(payload, context);\n      taskId = result.task.id;\n      return result;\n    },\n\n    async checkIfSuspended(args: CheckIfSuspendedPayload) {\n      const result = await manager.listTasks({\n        toolCallId: args.toolCallId,\n        runId: args.runId,\n        agentId: args.agentId,\n        threadId: args.threadId,\n        resourceId: args.resourceId,\n        toolName: args.toolName,\n        status: 'suspended',\n      });\n      if (result.total > 0) {\n        const task = result.tasks[0];\n        if (task) {\n          taskId = task.id;\n          return true;\n        }\n      }\n\n      return false;\n    },\n\n    async checkIfRunning(args: CheckIfRunningPayload) {\n      const result = await manager.listTasks({\n        toolCallId: args.toolCallId,\n        runId: args.runId,\n        agentId: args.agentId,\n        threadId: args.threadId,\n        resourceId: args.resourceId,\n        toolName: args.toolName,\n        status: 'running',\n      });\n      if (result.total > 0) {\n        const task = result.tasks[0];\n        if (task) {\n          taskId = task.id;\n          return true;\n        }\n      }\n\n      return false;\n    },\n\n    async resume(resumeData?: unknown) {\n      if (!taskId) throw new Error('Task has not been dispatched yet');\n      return manager.resume(taskId, resumeData);\n    },\n\n    async restart() {\n      if (!taskId) throw new Error('Task has not been dispatched yet');\n      return manager.restart(taskId, context);\n    },\n\n    async cancel() {\n      if (!taskId) throw new Error('Task has not been dispatched yet');\n      return manager.cancel(taskId);\n    },\n\n    async waitForCompletion(waitOptions) {\n      if (!taskId) throw new Error('Task has not been dispatched yet');\n      return manager.waitForNextTask([taskId], waitOptions);\n    },\n  };\n}\n","import type {\n  AgentBackgroundConfig,\n  AgentBackgroundToolConfig,\n  BackgroundTaskManagerConfig,\n  LLMBackgroundOverride,\n  ToolBackgroundConfig,\n} from './types';\n\nexport interface ResolvedBackgroundConfig {\n  runInBackground: boolean;\n  timeoutMs: number;\n  maxRetries: number;\n}\n\n/**\n * Resolves whether a tool call should run in the background, and with what config.\n *\n * Resolution order (highest to lowest priority):\n * 1. LLM per-call override (`_background` field in tool args)\n * 2. Agent-level backgroundTasks.tools config\n * 3. Tool-level background config\n * 4. Default: foreground\n *\n * Strips the `_background` field from args (mutates the args object).\n */\nexport function resolveBackgroundConfig({\n  llmBgOverrides,\n  toolName,\n  toolConfig,\n  agentConfig,\n  managerConfig,\n}: {\n  llmBgOverrides: Record<string, unknown>;\n  toolName: string;\n  toolConfig?: ToolBackgroundConfig;\n  agentConfig?: AgentBackgroundConfig;\n  managerConfig?: BackgroundTaskManagerConfig;\n}): ResolvedBackgroundConfig {\n  const llmOverride = llmBgOverrides as LLMBackgroundOverride | undefined;\n\n  // If this agent has background tasks disabled, short-c