inngest/components/execution/engine.cjs.map

Official SDK for Inngest.com. Inngest is the reliability layer for modern applications. Inngest combines durable execution, events, and queues into a zero-infra platform with built-in observability.

{"version":3,"file":"engine.cjs","names":["hashjs","createExecutionEngine: InngestExecutionFactory","InngestExecution","ExecutionVersion","options: InngestExecutionOptions","StepMode","MiddlewareManager","internalLoggerSymbol","trace","version","getAsyncLocalStorage","headerKeys","retryWithBackoff","defaultMaxRetries","commonCheckpointHandler: CheckpointHandlers[StepMode][\"\"]","StepOpCode","serializeError","syncHandlers: CheckpointHandlers[StepMode.Sync]","asyncHandlers: CheckpointHandlers[StepMode.Async]","asyncCheckpointingHandlers: CheckpointHandlers[StepMode.AsyncCheckpointing]","step","UnreachableError","outgoingOp: OutgoingOp","getAsyncCtx","interval: GoInterval | undefined","runAsPromise","createDeferredPromise","goIntervalTiming","innerHandler: () => Promise<unknown>","err: Error","NonRetriableError","StepError","RetryAfterError","validateEvents","undefinedToNull","createDeferredPromiseWithStack","loop: ExecutionState[\"loop\"]","experimentStepRunSymbol","createGroupTools","z","jsonErrorSchema","deserializeError","foundStepsToReport: Map<string, FoundStep>","unhandledFoundStepsToReport: Map<string, FoundStep>","expectedNextStepIndexes: Map<string, number>","remainingStepCompletionOrder: string[]","foundStepsReportPromise: Promise<void> | undefined","ErrCode","extensionPromise: Promise<void>","resolveNextTick","resolveAfterPending","stepHandler: StepHandler","getStepOptions","extraOpts: Record<string, unknown> | undefined","step: FoundStep","createStepTools","createTimeoutPromise","STEP_INDEXING_SUFFIX"],"sources":["../../../src/components/execution/engine.ts"],"sourcesContent":["import { trace } from \"@opentelemetry/api\";\nimport hashjs from \"hash.js\";\nimport ms, { type StringValue } from \"ms\";\nimport { z } from \"zod/v3\";\n\nimport {\n  defaultMaxRetries,\n  ExecutionVersion,\n  headerKeys,\n  internalEvents,\n} from \"../../helpers/consts.ts\";\n\nimport {\n  deserializeError,\n  ErrCode,\n  serializeError,\n} from \"../../helpers/errors.js\";\nimport { undefinedToNull } from \"../../helpers/functions.js\";\nimport {\n  createDeferredPromise,\n  createDeferredPromiseWithStack,\n  createTimeoutPromise,\n  type GoInterval,\n  goIntervalTiming,\n  resolveAfterPending,\n  resolveNextTick,\n  retryWithBackoff,\n  runAsPromise,\n} from \"../../helpers/promises.ts\";\nimport * as Temporal from \"../../helpers/temporal.ts\";\nimport type { MaybePromise, Simplify } from \"../../helpers/types.ts\";\nimport {\n  type APIStepPayload,\n  type Context,\n  type EventPayload,\n  type FailureEventArgs,\n  type Handler,\n  type HashedOp,\n  jsonErrorSchema,\n  type OutgoingOp,\n  StepMode,\n  StepOpCode,\n} from \"../../types.ts\";\nimport { version } from \"../../version.ts\";\nimport { internalLoggerSymbol } from \"../Inngest.ts\";\nimport { createGroupTools } from \"../InngestGroupTools.ts\";\nimport type {\n  MetadataKind,\n  MetadataOpcode,\n  MetadataScope,\n  MetadataUpdate,\n} from \"../InngestMetadata.ts\";\nimport {\n  createStepTools,\n  type ExperimentStepTools,\n  experimentStepRunSymbol,\n  type FoundStep,\n  getStepOptions,\n  STEP_INDEXING_SUFFIX,\n  type StepHandler,\n} from \"../InngestStepTools.ts\";\nimport { MiddlewareManager } from \"../middleware/index.ts\";\nimport type { Middleware } from \"../middleware/middleware.ts\";\nimport { UnreachableError } from \"../middleware/utils.ts\";\nimport { NonRetriableError } from \"../NonRetriableError.ts\";\nimport { RetryAfterError } from \"../RetryAfterError.ts\";\nimport { StepError } from \"../StepError.ts\";\nimport { validateEvents } from \"../triggers/utils.js\";\nimport { getAsyncCtx, getAsyncLocalStorage } from \"./als.ts\";\nimport {\n  type BasicFoundStep,\n  type ExecutionResult,\n  type IInngestExecution,\n  InngestExecution,\n  type InngestExecutionFactory,\n  type InngestExecutionOptions,\n  type MemoizedOp,\n} from \"./InngestExecution.ts\";\nimport { clientProcessorMap } from \"./otel/access.ts\";\n\nconst { sha1 } = hashjs;\n\n/**\n * Retry configuration for checkpoint operations.\n *\n * Checkpoint calls use exponential backoff with jitter to handle transient\n * network failures (e.g., dev server temporarily down, cloud hiccup). If\n * retries exhaust, the error propagates up - for Sync mode this results in a\n * 500 error, for AsyncCheckpointing the caller handles fallback.\n */\nconst CHECKPOINT_RETRY_OPTIONS = { maxAttempts: 5, baseDelay: 100 };\nconst STEP_NOT_FOUND_MAX_FOUND_STEPS = 25;\n\nexport const createExecutionEngine: InngestExecutionFactory = (options) => {\n  return new InngestExecutionEngine(options);\n};\n\nclass InngestExecutionEngine\n  extends InngestExecution\n  implements IInngestExecution\n{\n  public version = ExecutionVersion.V2;\n\n  private state: ExecutionState;\n  private fnArg: Context.Any;\n  private checkpointHandlers: CheckpointHandlers;\n  private timeoutDuration = 1000 * 10;\n  private execution: Promise<ExecutionResult> | undefined;\n  private userFnToRun: Handler.Any;\n  private middlewareManager: MiddlewareManager;\n\n  /**\n   * If we're supposed to run a particular step via `requestedRunStep`, this\n   * will be a `Promise` that resolves after no steps have been found for\n   * `timeoutDuration` milliseconds.\n   *\n   * If we're not supposed to run a particular step, this will be `undefined`.\n   */\n  private timeout?: ReturnType<typeof createTimeoutPromise>;\n  private rootSpanId?: string;\n\n  /**\n   * If we're checkpointing and have been given a maximum runtime, this will be\n   * a `Promise` that resolves after that duration has elapsed, allowing us to\n   * ensure that we end the execution in good time, especially in serverless\n   * environments.\n   */\n  private checkpointingMaxRuntimeTimer?: ReturnType<\n    typeof createTimeoutPromise\n  >;\n\n  /**\n   * If we're checkpointing and have been given a maximum buffer interval, this\n   * will be a `Promise` that resolves after that duration has elapsed, allowing\n   * us to periodically checkpoint even if the step buffer hasn't filled.\n   */\n  private checkpointingMaxBufferIntervalTimer?: ReturnType<\n    typeof createTimeoutPromise\n  >;\n\n  constructor(rawOptions: InngestExecutionOptions) {\n    const options: InngestExecutionOptions = {\n      ...rawOptions,\n      stepMode: rawOptions.stepMode ?? StepMode.Async,\n    };\n\n    super(options);\n\n    /**\n     * Check we have everything we need for checkpointing\n     */\n    if (this.options.stepMode === StepMode.Sync) {\n      if (!this.options.createResponse) {\n        throw new Error(\"createResponse is required for sync step mode\");\n      }\n    }\n\n    this.userFnToRun = this.getUserFnToRun();\n    this.state = this.createExecutionState();\n    this.fnArg = this.createFnArg();\n\n    // Setup middleware\n    const mwInstances =\n      this.options.middlewareInstances ??\n      (this.options.client.middleware || []).map((Cls) => {\n        return new Cls({ client: this.options.client });\n      });\n    this.middlewareManager = new MiddlewareManager(\n      this.fnArg,\n      () => this.state.stepState,\n      mwInstances,\n      this.options.fn,\n      this.options.client[internalLoggerSymbol],\n    );\n\n    this.checkpointHandlers = this.createCheckpointHandlers();\n    this.initializeTimer(this.state);\n    this.initializeCheckpointRuntimeTimer(this.state);\n\n    this.devDebug(\n      \"created new V1 execution for run;\",\n      this.options.requestedRunStep\n        ? `wanting to run step \"${this.options.requestedRunStep}\"`\n        : \"discovering steps\",\n    );\n\n    this.devDebug(\"existing state keys:\", Object.keys(this.state.stepState));\n  }\n\n  /**\n   * Idempotently start the execution of the user's function.\n   */\n  public start() {\n    if (!this.execution) {\n      this.devDebug(\"starting V1 execution\");\n\n      const tracer = trace.getTracer(\"inngest\", version);\n\n      this.execution = getAsyncLocalStorage().then((als) => {\n        return als.run(\n          {\n            app: this.options.client,\n            execution: {\n              ctx: this.fnArg,\n              instance: this,\n            },\n          },\n          async () => {\n            return tracer.startActiveSpan(\"inngest.execution\", (span) => {\n              this.rootSpanId = span.spanContext().spanId;\n              clientProcessorMap.get(this.options.client)?.declareStartingSpan({\n                span,\n                runId: this.options.runId,\n                traceparent: this.options.headers[headerKeys.TraceParent],\n                tracestate: this.options.headers[headerKeys.TraceState],\n              });\n\n              return this._start()\n                .then((result) => {\n                  this.devDebug(\"result:\", result);\n                  return result;\n                })\n                .finally(() => {\n                  span.end();\n                });\n            });\n          },\n        );\n      });\n    }\n\n    return this.execution;\n  }\n\n  public addMetadata(\n    stepId: string,\n    kind: MetadataKind,\n    scope: MetadataScope,\n    op: MetadataOpcode,\n    values: Record<string, unknown>,\n  ) {\n    if (!this.state.metadata) {\n      this.state.metadata = new Map();\n    }\n\n    const updates = this.state.metadata.get(stepId) ?? [];\n    updates.push({ kind, scope, op, values });\n    this.state.metadata.set(stepId, updates);\n\n    return true;\n  }\n\n  /**\n   * Starts execution of the user's function and the core loop.\n   */\n  private async _start(): Promise<ExecutionResult> {\n    try {\n      const allCheckpointHandler = this.getCheckpointHandler(\"\");\n      await this.startExecution();\n\n      let i = 0;\n\n      for await (const checkpoint of this.state.loop) {\n        await allCheckpointHandler(checkpoint, i);\n\n        const handler = this.getCheckpointHandler(checkpoint.type);\n        const result = await handler(checkpoint, i++);\n\n        if (result) {\n          return result;\n        }\n      }\n    } catch (error) {\n      return this.transformOutput({ error });\n    } finally {\n      void this.state.loop.return();\n    }\n\n    /**\n     * If we're here, the generator somehow finished without returning a value.\n     * This should never happen.\n     */\n    throw new Error(\"Core loop finished without returning a value\");\n  }\n\n  private async checkpoint(steps: OutgoingOp[]): Promise<void> {\n    if (this.options.stepMode === StepMode.Sync) {\n      if (!this.state.checkpointedRun) {\n        // We have to start the run\n        const res = await retryWithBackoff(\n          () =>\n            this.options.client[\"inngestApi\"].checkpointNewRun({\n              runId: this.fnArg.runId,\n              event: this.fnArg.event as APIStepPayload,\n              steps,\n              executionVersion: this.version,\n              retries: this.fnArg.maxAttempts ?? defaultMaxRetries,\n            }),\n          CHECKPOINT_RETRY_OPTIONS,\n        );\n\n        this.state.checkpointedRun = {\n          appId: res.data.app_id,\n          fnId: res.data.fn_id,\n          token: res.data.token,\n        };\n      } else {\n        await retryWithBackoff(\n          () =>\n            this.options.client[\"inngestApi\"].checkpointSteps({\n              appId: this.state.checkpointedRun!.appId,\n              fnId: this.state.checkpointedRun!.fnId,\n              runId: this.fnArg.runId,\n              steps,\n            }),\n          CHECKPOINT_RETRY_OPTIONS,\n        );\n      }\n    } else if (this.options.stepMode === StepMode.AsyncCheckpointing) {\n      if (!this.options.queueItemId) {\n        throw new Error(\n          \"Missing queueItemId for async checkpointing. This is a bug in the Inngest SDK.\",\n        );\n      }\n\n      if (!this.options.internalFnId) {\n        throw new Error(\n          \"Missing internalFnId for async checkpointing. This is a bug in the Inngest SDK.\",\n        );\n      }\n\n      await retryWithBackoff(\n        () =>\n          this.options.client[\"inngestApi\"].checkpointStepsAsync({\n            runId: this.fnArg.runId,\n            fnId: this.options.internalFnId!,\n            queueItemId: this.options.queueItemId!,\n            steps,\n          }),\n        CHECKPOINT_RETRY_OPTIONS,\n      );\n    } else {\n      throw new Error(\n        \"Checkpointing is only supported in Sync and AsyncCheckpointing step modes. This is a bug in the Inngest SDK.\",\n      );\n    }\n  }\n\n  private async checkpointAndSwitchToAsync(\n    steps: OutgoingOp[],\n  ): Promise<ExecutionResult> {\n    await this.checkpoint(steps);\n\n    if (!this.state.checkpointedRun?.token) {\n      throw new Error(\"Failed to checkpoint and switch to async mode\");\n    }\n\n    return {\n      type: \"change-mode\",\n      ctx: this.fnArg,\n      ops: this.ops,\n      to: StepMode.Async,\n      token: this.state.checkpointedRun?.token!,\n    };\n  }\n\n  /**\n   * Returns whether we're in the final attempt of execution, or `null` if we\n   * can't determine this in the SDK.\n   */\n  private inFinalAttempt(): boolean | null {\n    if (typeof this.fnArg.maxAttempts !== \"number\") {\n      return null;\n    }\n\n    return this.fnArg.attempt + 1 >= this.fnArg.maxAttempts;\n  }\n\n  /**\n   * Creates a handler for every checkpoint type, defining what to do when we\n   * reach that checkpoint in the core loop.\n   */\n  private createCheckpointHandlers(): CheckpointHandlers {\n    const commonCheckpointHandler: CheckpointHandlers[StepMode][\"\"] = (\n      checkpoint,\n    ) => {\n      this.devDebug(`${this.options.stepMode} checkpoint:`, checkpoint);\n    };\n\n    const stepRanHandler = async (\n      stepResult: OutgoingOp,\n    ): Promise<ExecutionResult> => {\n      const transformResult = await this.transformOutput(stepResult);\n\n      /**\n       * Transforming output will always return either function rejection or\n       * resolution. In most cases, this can be immediately returned, but in\n       * this particular case we want to handle it differently.\n       */\n      if (transformResult.type === \"function-resolved\") {\n        return {\n          type: \"step-ran\",\n          ctx: transformResult.ctx,\n          ops: transformResult.ops,\n          step: {\n            ...stepResult,\n            data: transformResult.data,\n          },\n        };\n      } else if (transformResult.type === \"function-rejected\") {\n        const stepForResponse = {\n          ...stepResult,\n          error: transformResult.error,\n        };\n\n        if (stepResult.op === StepOpCode.StepFailed) {\n          const ser = serializeError(transformResult.error);\n          stepForResponse.data = {\n            __serialized: true,\n            name: ser.name,\n            message: ser.message,\n            stack: \"\",\n          };\n        }\n\n        return {\n          type: \"step-ran\",\n          ctx: transformResult.ctx,\n          ops: transformResult.ops,\n          retriable: transformResult.retriable,\n          step: stepForResponse,\n        };\n      }\n\n      return transformResult;\n    };\n\n    const maybeReturnNewSteps = async (): Promise<\n      ExecutionResult | undefined\n    > => {\n      const newSteps = await this.filterNewSteps(\n        Array.from(this.state.steps.values()),\n      );\n      if (newSteps) {\n        return {\n          type: \"steps-found\",\n          ctx: this.fnArg,\n          ops: this.ops,\n          steps: newSteps,\n        };\n      }\n\n      return;\n    };\n\n    const attemptCheckpointAndResume = async (\n      stepResult?: OutgoingOp,\n      resume = true,\n      force = false,\n    ): Promise<ExecutionResult | undefined> => {\n      // If we're here, we successfully ran a step, so we may now need\n      // to checkpoint it depending on the step buffer configured.\n      if (stepResult) {\n        const stepToResume = this.resumeStepWithResult(stepResult, resume);\n\n        // Clear `executingStep` immediately after resuming, before any await.\n        // `resumeStepWithResult` resolves the step's promise, queuing a\n        // microtask for the function to continue. Any subsequent await (e.g.\n        // the `transformOutput` hook) yields and lets that microtask run, so\n        // `executingStep` must already be cleared to avoid a false positive\n        // NESTING_STEPS warning in the next step's handler.\n        delete this.state.executingStep;\n\n        // Buffer a copy with transformed data for checkpointing\n        this.state.checkpointingStepBuffer.push({\n          ...stepToResume,\n          data: stepResult.data,\n        });\n      }\n\n      if (\n        force ||\n        !this.options.checkpointingConfig?.bufferedSteps ||\n        this.state.checkpointingStepBuffer.length >=\n          this.options.checkpointingConfig.bufferedSteps\n      ) {\n        this.devDebug(\"checkpointing and resuming execution after step run\");\n\n        try {\n          this.devDebug(\n            `checkpointing all buffered steps:`,\n            this.state.checkpointingStepBuffer\n              .map((op) => op.displayName || op.id)\n              .join(\", \"),\n          );\n\n          return void (await this.checkpoint(\n            this.state.checkpointingStepBuffer,\n          ));\n        } catch (err) {\n          // If checkpointing fails for any reason, fall back to returning\n          // ALL buffered steps to the executor via the normal async flow.\n          // The executor persists completed steps and rediscovers any\n          // parallel/errored steps on the next invocation.\n          this.devDebug(\n            \"error checkpointing after step run, so falling back to async\",\n            err,\n          );\n\n          const buffered = this.state.checkpointingStepBuffer;\n\n          if (buffered.length) {\n            return {\n              type: \"steps-found\" as const,\n              ctx: this.fnArg,\n              ops: this.ops,\n              steps: buffered as [OutgoingOp, ...OutgoingOp[]],\n            };\n          }\n\n          return;\n        } finally {\n          // Clear the checkpointing buffer\n          this.state.checkpointingStepBuffer = [];\n        }\n      } else {\n        this.devDebug(\n          `not checkpointing yet, continuing execution as we haven't reached buffered step limit of ${this.options.checkpointingConfig?.bufferedSteps}`,\n        );\n      }\n\n      return;\n    };\n\n    const syncHandlers: CheckpointHandlers[StepMode.Sync] = {\n      /**\n       * Run for all checkpoints. Best used for logging or common actions.\n       * Use other handlers to return values and interrupt the core loop.\n       */\n      \"\": commonCheckpointHandler,\n\n      \"function-resolved\": async (checkpoint, i) => {\n        const transformedData = checkpoint.data;\n\n        await this.checkpoint([\n          {\n            op: StepOpCode.RunComplete,\n            id: _internals.hashId(\"complete\"), // ID is not important here\n            data: await this.options.createResponse!(transformedData),\n          },\n        ]);\n\n        // Apply middleware transformation before returning\n        return await this.transformOutput({ data: checkpoint.data });\n      },\n\n      \"function-rejected\": async (checkpoint) => {\n        // If the function throws during sync execution, we want to switch to\n        // async mode so that we can retry. The exception is that we're already\n        // at max attempts, in which case we do actually want to reject.\n        if (this.inFinalAttempt()) {\n          // Apply middleware transformation before returning\n          return await this.transformOutput({ error: checkpoint.error });\n        }\n\n        // Otherwise, checkpoint the error and switch to async mode\n        return this.checkpointAndSwitchToAsync([\n          {\n            id: _internals.hashId(\"complete\"), // ID is not important here\n            op: StepOpCode.StepError,\n            error: checkpoint.error,\n          },\n        ]);\n      },\n\n      \"step-not-found\": () => {\n        return {\n          type: \"function-rejected\",\n          ctx: this.fnArg,\n          error: new Error(\n            \"Step not found when checkpointing; this should never happen\",\n          ),\n          ops: this.ops,\n          retriable: false,\n        };\n      },\n\n      \"steps-found\": async ({ steps }) => {\n        // If we're entering parallelism or async mode, checkpoint and switch\n        // to async.\n        if (steps.length !== 1 || steps[0].mode !== StepMode.Sync) {\n          return this.checkpointAndSwitchToAsync(\n            steps.map((step) => ({ ...step, id: step.hashedId })),\n          );\n        }\n\n        // Otherwise we're good to start executing things right now.\n        const result = await this.executeStep(steps[0]);\n\n        const transformed = await stepRanHandler(result);\n        if (transformed.type !== \"step-ran\") {\n          throw new Error(\n            \"Unexpected checkpoint handler result type after running step in sync mode\",\n          );\n        }\n\n        if (result.error) {\n          return this.checkpointAndSwitchToAsync([transformed.step]);\n        }\n\n        // Resume the step with original data for user code\n        //\n        // Note: We should likely also pass this through `transformOutput` and\n        // then `transformInput` to mimic the entire middleware cycle, to ensure\n        // that any transformations that purposefully skew the resulting type\n        // are supported.\n        const stepToResume = this.resumeStepWithResult(result);\n\n        // Clear executingStep now that the step result has been processed.\n        // Without this, the next step discovery would see stale state and\n        // emit a false positive NESTING_STEPS warning.\n        delete this.state.executingStep;\n\n        // Checkpoint with transformed data from middleware\n        const stepForCheckpoint = {\n          ...stepToResume,\n          data: transformed.step.data,\n        };\n\n        return void (await this.checkpoint([stepForCheckpoint]));\n      },\n\n      \"checkpointing-runtime-reached\": () => {\n        return this.checkpointAndSwitchToAsync([\n          {\n            op: StepOpCode.DiscoveryRequest,\n\n            // Append with time because we don't want Executor-side\n            // idempotency to dedupe. There may have been a previous\n            // discovery request.\n            id: _internals.hashId(`discovery-request-${Date.now()}`),\n          },\n        ]);\n      },\n\n      \"checkpointing-buffer-interval-reached\": () => {\n        return attemptCheckpointAndResume(undefined, false, true);\n      },\n    };\n\n    const asyncHandlers: CheckpointHandlers[StepMode.Async] = {\n      /**\n       * Run for all checkpoints. Best used for logging or common actions.\n       * Use other handlers to return values and interrupt the core loop.\n       */\n      \"\": commonCheckpointHandler,\n\n      /**\n       * The user's function has completed and returned a value.\n       */\n      \"function-resolved\": async ({ data }) => {\n        // Check for unreported new steps (e.g. from `Promise.race` where\n        // the winning branch completed before losing branches reported)\n        const newStepsResult = await maybeReturnNewSteps();\n        if (newStepsResult) {\n          return newStepsResult;\n        }\n\n        // We need to do this even here for async, as we could be returning\n        // data from an API endpoint, even if we were triggered async.\n        if (this.options.createResponse) {\n          data = await this.options.createResponse(data);\n        }\n\n        return await this.transformOutput({ data });\n      },\n\n      /**\n       * The user's function has thrown an error.\n       */\n      \"function-rejected\": async (checkpoint) => {\n        return await this.transformOutput({ error: checkpoint.error });\n      },\n\n      /**\n       * We've found one or more steps. Here we may want to run a step or report\n       * them back to Inngest.\n       */\n      \"steps-found\": async ({ steps }) => {\n        const stepResult = await this.tryExecuteStep(steps);\n        if (stepResult) {\n          return stepRanHandler(stepResult);\n        }\n\n        return maybeReturnNewSteps();\n      },\n\n      /**\n       * While trying to find a step that Inngest has told us to run, we've\n       * timed out or have otherwise decided that it doesn't exist.\n       */\n      \"step-not-found\": ({ step }) => {\n        const { foundSteps, totalFoundSteps } = this.getStepNotFoundDetails();\n        return {\n          type: \"step-not-found\",\n          ctx: this.fnArg,\n          ops: this.ops,\n          step,\n          foundSteps,\n          totalFoundSteps,\n        };\n      },\n\n      \"checkpointing-runtime-reached\": () => {\n        throw new Error(\n          \"Checkpointing maximum runtime reached, but this is not in a checkpointing step mode. This is a bug in the Inngest SDK.\",\n        );\n      },\n\n      \"checkpointing-buffer-interval-reached\": () => {\n        throw new Error(\n          \"Checkpointing maximum buffer interval reached, but this is not in a checkpointing step mode. This is a bug in the Inngest SDK.\",\n        );\n      },\n    };\n\n    const asyncCheckpointingHandlers: CheckpointHandlers[StepMode.AsyncCheckpointing] =\n      {\n        \"\": commonCheckpointHandler,\n        \"function-resolved\": async (checkpoint, i) => {\n          const output = await asyncHandlers[\"function-resolved\"](\n            checkpoint,\n            i,\n          );\n          if (output?.type === \"function-resolved\") {\n            const steps = this.state.checkpointingStepBuffer.concat({\n              op: StepOpCode.RunComplete,\n              id: _internals.hashId(\"complete\"), // ID is not important here\n              data: output.data,\n            });\n\n            if (isNonEmpty(steps)) {\n              return {\n                type: \"steps-found\",\n                ctx: output.ctx,\n                ops: output.ops,\n                steps,\n              };\n            }\n          }\n\n          return;\n        },\n        \"function-rejected\": async (checkpoint) => {\n          // If we have buffered steps, attempt checkpointing them first\n          if (this.state.checkpointingStepBuffer.length) {\n            const fallback = await attemptCheckpointAndResume(\n              undefined,\n              false,\n              true,\n            );\n            if (fallback) {\n              return fallback;\n            }\n          }\n\n          return await this.transformOutput({ error: checkpoint.error });\n        },\n        \"step-not-found\": asyncHandlers[\"step-not-found\"],\n        \"steps-found\": async ({ steps }) => {\n          // Note that if we have a requested run step, we'll never be\n          // checkpointing, as that's an async parallel execution mode.\n\n          // Break found steps in to { stepsToResume, newSteps }\n          const { stepsToResume, newSteps } = steps.reduce(\n            (acc, step) => {\n              if (!step.hasStepState) {\n                acc.newSteps.push(step);\n              } else if (!step.fulfilled) {\n                acc.stepsToResume.push(step);\n              }\n\n              return acc;\n            },\n            { stepsToResume: [], newSteps: [] } as {\n              stepsToResume: FoundStep[];\n              newSteps: FoundStep[];\n            },\n          );\n\n          this.devDebug(\"split found steps in to:\", {\n            stepsToResume: stepsToResume.length,\n            newSteps: newSteps.length,\n          });\n\n          // Got new steps? Exit early.\n          if (!this.options.requestedRunStep && newSteps.length) {\n            const stepResult = await this.tryExecuteStep(newSteps);\n            if (stepResult) {\n              this.devDebug(`executed step \"${stepResult.id}\" successfully`);\n\n              // We executed a step!\n              //\n              // We know that because we're in this mode, we're always free to\n              // checkpoint and continue if we ran a step and it was successful.\n              if (stepResult.error) {\n                // Flush buffered steps before falling back to async,\n                // so previously-successful steps aren't lost.\n                if (this.state.checkpointingStepBuffer.length) {\n                  const fallback = await attemptCheckpointAndResume(\n                    undefined,\n                    false,\n                    true,\n                  );\n                  if (fallback) {\n                    return fallback;\n                  }\n                }\n\n                // If we failed, go back to the regular async flow.\n                return stepRanHandler(stepResult);\n              }\n\n              // If we're here, we successfully ran a step, so we may now need\n              // to checkpoint it depending on the step buffer configured.\n              return await attemptCheckpointAndResume(stepResult);\n            }\n\n            // Flush any buffered checkpoint steps before returning\n            // new steps to the executor. Without this, steps executed\n            // in-process during AsyncCheckpointing but not yet\n            // checkpointed would be lost. When the executor later calls\n            // back with requestedRunStep, those steps would be missing\n            // from stepState, causing \"step not found\" errors.\n            if (this.state.checkpointingStepBuffer.length) {\n              const fallback = await attemptCheckpointAndResume(\n                undefined,\n                false,\n                true,\n              );\n              if (fallback) {\n                return fallback;\n              }\n            }\n\n            return maybeReturnNewSteps();\n          }\n\n          // If we have stepsToResume, resume as many as possible and resume execution\n          if (stepsToResume.length) {\n            this.devDebug(`resuming ${stepsToResume.length} steps`);\n\n            for (const st of stepsToResume) {\n              this.resumeStepWithResult({\n                ...st,\n                id: st.hashedId,\n              });\n            }\n          }\n\n          return;\n        },\n        \"checkpointing-runtime-reached\": async () => {\n          return {\n            type: \"steps-found\",\n            ctx: this.fnArg,\n            ops: this.ops,\n            steps: [\n              {\n                op: StepOpCode.DiscoveryRequest,\n\n                // Append with time because we don't want Executor-side\n                // idempotency to dedupe. There may have been a previous\n                // discovery request.\n                id: _internals.hashId(`discovery-request-${Date.now()}`),\n              },\n            ],\n          };\n        },\n\n        \"checkpointing-buffer-interval-reached\": () => {\n          return attemptCheckpointAndResume(undefined, false, true);\n        },\n      };\n\n    return {\n      [StepMode.Async]: asyncHandlers,\n      [StepMode.Sync]: syncHandlers,\n      [StepMode.AsyncCheckpointing]: asyncCheckpointingHandlers,\n    };\n  }\n\n  private getCheckpointHandler(type: keyof CheckpointHandlers[StepMode]) {\n    return this.checkpointHandlers[this.options.stepMode][type] as (\n      checkpoint: Checkpoint,\n      iteration: number,\n    ) => MaybePromise<ExecutionResult | undefined>;\n  }\n\n  private async tryExecuteStep(\n    steps: FoundStep[],\n  ): Promise<OutgoingOp | undefined> {\n    const hashedStepIdToRun =\n      this.options.requestedRunStep || this.getEarlyExecRunStep(steps);\n    if (!hashedStepIdToRun) {\n      return;\n    }\n\n    const step = steps.find(\n      (step) => step.hashedId === hashedStepIdToRun && step.fn,\n    );\n\n    if (step) {\n      return await this.executeStep(step);\n    }\n\n    /**\n     * Ensure we reset the timeout if we have a requested run step but couldn't\n     * find it, but also that we don't reset if we found and executed it.\n     */\n    return void this.timeout?.reset();\n  }\n\n  /**\n   * Given a list of outgoing ops, decide if we can execute an op early and\n   * return the ID of the step to execute if we can.\n   */\n  private getEarlyExecRunStep(steps: FoundStep[]): string | undefined {\n    /**\n     * We may have been disabled due to parallelism, in which case we can't\n     * immediately execute unless explicitly requested.\n     */\n    if (this.options.disableImmediateExecution) return;\n\n    const unfulfilledSteps = steps.filter((step) => !step.fulfilled);\n    if (unfulfilledSteps.length !== 1) return;\n\n    const op = unfulfilledSteps[0];\n\n    if (\n      op &&\n      op.op === StepOpCode.StepPlanned\n      // TODO We must individually check properties here that we do not want to\n      // execute on, such as retry counts. Nothing exists here that falls in to\n      // this case, but should be accounted for when we add them.\n      // && typeof op.opts === \"undefined\"\n    ) {\n      return op.hashedId;\n    }\n\n    return;\n  }\n\n  private async filterNewSteps(\n    foundSteps: FoundStep[],\n  ): Promise<[OutgoingOp, ...OutgoingOp[]] | undefined> {\n    if (this.options.requestedRunStep) {\n      return;\n    }\n\n    const newSteps = foundSteps.reduce((acc, step) => {\n      if (!step.hasStepState) {\n        acc.push(step);\n      }\n\n      return acc;\n    }, [] as FoundStep[]);\n\n    if (!newSteps.length) {\n      return;\n    }\n\n    await this.middlewareManager.onMemoizationEnd();\n\n    const stepList = newSteps.map<OutgoingOp>((step) => {\n      return {\n        displayName: step.displayName,\n        op: step.op,\n        id: step.hashedId,\n        name: step.name,\n        opts: step.opts,\n        userland: step.userland,\n      };\n    });\n\n    if (!isNonEmpty(stepList)) {\n      throw new UnreachableError(\"stepList is empty\");\n    }\n\n    return stepList;\n  }\n\n  private async executeStep(foundStep: FoundStep): Promise<OutgoingOp> {\n    const { id, name, opts, fn, displayName, userland, hashedId } = foundStep;\n    const { stepInfo, wrappedHandler, setActualHandler } = foundStep.middleware;\n\n    this.devDebug(`preparing to execute step \"${id}\"`);\n\n    this.timeout?.clear();\n\n    const outgoingOp: OutgoingOp = {\n      id: hashedId,\n      op: StepOpCode.StepRun,\n      name,\n      opts,\n      displayName,\n      userland,\n    };\n    this.state.executingStep = outgoingOp;\n\n    const store = await getAsyncCtx();\n\n    if (store?.execution) {\n      store.execution.executingStep = {\n        id,\n        name: displayName,\n      };\n    }\n\n    this.devDebug(`executing step \"${id}\"`);\n\n    if (this.rootSpanId && this.options.checkpointingConfig) {\n      clientProcessorMap\n        .get(this.options.client)\n        ?.declareStepExecution(\n          this.rootSpanId,\n          hashedId,\n          this.options.data?.attempt ?? 0,\n        );\n    }\n\n    let interval: GoInterval | undefined;\n\n    // `fn` already has middleware-transformed args baked in via `fnArgs` (i.e.\n    // the `transformStepInput` middleware hook already ran).\n    const actualHandler = () => runAsPromise(fn);\n\n    await this.middlewareManager.onMemoizationEnd();\n    await this.middlewareManager.onStepStart(stepInfo);\n\n    // If wrappedHandler hasn't been called yet (no deferred from discovery),\n    // set one up so wrapStep's next() still blocks until memoization.\n    if (!foundStep.memoizationDeferred) {\n      const deferred = createDeferredPromise<unknown>();\n      foundStep.memoizationDeferred = deferred;\n      setActualHandler(() => deferred.promise);\n      foundStep.transformedResultPromise = wrappedHandler();\n      foundStep.transformedResultPromise.catch(() => {\n        // Swallow — errors handled by handle()\n      });\n    }\n\n    // Build wrapStepHandler chain around the actual handler\n    const wrappedActualHandler =\n      this.middlewareManager.buildWrapStepHandlerChain(actualHandler, stepInfo);\n\n    return goIntervalTiming(() => wrappedActualHandler())\n      .finally(() => {\n        this.devDebug(`finished executing step \"${id}\"`);\n\n        this.state.executingStep = undefined;\n\n        if (this.rootSpanId && this.options.checkpointingConfig) {\n          clientProcessorMap\n            .get(this.options.client)\n            ?.clearStepExecution(this.rootSpanId);\n        }\n\n        if (store?.execution) {\n          delete store.execution.executingStep;\n        }\n      })\n      .then<OutgoingOp>(async ({ resultPromise, interval: _interval }) => {\n        interval = _interval;\n        const metadata = this.state.metadata?.get(id);\n        const serverData = await resultPromise;\n\n        // Don't resolve memoizationDeferred here. wrapStep's next() must\n        // block until the step is actually memoized (i.e. handle() fires\n        // with confirmed data from the server). handle() resolves it.\n        await this.middlewareManager.onStepComplete(stepInfo, serverData);\n\n        return {\n          ...outgoingOp,\n          data: serverData,\n          ...(metadata && metadata.length > 0 ? { metadata: metadata } : {}),\n        };\n      })\n      .catch<OutgoingOp>((error) => {\n        // Don't reject memoizationDeferred — handle() will reject it when\n        // the error is memoized.\n        return this.buildStepErrorOp({\n          error,\n          id,\n          outgoingOp,\n          stepInfo,\n        });\n      })\n      .then((op) => ({\n        ...op,\n        timing: interval,\n      }));\n  }\n\n  /**\n   * Starts execution of the user's function, including triggering checkpoints\n   * and middleware hooks where appropriate.\n   */\n  private async startExecution(): Promise<void> {\n    /**\n     * Start the timer to time out the run if needed.\n     */\n    void this.timeout?.start();\n    void this.checkpointingMaxRuntimeTimer?.start();\n    void this.checkpointingMaxBufferIntervalTimer?.start();\n\n    const fnInputResult = await this.middlewareManager.transformFunctionInput();\n    this.applyFunctionInputMutations(fnInputResult);\n\n    if (this.state.allStateUsed()) {\n      await this.middlewareManager.onMemoizationEnd();\n    }\n\n    if (this.state.stepsToFulfill === 0 && this.fnArg.attempt === 0) {\n      await this.middlewareManager.onRunStart();\n    }\n\n    const innerHandler: () => Promise<unknown> = async () => {\n      await this.validateEventSchemas();\n      return this.userFnToRun(this.fnArg);\n    };\n\n    const runHandler = this.middlewareManager.wrapRunHandler(innerHandler);\n\n    runAsPromise(runHandler)\n      .then(async (data) => {\n        await this.middlewareManager.onRunComplete(data);\n        this.state.setCheckpoint({ type: \"function-resolved\", data });\n      })\n      .catch(async (error) => {\n        // Preserve Error instances; stringify non-Error throws (e.g. `throw {}`)\n        let err: Error;\n        if (error instanceof Error) {\n          err = error;\n        } else if (typeof error === \"object\") {\n          err = new Error(JSON.stringify(error));\n        } else {\n          err = new Error(String(error));\n        }\n\n        await this.middlewareManager.onRunError(err, !this.retriability(err));\n        this.state.setCheckpoint({ type: \"function-rejected\", error: err });\n      });\n  }\n\n  /**\n   * Determine whether the given error is retriable. Returns `false` when the\n   * run should not be retried, a duration string for `RetryAfterError`, or\n   * `true` for normal retry behavior.\n   */\n  private retriability(error: unknown): boolean | string {\n    const areRetriesExhausted =\n      this.fnArg.maxAttempts &&\n      this.fnArg.maxAttempts - 1 === this.fnArg.attempt;\n    if (areRetriesExhausted) {\n      return false;\n    }\n\n    // TODO: Replace this fragile inheritance + name check. Maybe we should have\n    // an \"~inngest\" field with an object that specifies \"isRetriable\".\n    if (\n      error instanceof NonRetriableError ||\n      // biome-ignore lint/suspicious/noExplicitAny: instanceof fails across module boundaries\n      (error as any)?.name === \"NonRetriableError\"\n    ) {\n      return false;\n    }\n\n    // If the function-level code did not change the error, then we don't want\n    // to retry. The vast majority of the time this means there wasn't a `catch`\n    // block.\n    const isUncaughtStepError =\n      error instanceof StepError &&\n      error === this.state.recentlyRejectedStepError;\n    if (isUncaughtStepError) {\n      return false;\n    }\n\n    // TODO: Replace this fragile inheritance + name check. Maybe we should have\n    // an \"~inngest\" field with an object that specifies \"retryAfter\".\n    if (\n      error instanceof RetryAfterError ||\n      // biome-ignore lint/suspicious/noExplicitAny: instanceof fails across module boundaries\n      (error as any)?.name === \"RetryAfterError\"\n    ) {\n      return (error as RetryAfterError).retryAfter;\n    }\n\n    return true;\n  }\n\n  /**\n   * Build the OutgoingOp for a failed step, notifying middleware and choosing\n   * retriable vs non-retriable opcode.\n   */\n  private async buildStepErrorOp({\n    error,\n    id,\n    outgoingOp,\n    stepInfo,\n  }: {\n    error: unknown;\n    id: string;\n    outgoingOp: OutgoingOp;\n    stepInfo: Middleware.StepInfo;\n  }): Promise<OutgoingOp> {\n    const isFinal = !this.retriability(error);\n    const metadata = this.state.metadata?.get(id);\n\n    await this.middlewareManager.onStepError(\n      stepInfo,\n      error instanceof Error ? error : new Error(String(error)),\n      isFinal,\n    );\n\n    // Serialize the error so it survives JSON.stringify (raw Error\n    // objects have non-enumerable properties that get dropped).\n    // This is critical for checkpoint requests where the error is\n    // sent as JSON in the request body.\n    const serialized = serializeError(error);\n\n    return {\n      ...outgoingOp,\n      error: serialized,\n      op: isFinal ? StepOpCode.StepFailed : StepOpCode.StepError,\n      ...(metadata && metadata.length > 0 ? { metadata } : {}),\n    };\n  }\n\n  /**\n   * Validate event data against schemas defined in function triggers.\n   */\n  private async validateEventSchemas(): Promise<void> {\n    if (this.options.isFailureHandler) {\n      // Skip validation because the main function's triggers don't apply to its\n      // `onFailure` handler. The `onFailure` handler is a separate Inngest\n      // function that's implicitly triggered by the \"inngest/function.failed\"\n      // event.\n      return;\n    }\n\n    const triggers = this.options.fn.opts.triggers;\n    if (!triggers || triggers.length === 0) return;\n\n    const fnArgEvents = this.fnArg.events;\n    if (!fnArgEvents || fnArgEvents.length === 0) return;\n\n    const events = fnArgEvents.map((event) => ({\n      name: event.name,\n      data: event.data,\n    }));\n\n    await validateEvents(events, triggers);\n  }\n\n  /**\n   * Using middleware, transform output before returning.\n   */\n  private transformOutput(dataOrError: {\n    data?: unknown;\n    error?: unknown;\n  }): ExecutionResult {\n    const { data, error } = dataOrError;\n\n    if (typeof error !== \"undefined\") {\n      const retriable = this.retriability(error);\n      const serializedError = serializeError(error);\n\n      return {\n        type: \"function-rejected\",\n        ctx: this.fnArg,\n        ops: this.ops,\n        error: serializedError,\n        retriable,\n      };\n    }\n\n    return {\n      type: \"function-resolved\",\n      ctx: this.fnArg,\n      ops: this.ops,\n      data: undefinedToNull(data),\n    };\n  }\n\n  private createExecutionState(): ExecutionState {\n    const d = createDeferredPromiseWithStack<Checkpoint>();\n    let checkpointResolve = d.deferred.resolve;\n    const checkpointResults = d.results;\n\n    const loop: ExecutionState[\"loop\"] = (async function* (\n      cleanUp?: () => void,\n    ) {\n      try {\n        while (true) {\n          const res = (await checkpointResults.next()).value;\n          if (res) {\n            yield res;\n          }\n        }\n      } finally {\n        cleanUp?.();\n      }\n    })(() => {\n      this.timeout?.clear();\n      this.checkpointingMaxRuntimeTimer?.clear();\n      this.checkpointingMaxBufferIntervalTimer?.clear();\n      void checkpointResults.return();\n    });\n\n    const stepsToFulfill = Object.keys(this.options.stepState).length;\n\n    const state: ExecutionState = {\n      stepState: this.options.stepState,\n      stepsToFulfill,\n      steps: new Map(),\n      loop,\n      hasSteps: Boolean(stepsToFulfill),\n      stepCompletionOrder: [...this.options.stepCompletionOrder],\n      remainingStepsToBeSeen: new Set(this.options.stepCompletionOrder),\n      setCheckpoint: (checkpoint: Checkpoint) => {\n        this.devDebug(\"setting checkpoint:\", checkpoint.type);\n\n        ({ resolve: checkpointResolve } = checkpointResolve(checkpoint));\n      },\n      allStateUsed: () => {\n        return this.state.remainingStepsToBeSeen.size === 0;\n      },\n      checkpointingStepBuffer: [],\n      metadata: new Map(),\n    };\n\n    return state;\n  }\n\n  get ops(): Record<string, MemoizedOp> {\n    return Object.fromEntries(this.state.steps);\n  }\n\n  private createFnArg(): Context.Any {\n    const step = this.createStepTools();\n    const experimentStepRun = (step as unknown as ExperimentStepTools)[\n      experimentStepRunSymbol\n    ];\n\n    let fnArg = {\n      ...(this.options.data as { event: EventPayload }),\n      step,\n      group: createGroupTools({ experimentStepRun }),\n    } as Context.Any;\n\n    /**\n     * Handle use of the `onFailure` option by deserializing the error.\n     */\n    if (this.options.isFailureHandler) {\n      const eventData = z\n        .object({ error: jsonErrorSchema })\n        .parse(fnArg.event?.data);\n\n      (fnArg as Partial<Pick<FailureEventArgs, \"error\">>) = {\n        ...fnArg,\n        error: deserializeError(eventData.error),\n      };\n    }\n\n    return this.options.transformCtx?.(fnArg) ?? fnArg;\n  }\n\n  /**\n   * Apply mutations from `transformFunctionInput` back to execution state.\n   * Allows middleware to modify event data, step tools, memoized step data,\n   * and inject custom fields into the handler context.\n   */\n  private applyFunctionInputMutations(\n    result: Middleware.TransformFunctionInputArgs,\n  ): void {\n    const { event, events, step, ...extensions } = result.ctx;\n\n    // Mutate in place so the ALS store's reference to this.fnArg stays valid.\n    if (event !== this.fnArg.event) {\n      this.fnArg.event = event;\n    }\n\n    if (events !== this.fnArg.events) {\n      this.fnArg.events = events;\n    }\n\n    if (step !== this.fnArg.step) {\n      this.fnArg.step = step;\n    }\n\n    if (Object.keys(extensions).length > 0) {\n      Object.assign(this.fnArg, extensions);\n    }\n\n    // Apply step data mutations\n    for (const [hashedId, stepData] of Object.entries(result.steps)) {\n      const existing = this.state.stepState[hashedId];\n      if (\n        existing &&\n        stepData &&\n        stepData.type === \"data\" &&\n        stepData.data !== existing.data\n      ) {\n        this.state.stepState[hashedId] = { ...existing, data: stepData.data };\n      }\n    }\n  }\n\n  private createStepTools(): ReturnType<typeof createStepTools> {\n    /**\n     * A list of steps that have been found and are being rolled up before being\n     * reported to the core loop.\n     */\n    const foundStepsToReport: Map<string, FoundStep> = new Map();\n\n    /**\n     * A map of the subset of found steps to report that have not yet been\n     * handled. Used for fast access to steps that need to be handled in order.\n     */\n    const unhandledFoundStepsToReport: Map<string, FoundStep> = new Map();\n\n    /**\n     * A map of the latest sequential step indexes found for each step ID. Used\n     * to ensure that we don't index steps in parallel.\n     *\n     * Note that these must be sequential; if we've seen or assigned `a:1`,\n     * `a:2` and `a:4`, the latest sequential step index is `2`.\n     *\n     */\n    const expectedNextStepIndexes: Map<string, number> = new Map();\n\n    /**\n     * An ordered list of step IDs that have yet to be handled in this\n     * execution. Used to ensure that we handle steps in the order they were\n     * found and based on the `stepCompletionOrder` in this execution's state.\n     */\n    const remainingStepCompletionOrder: string[] =\n      this.state.stepCompletionOrder.slice();\n\n    /**\n     * A promise that's used to ensure that step reporting cannot be run more than\n     * once in a given asynchronous time span.\n     */\n    let foundStepsReportPromise: Promise<void> | undefined;\n\n    /**\n     * A flag used to ensure that we only warn about parallel indexing once per\n     * execution to avoid spamming the console.\n     */\n    let warnOfParallelIndexing = false;\n\n    /**\n     * Counts the number of times we've extended this tick.\n     */\n    let tickExtensionCount = 0;\n\n    /**\n     * Given a colliding step ID, maybe warn the user about parallel indexing.\n     */\n    const maybeWarnOfParallelIndexing = (userlandCollisionId: string) => {\n      if (warnOfParallelIndexing) {\n        return;\n      }\n\n      const hashedCollisionId = _internals.hashId(userlandCollisionId);\n\n      const stepExists = this.state.steps.has(hashedCollisionId);\n      if (stepExists) {\n        const stepFoundThisTick = foundStepsToReport.has(hashedCollisionId);\n        if (!stepFoundThisTick) {\n          warnOfParallelIndexing = true;\n\n          this.options.cli