eve
Version:
Filesystem-first framework for durable backend AI agents that run anywhere.
282 lines • 15.7 kB
TypeScript
import type { Span } from '#compiled/@opentelemetry/api/index.js';
import { type SerializedData, type TraceCarrier, type ValidQueueName, type WorkflowRun, type World } from '#compiled/@workflow/world/index.js';
import type { StepInvocationQueueItem, WorkflowSuspension } from '../global.js';
import { type GuestCodeExecution } from '../serialization/hardened.js';
import { type LoadedEventLog } from './helpers.js';
import { ReplayRecoveryReporter } from './replay-recovery-reporter.js';
import type { PreclaimedInlineStart } from './step-executor.js';
export interface SuspensionHandlerParams {
suspension: WorkflowSuspension;
world: World;
run: WorkflowRun;
span?: Span;
requestId?: string;
/**
* The runtime's loaded event log. Every event creation this suspension makes
* names the position it was derived from, so a backend that has recorded
* events the replay did not see can report them back on the write, or, if
* it would rather refuse than report, reject it with a 412. A rejection is
* not retried here: the event's correlation id was minted by *this* replay's
* seeded sequence, so re-committing it against a corrected log would persist
* an event no correct replay produces. The caller restarts the replay
* instead.
*
* Extended in place by what those writes report back — the events on slots
* they skipped over, and (on the hook create) the inline delta since its
* cursor. Whether the log is complete afterwards is answered by
* {@link SuspensionHandlerResult.eventLogCarriedForward}.
*/
eventLog?: LoadedEventLog;
/**
* Turbo mode only: a promise that resolves once the backgrounded
* `run_started` has landed (the run exists). When present, every world write
* this suspension performs (`hook_created`, `wait_created`, eager overflow
* `step_created`, …) is gated on it so the write never races ahead of the
* run's creation. The pure inline hot path defers all of its steps and writes
* nothing here, so it never awaits this barrier. `undefined` outside turbo,
* where `run_started` was already awaited up front.
*/
runReadyBarrier?: Promise<unknown>;
/** One-shot telemetry reporter, activated only after replay has recovered. */
replayRecoveryReporter?: ReplayRecoveryReporter;
/**
* Resilient step dispatch: when provided (and the per-step eligibility gates
* pass, see the step ops below), each newly created non-inline step's
* `step_created` write is parallelized with its step-execution queue
* publish, and the queue message carries the serialized step input
* (`stepInput`) so the consumer can idempotently re-ensure the event if the
* direct write failed transiently. Steps queued this way are reported in
* {@link SuspensionHandlerResult.queuedStepCorrelationIds} so the caller
* skips them in its own dispatch pass. Omitted by callers that must not
* queue (terminal drain, tests); creates then behave exactly as before.
*/
stepDispatch?: {
/** The unified workflow queue this run's step messages are published to. */
queueName: ValidQueueName;
/**
* Lazily resolves the trace carrier to stamp on the step messages.
* Called at most once per suspension (memoized here).
*/
getTraceCarrier: () => Promise<TraceCarrier>;
};
/**
* Inline step ownership: the queue message ID of the invocation this
* suspension runs in (the queue handler's meta). When present AND the
* batched fan-out engages, the lazy-inline steps' deferred writes are
* folded into the batch as `step_created` + `step_started` pairs (the
* started row stamped with this ID, exactly like the lazy claim it
* replaces), pre-claiming the steps the caller is about to run inline. See
* {@link SuspensionHandlerResult.inlineClaims}. Callers that never
* inline-execute (terminal drain) omit it, keeping their lazy steps on the
* plain deferred path.
*/
ownerMessageId?: string;
/**
* Lets the batched fan-out return before every chunk has committed: only
* the chunk carrying the pre-claimed inline pairs gates the handler's
* return (its claims are what the caller starts bodies from), while the
* trailing chunks' commits (and every chunk's in-flush step-message
* publishes) ride {@link SuspensionHandlerResult.deferredBatchWork}. A
* caller that opts in MUST await that promise before acking its delivery:
* the durability contract ("every create durable before ack") moves from
* the handler's return to that join, and nothing else re-drives a lost
* trailing chunk. Callers that don't opt in (terminal drain, default)
* keep the everything-durable-at-return behavior.
*/
allowDeferredBatchWork?: boolean;
}
/**
* Result of handling a suspension. Returns pending step items so the caller
* can decide which to execute inline vs queue to background.
*/
export interface SuspensionHandlerResult {
/** Pending step items with events created but NOT queued */
pendingSteps: StepInvocationQueueItem[];
/**
* Correlation IDs for which this suspension call actually wrote the
* step_created event (as opposed to catching EntityConflictError because
* a concurrent handler wrote it first). Only the handler that wrote the
* step_created event should queue / inline-execute the step; this
* guarantees a single owner per step, even when multiple handlers race
* into the same batch boundary.
*/
createdStepCorrelationIds: Set<string>;
/**
* Correlation IDs of steps whose arguments failed to serialize. Each was
* finalized here as `step_created` (with a placeholder input; the real
* input is precisely what refused to serialize) followed by `step_failed`
* carrying the SerializationError, so the next replay rejects the step's
* promise and a try/catch around the step call observes the error,
* exactly like a step-body failure. No step-execution message is
* dispatched for these, so the caller MUST force an in-process replay:
* when the failed step was the only pending work, nothing else will ever
* re-invoke the run to observe the terminal event.
*/
failedStepCorrelationIds: Set<string>;
/**
* Correlation IDs of steps this suspension call already published
* step-execution queue messages for, via resilient step dispatch (the
* `step_created` write parallelized with a `stepInput`-carrying queue
* publish). The caller MUST NOT dispatch these again: the message is
* already out (a duplicate would be deduped by its idempotency key, but
* costs a wasted round-trip). Empty when {@link SuspensionHandlerParams.stepDispatch}
* was not provided or no step was eligible.
*/
queuedStepCorrelationIds: Set<string>;
/**
* How many events this phase's writes reported back as occupying slots they
* skipped over, already merged into the caller's `eventLog.events`. Nonzero
* means the array was reordered to restore slot order, so any index the
* caller cached into it (payload prewarm scan position) is stale.
*/
reportedEventCount: number;
/**
* The steps whose `step_created` writes were intentionally deferred so the
* caller can run them inline via lazy `step_started` events (which create
* the step on the fly), saving one world round-trip per inline step. Up to
* `getMaxInlineSteps()` steps are deferred; the caller runs them inline in
* parallel and queues the rest. Empty when no step was deferred (nothing
* pending, or a `hook.getConflict()` awaiter is present so nothing is
* executed inline). The caller passes each `dehydratedInput` straight to
* `executeStep`, which sends it as the `step_started` payload. The atomic
* create-claim inside each `step_started` is the exactly-one-owner gate that
* the standalone `step_created` provided before: the loser of the race gets
* `EntityConflictError` → `skipped` and does not run the body.
*/
lazyInlineSteps: Array<{
correlationId: string;
stepName: string;
dehydratedInput: SerializedData;
}>;
/**
* Pre-claimed inline starts, by correlation id: the per-step verdicts of
* the `step_created` + `step_started` pairs the batched fan-out committed
* for the lazy-inline steps. A step with an entry here is passed to
* `executeStep` as `preclaimedStart` INSTEAD of `lazyStepInput`: its
* input already rode the pair, and the claim is settled: `owned: true`
* carries the started attempt-1 entity (input re-attached) so the body
* runs straight off the batch commit with no start write of its own;
* `owned: false` lost the pair's atomic create-claim to a concurrent
* writer, and executeStep returns `skipped` without running the body,
* the same outcome as losing the lazy claim. Empty whenever the fold did
* not engage (batching off, no `ownerMessageId`, or the lone-inline case,
* which keeps the optimistic lazy path and its claim/body overlap).
*
* Crash window: the pair commits before the caller runs the body, so a
* crash between them leaves a started step stamped with this message's
* ID. Redelivery of the same message re-executes it via the owned-recovery
* path, the exact machinery the lazy claim's crash window already uses.
*/
inlineClaims: Map<string, PreclaimedInlineStart>;
/**
* The batched fan-out's deferred work, present only when the caller opted
* in via {@link SuspensionHandlerParams.allowDeferredBatchWork} and
* trailing work exists: the commits of every chunk except the pair chunk,
* plus every chunk's step-message publishes (each chained on ITS OWN
* chunk's commit, so publish-after-create holds per step). The caller
* MUST await it before acking: a rejection here is a failed suspension
* write and fails the delivery exactly as it would have at the handler's
* return. Steps whose messages this work publishes are already in
* {@link queuedStepCorrelationIds} at return time.
*/
deferredBatchWork?: Promise<void>;
/**
* The soonest pending wait, if any: seconds until it elapses and the
* correlationId of the wait that produced that timeout. The
* correlationId seeds the idempotency key for the wait-continuation
* queue message so that repeated suspension passes over the same
* pending wait collapse into a single delayed continuation.
*/
waitTimeout?: {
seconds: number;
correlationId: string;
};
/**
* Whether a hook create committed a `hook_conflict` — the token was already
* claimed, so this run's hook was never created and the workflow must
* observe the conflict before anything else this suspension scheduled runs.
* The caller answers it by advancing the workflow over the committed event.
*/
hasHookConflict: boolean;
/** Whether a `hook.getConflict()` awaiter needs the workflow to continue immediately */
hasAwaitedHookCreation: boolean;
/**
* Correlation ids of the hooks this suspension committed a `hook_created`
* for while a `hook.getConflict()` awaiter was waiting on it — the events
* that resolve those awaiters on the next pass. Empty exactly when
* {@link hasAwaitedHookCreation} is false.
*
* Reported by id, not just counted, so a caller that continues in this
* process can tell a continuation that got somewhere from one that is
* repeating: a workflow may create one awaited hook after another, and each
* new id is a pass that made progress, while the same id coming back means
* the pass ran over a log that still did not hold its event and continuing
* again cannot change that.
*/
awaitedHookCorrelationIds: string[];
/**
* Correlation ids of the hooks whose create committed a `hook_conflict`.
* Empty exactly when {@link hasHookConflict} is false.
*
* By id for the same reason as {@link awaitedHookCorrelationIds}, and
* against the same hazard: a conflict is resolved by the `hook_conflict`
* this suspension committed, and until the workflow observes it the hook
* stays in the invocations queue and the next pass writes the create again.
* A fresh id is progress; the same id coming back is a pass that ran over a
* log which still did not hold the event, so continuing again cannot change
* that.
*/
hookConflictCorrelationIds: string[];
/**
* Whether the caller's `eventLog` now holds every event this suspension
* committed, so a caller that continues in this process can replay — or
* resume a retained VM — straight off it with no read.
*
* True only when the hook create's inline delta came back complete and was
* folded in (see `hookDeltaCursor` below), and nothing else in this
* suspension wrote an event. False whenever a read is needed first: no
* delta was asked for or returned, it was truncated, or a step / wait /
* attribute / abort write landed above it and is therefore not in it.
*
* Indifferent to which event the create committed: the delta is the slice
* of the log after the caller's cursor either way, so it carries a
* `hook_conflict` exactly as it carries a `hook_created`.
*/
eventLogCarriedForward: boolean;
/** Whether native workflow attribute events were written for replay. */
hasAttributeEvents: boolean;
/**
* Whether this suspension created any hook (`hook_created`) events. Unlike
* `hasHookConflict` / `hasAwaitedHookCreation`, this is true even for a plain
* fire-and-forget hook with no conflict and no awaiter. Turbo mode uses it to
* detect "a hook was created this suspension" and stop forcing optimistic
* inline start (a hook introduces later resume invocations that could race).
*/
hasHookEvents: boolean;
/**
* Wall-clock ms spent committing this suspension's `hook_created` events
* (0 when it created none). The caller accumulates this across iterations
* and subtracts it from the TTFS latency measurement, so time spent
* durably creating the user's hooks doesn't count as runtime overhead.
*/
hookCreationMs: number;
/** Exact number of workflow-code executions observed during serialization. */
serializationBlockerCount: number;
/** Bounded sample used only for retention diagnostics. */
serializationBlockers: SuspensionSerializationBlocker[];
}
export interface SuspensionSerializationBlocker extends GuestCodeExecution {
source: 'step_input' | 'hook_metadata' | 'hook_abort';
correlationId: string;
}
/**
* Handles a workflow suspension by processing all pending operations (hooks, steps, waits).
* Creates events for all operations but does NOT queue step messages; returns the pending
* steps so the caller can decide which to execute inline vs queue to background.
*
* Processing order:
* 1. Hooks are processed first to prevent race conditions with webhook receivers
* 2. Step events and wait events are created in parallel
*/
export declare function handleSuspension({ suspension, world, run, span, requestId, eventLog, runReadyBarrier, replayRecoveryReporter, stepDispatch, ownerMessageId, allowDeferredBatchWork, }: SuspensionHandlerParams): Promise<SuspensionHandlerResult>;
//# sourceMappingURL=suspension-handler.d.ts.map