inngest
Version:
Official SDK for Inngest.com. Inngest is the reliability layer for modern applications. Inngest combines durable execution, events, and queues into a zero-infra platform with built-in observability.
1,390 lines (1,389 loc) • 61.3 kB
JavaScript
import { __export } from "../../_virtual/rolldown_runtime.js";
import { ExecutionVersion, defaultMaxRetries, headerKeys } from "../../helpers/consts.js";
import { version } from "../../version.js";
import { NonRetriableError } from "../NonRetriableError.js";
import { ErrCode, deserializeError, serializeError } from "../../helpers/errors.js";
import { StepMode, StepOpCode, jsonErrorSchema } from "../../types.js";
import { InngestExecution } from "./InngestExecution.js";
import { isRecord } from "../../helpers/types.js";
import { undefinedToNull } from "../../helpers/functions.js";
import { isDeferredFunction } from "../../helpers/marker.js";
import { isTemporalDuration } from "../../helpers/temporal.js";
import { createDeferredPromise, createDeferredPromiseWithStack, createTimeoutPromise, goIntervalTiming, resolveAfterPending, resolveNextTick, retryWithBackoff, runAsPromise } from "../../helpers/promises.js";
import { getAsyncCtx, getAsyncLocalStorage } from "./als.js";
import { STEP_INDEXING_SUFFIX, createStepTools, experimentStepRunSymbol, getStepOptions } from "../InngestStepTools.js";
import { UnreachableError } from "../middleware/utils.js";
import { MiddlewareManager } from "../middleware/manager.js";
import { internalLoggerSymbol } from "../Inngest.js";
import { createGroupTools } from "../InngestGroupTools.js";
import { RetryAfterError } from "../RetryAfterError.js";
import { StepError } from "../StepError.js";
import { buildSseMetadataEvent, prependToStream } from "./streaming.js";
import { Stream } from "../StreamTools.js";
import { validateEvents } from "../triggers/utils.js";
import { LazyOps, isLazyOp } from "./lazyOps.js";
import { clientProcessorMap } from "./otel/access.js";
import { z } from "zod/v3";
import hashjs from "hash.js";
import ms from "ms";
import { trace } from "@opentelemetry/api";
//#region src/components/execution/engine.ts
var engine_exports = /* @__PURE__ */ __export({
_internals: () => _internals,
createExecutionEngine: () => createExecutionEngine
});
const { sha1 } = hashjs;
/**
* Retry configuration for checkpoint operations.
*
* Checkpoint calls use exponential backoff with jitter to handle transient
* network failures (e.g., dev server temporarily down, cloud hiccup). If
* retries exhaust, the error propagates up - for Sync mode this results in a
* 500 error, for AsyncCheckpointing the caller handles fallback.
*/
const CHECKPOINT_RETRY_OPTIONS = {
maxAttempts: 5,
baseDelay: 100
};
function errorMessage(error) {
if (error instanceof Error) return error.message;
if (isRecord(error) && typeof error.message === "string") return error.message;
return String(error);
}
/**
* Placeholder step ID used when completing a checkpointed run.
*/
const RUN_COMPLETE_STEP_ID = "complete";
const STEP_NOT_FOUND_MAX_FOUND_STEPS = 25;
const createExecutionEngine = (options) => {
return new InngestExecutionEngine(options);
};
function extractSseResponse(response, body) {
const headers = {};
response.headers.forEach((value, key) => {
headers[key] = value;
});
return {
body,
statusCode: response.status,
headers
};
}
function defaultSseResponse(data) {
return {
body: JSON.stringify(data),
statusCode: 200,
headers: { "content-type": "application/json" }
};
}
function jsonResponse(data, status = 200) {
return new Response(JSON.stringify(data), {
status,
headers: { "Content-Type": "application/json" }
});
}
var InngestExecutionEngine = class extends InngestExecution {
version = ExecutionVersion.V2;
state;
fnArg;
checkpointHandlers;
timeoutDuration = 1e3 * 10;
execution;
userFnToRun;
middlewareManager;
/**
* Close the stream via {@link streamCloseSucceeded}, {@link streamCloseFailed},
* or {@link streamEnd} — never call `streamTools.close*`/`end` directly, as
* the wrappers ensure the redirect event is flushed first.
*/
streamTools;
/**
* Resolved when `stream.push()`/`pipe()` is first called in sync mode,
* allowing `_start()` to return the SSE Response to the HTTP layer while
* the core loop continues executing steps in the background.
*/
earlyStreamResponse;
/**
* Whether the `inngest.redirect_info` SSE event has already been sent.
* Prevents duplicate redirect events.
*/
redirectSent = false;
/**
* Promise that resolves once the redirect event has been written (or the
* attempt completes). Stored so that `checkpointAndSwitchToAsync` can
* await it before closing the writer.
*/
redirectPromise = Promise.resolve();
/**
* If we're supposed to run a particular step via `requestedRunStep`, this
* will be a `Promise` that resolves after no steps have been found for
* `timeoutDuration` milliseconds.
*
* If we're not supposed to run a particular step, this will be `undefined`.
*/
timeout;
rootSpanId;
/**
* If we're checkpointing and have been given a maximum runtime, this will be
* a `Promise` that resolves after that duration has elapsed, allowing us to
* ensure that we end the execution in good time, especially in serverless
* environments.
*/
checkpointingMaxRuntimeTimer;
/**
* If we're checkpointing and have been given a maximum buffer interval, this
* will be a `Promise` that resolves after that duration has elapsed, allowing
* us to periodically checkpoint even if the step buffer hasn't filled.
*/
checkpointingMaxBufferIntervalTimer;
constructor(rawOptions) {
const options = {
...rawOptions,
stepMode: rawOptions.stepMode ?? StepMode.Async
};
super(options);
/**
* Check we have everything we need for checkpointing
*/
if (this.options.stepMode === StepMode.Sync) {
if (!this.options.createResponse) throw new Error("createResponse is required for sync step mode");
}
this.userFnToRun = this.getUserFnToRun();
this.streamTools = new Stream({
onActivated: () => this.handleStreamActivated(),
onWriteError: (err) => this.devDebug("stream write error (client may have disconnected):", err)
});
this.state = this.createExecutionState();
this.fnArg = this.createFnArg();
const mwInstances = this.options.middlewareInstances ?? (this.options.client.middleware || []).map((Cls) => {
return new Cls({ client: this.options.client });
});
this.middlewareManager = new MiddlewareManager(this.fnArg, () => this.state.stepState, mwInstances, this.options.fn, this.options.client[internalLoggerSymbol]);
this.checkpointHandlers = this.createCheckpointHandlers();
this.initializeTimer(this.state);
this.initializeCheckpointRuntimeTimer(this.state);
this.devDebug("created new V1 execution for run;", this.options.requestedRunStep ? `wanting to run step "${this.options.requestedRunStep}"` : "discovering steps");
this.devDebug("existing state keys:", Object.keys(this.state.stepState));
}
/**
* Idempotently start the execution of the user's function.
*/
start() {
if (!this.execution) {
this.devDebug("starting V1 execution");
const tracer = trace.getTracer("inngest", version);
this.execution = getAsyncLocalStorage().then((als) => {
return als.run({
app: this.options.client,
execution: {
ctx: this.fnArg,
instance: this,
stream: this.streamTools
}
}, async () => {
return tracer.startActiveSpan("inngest.execution", (span) => {
this.rootSpanId = span.spanContext().spanId;
clientProcessorMap.get(this.options.client)?.declareStartingSpan({
span,
runId: this.options.runId,
traceparent: this.options.headers[headerKeys.TraceParent],
tracestate: this.options.headers[headerKeys.TraceState]
});
return this._start().then((result) => {
this.devDebug("result:", result);
return result;
}).finally(() => {
span.end();
});
});
});
});
}
return this.execution;
}
addMetadata(stepId, kind, scope, op, values) {
if (!this.state.metadata) this.state.metadata = /* @__PURE__ */ new Map();
const updates = this.state.metadata.get(stepId) ?? [];
updates.push({
kind,
scope,
op,
values
});
this.state.metadata.set(stepId, updates);
return true;
}
/**
* Starts execution of the user's function and the core loop.
*/
async _start() {
if (this.options.stepMode === StepMode.Sync && this.options.acceptsSse) this.earlyStreamResponse = createDeferredPromise();
const coreLoop = this.runCoreLoop();
if (this.earlyStreamResponse) {
coreLoop.catch((err) => {
this.options.client[internalLoggerSymbol].error({ err }, "Core loop rejected after early stream response was sent");
});
return Promise.race([this.earlyStreamResponse.promise, coreLoop]);
}
return coreLoop;
}
/**
* The core checkpoint loop: processes checkpoints until a handler returns
* a result.
*/
async runCoreLoop() {
try {
const allCheckpointHandler = this.getCheckpointHandler("");
await this.startExecution();
let i = 0;
for await (const checkpoint of this.state.loop) {
await allCheckpointHandler(checkpoint, i);
const result = await this.getCheckpointHandler(checkpoint.type)(checkpoint, i++);
if (result) return result;
}
} catch (error) {
if (this.earlyStreamResponse) {
await this.streamCloseFailed("Internal execution error");
const result = this.transformOutput({ error });
this.earlyStreamResponse.resolve(result);
return result;
}
return this.transformOutput({ error });
} finally {
this.state.loop.return();
}
/**
* If we're here, the generator somehow finished without returning a value.
* This should never happen.
*/
throw new Error("Core loop finished without returning a value");
}
async checkpoint(steps) {
const lazyOps = this.state.lazyOps.drain();
if (lazyOps.length > 0) steps = [...steps, ...lazyOps];
if (this.options.stepMode === StepMode.Sync) if (!this.state.checkpointedRun) {
const res = await retryWithBackoff(() => this.options.client["inngestApi"].checkpointNewRun({
runId: this.fnArg.runId,
event: this.fnArg.event,
steps,
executionVersion: this.version,
retries: this.fnArg.maxAttempts ?? defaultMaxRetries
}), CHECKPOINT_RETRY_OPTIONS);
this.state.checkpointedRun = {
appId: res.data.app_id,
fnId: res.data.fn_id,
token: res.data.token,
realtimeToken: res.data.realtime_token
};
this.sendRedirectIfReady();
} else await retryWithBackoff(() => this.options.client["inngestApi"].checkpointSteps({
appId: this.state.checkpointedRun.appId,
fnId: this.state.checkpointedRun.fnId,
runId: this.fnArg.runId,
steps
}), CHECKPOINT_RETRY_OPTIONS);
else if (this.options.stepMode === StepMode.AsyncCheckpointing) {
const { internalFnId, queueItemId } = this.options;
if (!queueItemId) throw new Error("Missing queueItemId for async checkpointing. This is a bug in the Inngest SDK.");
if (!internalFnId) throw new Error("Missing internalFnId for async checkpointing. This is a bug in the Inngest SDK.");
await retryWithBackoff(() => this.options.client["inngestApi"].checkpointStepsAsync({
runId: this.fnArg.runId,
fnId: internalFnId,
queueItemId,
steps
}), CHECKPOINT_RETRY_OPTIONS);
} else throw new Error("Checkpointing is only supported in Sync and AsyncCheckpointing step modes. This is a bug in the Inngest SDK.");
}
async checkpointAndSwitchToAsync(steps, stepError) {
await this.checkpoint(steps);
if (!this.state.checkpointedRun?.token) throw new Error("Failed to checkpoint and switch to async mode");
const token = this.state.checkpointedRun.token;
if (this.streamTools.activated) if (stepError && !this.retriability(stepError)) await this.streamCloseFailed(errorMessage(stepError));
else await this.streamEnd();
else if (this.options.acceptsSse) {
await this.streamEnd();
return {
type: "function-resolved",
ctx: this.fnArg,
ops: this.ops,
data: this.buildSyncSseResponse()
};
}
return {
type: "change-mode",
ctx: this.fnArg,
ops: this.ops,
to: StepMode.Async,
token
};
}
/**
* Prepend the `inngest.metadata` SSE event to the stream's readable side.
* The returned stream can be used as a fetch body or Response body.
*
* NOTE: `this.streamTools.readable` can only be consumed once, so only one
* of `buildSyncSseResponse` or `postCheckpointStream` may be called per
* execution.
*/
buildMetadataPrefixedStream() {
const metadataEvent = buildSseMetadataEvent(this.fnArg.runId);
return prependToStream(new TextEncoder().encode(metadataEvent), this.streamTools.readable);
}
/**
* Build the initial SSE `Response` that marks the start of streaming to the
* client. Only used in sync mode. In async mode, the stream is POSTed to the
* Inngest Server via {@link postCheckpointStream} instead.
*
* The response body is the stream's readable side, prefixed with the
* `inngest.metadata` SSE event.
*/
buildSyncSseResponse() {
return new Response(this.buildMetadataPrefixedStream(), {
status: 200,
headers: {
"Content-Type": "text/event-stream",
"Cache-Control": "no-cache"
}
});
}
/**
* Wraps a plain return value as an SSE Response.
*
* Used when the client sent `Accept: text/event-stream` but
* `stream.push()`/`pipe()` was NOT called during execution. The
* checkpointable data is the function's return value. The SSE events are just
* a delivery mechanism.
*/
async wrapResultAsSse(checkpoint, sseResponse) {
const resultData = checkpoint.data;
await this.streamCloseSucceeded(sseResponse);
const clientResponse = this.buildSyncSseResponse();
const streamingResult = {
...this.transformOutput({ data: resultData }),
data: clientResponse
};
this.checkpointReturnValue(resultData);
return streamingResult;
}
/**
* Called when `stream.push()`/`pipe()` is first invoked during sync
* execution. Resolves {@link earlyStreamResponse} so that `_start()` can
* return the SSE Response to the HTTP layer immediately, while the core
* checkpoint loop keeps running steps in the background.
*/
handleStreamActivated() {
if (this.earlyStreamResponse) {
this.earlyStreamResponse.resolve({
type: "function-resolved",
ctx: this.fnArg,
ops: this.ops,
data: this.buildSyncSseResponse()
});
this.sendRedirectIfReady();
return;
}
if (this.options.stepMode !== StepMode.Sync) this.postCheckpointStream();
this.sendRedirectIfReady();
}
/**
* Sends the `inngest.redirect_info` SSE event when both conditions are met:
* 1. The client accepts SSE (so there's a stream to write the event to)
* 2. We have a realtime token (first checkpoint has completed)
*
* Called after the first checkpoint AND on stream activation, whichever
* comes second, so the redirect is sent as early as possible.
*/
sendRedirectIfReady() {
if (this.redirectSent) return;
if (!this.options.acceptsSse) return;
if (!this.state.checkpointedRun) return;
this.redirectSent = true;
const { realtimeToken } = this.state.checkpointedRun;
this.redirectPromise = (async () => {
try {
const redirect = await this.options.client["inngestApi"].getRealtimeStreamRedirect(realtimeToken);
this.streamTools.sendRedirectInfo({
runId: this.fnArg.runId,
url: redirect.url
});
} catch (err) {
this.options.client[internalLoggerSymbol].warn({ err }, "Failed to fetch realtime stream redirect URL");
}
})();
}
/**
* Await the pending redirect-info fetch, then close the stream with a
* succeeded result. Awaiting first guarantees the redirect event is
* enqueued on the write chain before the close event.
*/
async streamCloseSucceeded(response) {
await this.redirectPromise;
this.streamTools.closeSucceeded(response);
}
/**
* Await the pending redirect-info fetch, then close the stream with a
* failed result.
*/
async streamCloseFailed(error) {
await this.redirectPromise;
this.streamTools.closeFailed(error);
}
/**
* Await the pending redirect-info fetch, then close the stream without
* a result event.
*/
async streamEnd() {
await this.redirectPromise;
this.streamTools.end();
}
/**
* POST stream data to the checkpoint stream ingest endpoint.
*
* Called eagerly from handleStreamActivated so chunks flow in
* real-time, or after completion if stream.push() was never called.
*/
postCheckpointStream() {
try {
this.options.client["inngestApi"].checkpointStream({
runId: this.fnArg.runId,
body: this.buildMetadataPrefixedStream()
}).catch((err) => {
this.devDebug("checkpoint stream POST error:", err);
});
} catch (err) {
this.devDebug("checkpoint stream POST error:", err);
}
}
/**
* Checkpoints the return value of a function that was delivered via SSE.
* Runs in the background so it doesn't block the client stream.
*/
async checkpointReturnValue(data) {
try {
if (this.options.createResponse) await this.checkpoint([{
op: StepOpCode.RunComplete,
id: hashId(RUN_COMPLETE_STEP_ID),
data: await this.options.createResponse(jsonResponse(data))
}]);
} catch (err) {
this.devDebug("error during background checkpoint of SSE result, client stream unaffected:", err);
}
}
/**
* Creates a handler for every checkpoint type, defining what to do when we
* reach that checkpoint in the core loop.
*/
createCheckpointHandlers() {
const commonCheckpointHandler = (checkpoint) => {
this.devDebug(`${this.options.stepMode} checkpoint:`, checkpoint);
};
const stepRanHandler = async (stepResult) => {
const transformResult = await this.transformOutput(stepResult);
/**
* Transforming output will always return either function rejection or
* resolution. In most cases, this can be immediately returned, but in
* this particular case we want to handle it differently.
*/
if (transformResult.type === "function-resolved") return {
type: "step-ran",
ctx: transformResult.ctx,
ops: transformResult.ops,
step: {
...stepResult,
data: transformResult.data
}
};
else if (transformResult.type === "function-rejected") {
const stepForResponse = {
...stepResult,
error: transformResult.error
};
if (stepResult.op === StepOpCode.StepFailed) {
const ser = serializeError(transformResult.error);
stepForResponse.data = {
__serialized: true,
name: ser.name,
message: ser.message,
stack: ""
};
}
return {
type: "step-ran",
ctx: transformResult.ctx,
ops: transformResult.ops,
retriable: transformResult.retriable,
step: stepForResponse
};
}
return transformResult;
};
const maybeReturnNewSteps = async () => {
const allSteps = [...await this.filterNewSteps(Array.from(this.state.steps.values())) ?? [], ...this.state.lazyOps.drain()];
if (allSteps.length === 0) return;
return {
type: "steps-found",
ctx: this.fnArg,
ops: this.ops,
steps: allSteps
};
};
const attemptCheckpointAndResume = async (stepResult, resume = true, force = false) => {
if (stepResult) {
const stepToResume = this.resumeStepWithResult(stepResult, resume);
delete this.state.executingStep;
this.state.checkpointingStepBuffer.push({
...stepToResume,
data: stepResult.data
});
}
if (force || !this.options.checkpointingConfig?.bufferedSteps || this.state.checkpointingStepBuffer.length >= this.options.checkpointingConfig.bufferedSteps) {
this.devDebug("checkpointing and resuming execution after step run");
try {
this.devDebug(`checkpointing all buffered steps:`, this.state.checkpointingStepBuffer.map((op) => op.displayName || op.id).join(", "));
await this.checkpoint(this.state.checkpointingStepBuffer);
return;
} catch (err) {
this.devDebug("error checkpointing after step run, so falling back to async", err);
const buffered = this.state.checkpointingStepBuffer;
if (buffered.length) return {
type: "steps-found",
ctx: this.fnArg,
ops: this.ops,
steps: buffered
};
return;
} finally {
this.state.checkpointingStepBuffer = [];
}
} else this.devDebug(`not checkpointing yet, continuing execution as we haven't reached buffered step limit of ${this.options.checkpointingConfig?.bufferedSteps}`);
};
const syncHandlers = {
"": commonCheckpointHandler,
"function-resolved": async (checkpoint) => {
const usingSseStream = !!this.earlyStreamResponse;
if (this.streamTools.activated) {
let resultData = checkpoint.data;
let sseResponse;
if (checkpoint.data instanceof Response) {
const body = await (usingSseStream ? checkpoint.data.text() : checkpoint.data.clone().text());
sseResponse = extractSseResponse(checkpoint.data, body);
resultData = body;
} else sseResponse = defaultSseResponse(resultData);
await this.streamCloseSucceeded(sseResponse);
if (usingSseStream) {
this.checkpointReturnValue(resultData);
return this.transformOutput({ data: resultData });
}
}
if (this.options.acceptsSse) {
let sseResponse;
if (checkpoint.data instanceof Response) {
const body = await checkpoint.data.text();
sseResponse = extractSseResponse(checkpoint.data, body);
checkpoint = {
...checkpoint,
data: body
};
} else sseResponse = defaultSseResponse(checkpoint.data);
return this.wrapResultAsSse(checkpoint, sseResponse);
}
if (checkpoint.data instanceof Response) {
this.checkpointReturnValue(null);
return this.transformOutput({ data: checkpoint.data });
}
await this.checkpoint([{
op: StepOpCode.RunComplete,
id: hashId(RUN_COMPLETE_STEP_ID),
data: await this.options.createResponse(jsonResponse(checkpoint.data))
}]);
return this.transformOutput({ data: checkpoint.data });
},
"function-rejected": async (checkpoint) => {
const usingSseStream = !!this.earlyStreamResponse;
const isFinal = !this.retriability(checkpoint.error);
if (this.streamTools.activated && usingSseStream) {
(async () => {
try {
await this.checkpoint([{
id: hashId(RUN_COMPLETE_STEP_ID),
op: isFinal ? StepOpCode.StepFailed : StepOpCode.StepError,
error: checkpoint.error
}]);
} catch (err) {
this.options.client[internalLoggerSymbol].warn({ err }, "Failed to checkpoint function error");
}
if (isFinal) await this.streamCloseFailed(errorMessage(checkpoint.error));
else await this.streamEnd();
})();
return this.transformOutput({ error: checkpoint.error });
}
if (isFinal) return this.transformOutput({ error: checkpoint.error });
return this.checkpointAndSwitchToAsync([{
id: hashId(RUN_COMPLETE_STEP_ID),
op: StepOpCode.StepError,
error: checkpoint.error
}]);
},
"step-not-found": () => {
return {
type: "function-rejected",
ctx: this.fnArg,
error: /* @__PURE__ */ new Error("Step not found when checkpointing; this should never happen"),
ops: this.ops,
retriable: false
};
},
"steps-found": async ({ steps }) => {
if (steps.length !== 1 || steps[0].mode !== StepMode.Sync) return this.checkpointAndSwitchToAsync(steps.map((step) => ({
...step,
id: step.hashedId
})));
const result = await this.executeStep(steps[0]);
const transformed = await stepRanHandler(result);
if (transformed.type !== "step-ran") throw new Error("Unexpected checkpoint handler result type after running step in sync mode");
if (result.error) return this.checkpointAndSwitchToAsync([transformed.step], result.error);
const stepToResume = this.resumeStepWithResult(result);
delete this.state.executingStep;
const stepForCheckpoint = {
...stepToResume,
data: transformed.step.data
};
await this.checkpoint([stepForCheckpoint]);
},
"checkpointing-runtime-reached": () => {
return this.checkpointAndSwitchToAsync([{
op: StepOpCode.DiscoveryRequest,
id: _internals.hashId(`discovery-request-${Date.now()}`)
}]);
},
"checkpointing-buffer-interval-reached": () => {
return attemptCheckpointAndResume(void 0, false, true);
}
};
const asyncHandlers = {
"": commonCheckpointHandler,
"function-resolved": async ({ data }) => {
let resultData = data;
let sseResponse;
if (data instanceof Response) {
const body = await data.text();
sseResponse = extractSseResponse(data, body);
resultData = body;
} else sseResponse = defaultSseResponse(resultData);
const newSteps = await this.filterNewSteps(Array.from(this.state.steps.values()));
if (newSteps?.length) return this.attachLazyOps({
type: "steps-found",
ctx: this.fnArg,
ops: this.ops,
steps: newSteps
});
await this.streamCloseSucceeded(sseResponse);
if (!this.streamTools.activated) this.postCheckpointStream();
if (this.options.createResponse) data = await this.options.createResponse(jsonResponse(resultData));
return this.attachLazyOps(this.transformOutput({ data }));
},
"function-rejected": async (checkpoint) => {
if (!this.retriability(checkpoint.error)) await this.streamCloseFailed(errorMessage(checkpoint.error));
else await this.streamEnd();
if (!this.streamTools.activated) this.postCheckpointStream();
return this.attachLazyOps(this.transformOutput({ error: checkpoint.error }));
},
"steps-found": async ({ steps }) => {
const stepResult = await this.tryExecuteStep(steps);
if (!stepResult) return maybeReturnNewSteps();
if (this.state.lazyOps.length === 0) return stepRanHandler(stepResult);
const transformed = await stepRanHandler(stepResult);
if (transformed.type !== "step-ran") return transformed;
return this.attachLazyOps({
type: "steps-found",
ctx: transformed.ctx,
ops: transformed.ops,
steps: [transformed.step]
});
},
"step-not-found": ({ step }) => {
const { foundSteps, totalFoundSteps } = this.getStepNotFoundDetails();
return {
type: "step-not-found",
ctx: this.fnArg,
ops: this.ops,
step,
foundSteps,
totalFoundSteps
};
},
"checkpointing-runtime-reached": () => {
throw new Error("Checkpointing maximum runtime reached, but this is not in a checkpointing step mode. This is a bug in the Inngest SDK.");
},
"checkpointing-buffer-interval-reached": () => {
throw new Error("Checkpointing maximum buffer interval reached, but this is not in a checkpointing step mode. This is a bug in the Inngest SDK.");
}
};
const asyncCheckpointingHandlers = {
"": commonCheckpointHandler,
"function-resolved": async (checkpoint, i) => {
const lazyOps = this.state.lazyOps.drain();
const output = await asyncHandlers["function-resolved"](checkpoint, i);
if (output?.type === "function-resolved") {
const steps = [
...this.state.checkpointingStepBuffer,
...lazyOps,
{
op: StepOpCode.RunComplete,
id: hashId(RUN_COMPLETE_STEP_ID),
data: output.data
}
];
if (isNonEmpty(steps)) return {
type: "steps-found",
ctx: output.ctx,
ops: output.ops,
steps
};
}
if (output?.type === "steps-found" && lazyOps.length) return {
...output,
steps: [...output.steps, ...lazyOps]
};
return output;
},
"function-rejected": async (checkpoint) => {
if (this.state.checkpointingStepBuffer.length || this.state.lazyOps.length > 0) {
const fallback = await attemptCheckpointAndResume(void 0, false, true);
if (fallback) return fallback;
}
return await this.transformOutput({ error: checkpoint.error });
},
"step-not-found": asyncHandlers["step-not-found"],
"steps-found": async ({ steps }) => {
const { stepsToResume, newSteps } = steps.reduce((acc, step) => {
if (!step.hasStepState) acc.newSteps.push(step);
else if (!step.fulfilled) acc.stepsToResume.push(step);
return acc;
}, {
stepsToResume: [],
newSteps: []
});
this.devDebug("split found steps in to:", {
stepsToResume: stepsToResume.length,
newSteps: newSteps.length
});
if (!this.options.requestedRunStep && newSteps.length) {
if (this.state.checkpointingRuntimeExceeded) {
if (this.state.checkpointingStepBuffer.length) {
const fallback = await attemptCheckpointAndResume(void 0, false, true);
if (fallback) return fallback;
}
return maybeReturnNewSteps();
}
const stepResult = await this.tryExecuteStep(newSteps);
if (stepResult) {
this.devDebug(`executed step "${stepResult.id}" successfully`);
if (stepResult.error) {
if (this.state.checkpointingStepBuffer.length) {
const fallback = await attemptCheckpointAndResume(void 0, false, true);
if (fallback) return fallback;
}
return stepRanHandler(stepResult);
}
return await attemptCheckpointAndResume(stepResult);
}
if (this.state.checkpointingStepBuffer.length) {
const fallback = await attemptCheckpointAndResume(void 0, false, true);
if (fallback) return fallback;
}
return maybeReturnNewSteps();
}
if (stepsToResume.length) {
this.devDebug(`resuming ${stepsToResume.length} steps`);
for (const st of stepsToResume) this.resumeStepWithResult({
...st,
id: st.hashedId
});
}
},
"checkpointing-runtime-reached": async () => {
this.state.checkpointingRuntimeExceeded = true;
},
"checkpointing-buffer-interval-reached": () => {
return attemptCheckpointAndResume(void 0, false, true);
}
};
return {
[StepMode.Async]: asyncHandlers,
[StepMode.Sync]: syncHandlers,
[StepMode.AsyncCheckpointing]: asyncCheckpointingHandlers
};
}
getCheckpointHandler(type) {
return this.checkpointHandlers[this.options.stepMode][type];
}
async tryExecuteStep(steps) {
const hashedStepIdToRun = this.options.requestedRunStep || this.getEarlyExecRunStep(steps);
if (!hashedStepIdToRun) return;
const step = steps.find((step$1) => step$1.hashedId === hashedStepIdToRun && step$1.fn);
if (step) return await this.executeStep(step);
this.timeout?.reset();
}
/**
* Given a list of outgoing ops, decide if we can execute an op early and
* return the ID of the step to execute if we can.
*/
getEarlyExecRunStep(steps) {
/**
* We may have been disabled due to parallelism, in which case we can't
* immediately execute unless explicitly requested.
*/
if (this.options.disableImmediateExecution) return;
const unfulfilledSteps = steps.filter((step) => !step.fulfilled);
if (unfulfilledSteps.length !== 1) return;
const op = unfulfilledSteps[0];
if (op && op.op === StepOpCode.StepPlanned) return op.hashedId;
}
async filterNewSteps(foundSteps) {
if (this.options.requestedRunStep) return;
const newSteps = foundSteps.reduce((acc, step) => {
if (!step.hasStepState) acc.push(step);
return acc;
}, []);
if (!newSteps.length) return;
await this.middlewareManager.onMemoizationEnd();
const stepList = newSteps.map((step) => {
return {
displayName: step.displayName,
op: step.op,
id: step.hashedId,
name: step.name,
opts: step.opts,
userland: step.userland
};
});
if (!isNonEmpty(stepList)) throw new UnreachableError("stepList is empty");
return stepList;
}
async executeStep(foundStep) {
const { id, name, opts, fn, displayName, userland, hashedId } = foundStep;
const { stepInfo, wrappedHandler, setActualHandler } = foundStep.middleware;
this.devDebug(`preparing to execute step "${id}"`);
this.timeout?.clear();
const outgoingOp = {
id: hashedId,
op: StepOpCode.StepRun,
name,
opts,
displayName,
userland
};
this.state.executingStep = outgoingOp;
const store = await getAsyncCtx();
if (store?.execution) store.execution.executingStep = {
id,
name: displayName,
hashedId
};
this.devDebug(`executing step "${id}"`);
if (this.rootSpanId && this.options.checkpointingConfig) clientProcessorMap.get(this.options.client)?.declareStepExecution(this.rootSpanId, userland.id ?? "", userland.index ?? 0, hashedId, this.options.data?.attempt ?? 0);
let interval;
const actualHandler = () => runAsPromise(fn);
await this.middlewareManager.onMemoizationEnd();
await this.middlewareManager.onStepStart(stepInfo);
if (!foundStep.memoizationDeferred) {
const deferred = createDeferredPromise();
foundStep.memoizationDeferred = deferred;
setActualHandler(() => deferred.promise);
foundStep.transformedResultPromise = wrappedHandler();
foundStep.transformedResultPromise.catch(() => {});
}
const wrappedActualHandler = this.middlewareManager.buildWrapStepHandlerChain(actualHandler, stepInfo);
return goIntervalTiming(() => wrappedActualHandler()).finally(() => {
this.devDebug(`finished executing step "${id}"`);
this.state.executingStep = void 0;
if (this.rootSpanId && this.options.checkpointingConfig) clientProcessorMap.get(this.options.client)?.clearStepExecution(this.rootSpanId);
if (store?.execution) delete store.execution.executingStep;
}).then(async ({ resultPromise, interval: _interval }) => {
interval = _interval;
const metadata = this.state.metadata?.get(id);
const serverData = await resultPromise;
await this.middlewareManager.onStepComplete(stepInfo, serverData);
this.streamTools.commit(hashedId);
return {
...outgoingOp,
data: serverData,
...metadata && metadata.length > 0 ? { metadata } : {}
};
}).catch((error) => {
return this.buildStepErrorOp({
error,
id,
outgoingOp,
stepInfo
});
}).then((op) => ({
...op,
timing: interval
}));
}
/**
* Starts execution of the user's function, including triggering checkpoints
* and middleware hooks where appropriate.
*/
async startExecution() {
/**
* Start the timer to time out the run if needed.
*/
this.timeout?.start();
this.checkpointingMaxRuntimeTimer?.start();
this.checkpointingMaxBufferIntervalTimer?.start();
const fnInputResult = await this.middlewareManager.transformFunctionInput();
this.applyFunctionInputMutations(fnInputResult);
if (this.state.allStateUsed()) await this.middlewareManager.onMemoizationEnd();
if (this.state.stepsToFulfill === 0 && this.fnArg.attempt === 0) await this.middlewareManager.onRunStart();
const innerHandler = async () => {
await this.validateEventSchemas();
return this.userFnToRun(this.fnArg);
};
runAsPromise(this.middlewareManager.wrapRunHandler(innerHandler)).then(async (data) => {
await this.middlewareManager.onRunComplete(data);
this.state.setCheckpoint({
type: "function-resolved",
data
});
}).catch(async (error) => {
let err;
if (error instanceof Error) err = error;
else if (typeof error === "object") err = new Error(JSON.stringify(error));
else err = new Error(String(error));
await this.middlewareManager.onRunError(err, !this.retriability(err));
this.state.setCheckpoint({
type: "function-rejected",
error: err
});
});
}
/**
* Determine whether the given error is retriable. Returns `false` when the
* run should not be retried, a duration string for `RetryAfterError`, or
* `true` for normal retry behavior.
*/
retriability(error) {
if (this.fnArg.maxAttempts && this.fnArg.maxAttempts - 1 === this.fnArg.attempt) return false;
if (error instanceof NonRetriableError || error?.name === "NonRetriableError") return false;
if (error instanceof StepError && error === this.state.recentlyRejectedStepError) return false;
if (error instanceof RetryAfterError || error?.name === "RetryAfterError") return error.retryAfter;
return true;
}
/**
* Build the OutgoingOp for a failed step, notifying middleware and choosing
* retriable vs non-retriable opcode.
*/
async buildStepErrorOp({ error, id, outgoingOp, stepInfo }) {
const isFinal = !this.retriability(error);
const metadata = this.state.metadata?.get(id);
await this.middlewareManager.onStepError(stepInfo, error instanceof Error ? error : new Error(String(error)), isFinal);
this.streamTools.rollback(outgoingOp.id);
const serialized = serializeError(error);
return {
...outgoingOp,
error: serialized,
op: isFinal ? StepOpCode.StepFailed : StepOpCode.StepError,
...metadata && metadata.length > 0 ? { metadata } : {}
};
}
/**
* Validate event data against schemas defined in function triggers.
*/
async validateEventSchemas() {
if (this.options.handlerKind === "failure") return;
if (this.options.handlerKind === "defer") {
await this.validateDeferEventSchema();
return;
}
const triggers = this.options.fn.opts.triggers;
if (!triggers || triggers.length === 0) return;
const fnArgEvents = this.fnArg.events;
if (!fnArgEvents || fnArgEvents.length === 0) return;
await validateEvents(fnArgEvents.map((event) => ({
name: event.name,
data: event.data
})), triggers);
}
/**
* Validate the deferred event's data against the defer function's own
* schema (set via `createDefer`'s `opts.schema`).
*/
async validateDeferEventSchema() {
const fn = this.options.fn;
if (!isDeferredFunction(fn) || !fn.schema) return;
const eventData = this.fnArg.event?.data;
const result = await fn.schema["~standard"].validate(eventData);
if (result.issues) throw new NonRetriableError(`defer handler "${fn.id(this.options.client.id)}" schema validation failed: ${JSON.stringify(result.issues)}`);
}
/**
* Using middleware, transform output before returning.
*/
transformOutput(dataOrError) {
const { data, error } = dataOrError;
if (typeof error !== "undefined") {
const retriable = this.retriability(error);
const serializedError = serializeError(error);
return {
type: "function-rejected",
ctx: this.fnArg,
ops: this.ops,
error: serializedError,
retriable
};
}
return {
type: "function-resolved",
ctx: this.fnArg,
ops: this.ops,
data: undefinedToNull(data)
};
}
/**
* Drain buffered lazy ops (e.g. `DeferAdd` from `defer()`) and merge them
* into `result` so they ship in the same outbound message. Lazy ops are
* fire-and-forget and have no natural shipping moment, so each terminal code
* path must ship them or they're silently dropped.
*/
attachLazyOps(result, extras = []) {
const lazyOps = this.state.lazyOps.drain();
if (lazyOps.length === 0 && extras.length === 0) return result;
switch (result.type) {
case "function-resolved": {
const steps = [
...extras,
...lazyOps,
{
op: StepOpCode.RunComplete,
id: hashId(RUN_COMPLETE_STEP_ID),
data: undefinedToNull(result.data)
}
];
return {
type: "steps-found",
ctx: result.ctx,
ops: result.ops,
steps
};
}
case "function-rejected": {
const isFinal = result.retriable === false;
const steps = [
...extras,
...lazyOps,
{
op: isFinal ? StepOpCode.StepFailed : StepOpCode.StepError,
id: hashId(RUN_COMPLETE_STEP_ID),
error: result.error
}
];
return {
type: "steps-found",
ctx: result.ctx,
ops: result.ops,
steps
};
}
case "steps-found": return {
...result,
steps: [
...result.steps,
...extras,
...lazyOps
]
};
default:
for (const op of lazyOps) this.state.lazyOps.push(op);
return result;
}
}
createExecutionState() {
const d = createDeferredPromiseWithStack();
let checkpointResolve = d.deferred.resolve;
const checkpointResults = d.results;
const loop = (async function* (cleanUp) {
try {
while (true) {
const res = (await checkpointResults.next()).value;
if (res) yield res;
}
} finally {
cleanUp?.();
}
})(() => {
this.timeout?.clear();
this.checkpointingMaxRuntimeTimer?.clear();
this.checkpointingMaxBufferIntervalTimer?.clear();
checkpointResults.return();
});
const stepsToFulfill = Object.keys(this.options.stepState).length;
return {
stepState: this.options.stepState,
priorDefers: this.options.priorDefers ?? {},
stepsToFulfill,
steps: /* @__PURE__ */ new Map(),
loop,
hasSteps: Boolean(stepsToFulfill),
stepCompletionOrder: [...this.options.stepCompletionOrder],
remainingStepsToBeSeen: new Set(this.options.stepCompletionOrder),
setCheckpoint: (checkpoint) => {
this.devDebug("setting checkpoint:", checkpoint.type);
({resolve: checkpointResolve} = checkpointResolve(checkpoint));
},
allStateUsed: () => {
return this.state.remainingStepsToBeSeen.size === 0;
},
checkpointingStepBuffer: [],
lazyOps: new LazyOps(),
metadata: /* @__PURE__ */ new Map()
};
}
get ops() {
return Object.fromEntries(this.state.steps);
}
createFnArg() {
const { step, defer } = this.createStepTools();
const experimentStepRun = step[experimentStepRunSymbol];
let fnArg = {
...this.options.data,
step,
group: createGroupTools({ experimentStepRun }),
defer
};
if (this.options.handlerKind === "defer") {
delete fnArg.event.data._inngest;
for (const event of fnArg.events) delete event.data._inngest;
}
/**
* Handle use of the `onFailure` option by deserializing the error.
*/
if (this.options.handlerKind === "failure") {
const eventData = z.object({ error: jsonErrorSchema }).parse(fnArg.event?.data);
fnArg = {
...fnArg,
error: deserializeError(eventData.error)
};
}
return this.options.transformCtx?.(fnArg) ?? fnArg;
}
/**
* Apply mutations from `transformFunctionInput` back to execution state.
* Allows middleware to modify event data, step tools, memoized step data,
* and inject custom fields into the handler context.
*/
applyFunctionInputMutations(result) {
const { event, events, step, ...extensions } = result.ctx;
if (event !== this.fnArg.event) this.fnArg.event = event;
if (events !== this.fnArg.events) this.fnArg.events = events;
if (step !== this.fnArg.step) this.fnArg.step = step;
if (Object.keys(extensions).length > 0) Object.assign(this.fnArg, extensions);
for (const [hashedId, stepData] of Object.entries(result.steps)) {
const existing = this.state.stepState[hashedId];
if (existing && stepData && stepData.type === "data" && stepData.data !== existing.data) this.state.stepState[hashedId] = {
...existing,
data: stepData.data
};
}
}
createStepTools() {
/**
* A list of steps that have been found and are being rolled up before being
* reported to the core loop.
*/
const foundStepsToReport = /* @__PURE__ */ new Map();
/**
* A map of the subset of found steps to report that have not yet been
* handled. Used for fast access to steps that need to be handled in order.
*/
const unhandledFoundStepsToReport = /* @__PURE__ */ new Map();
/**
* A map of the latest sequential step indexes found for each step ID. Used
* to ensure that we don't index steps in parallel.
*
* Note that these must be sequential; if we've seen or assigned `a:1`,
* `a:2` and `a:4`, the latest sequential step index is `2`.
*
*/
const expectedNextStepIndexes = /* @__PURE__ */ new Map();
/**
* An ordered list of step IDs that have yet to be handled in this
* execution. Used to ensure that we handle steps in the order they were
* found and based on the `stepCompletionOrder` in this execution's state.
*/
const remainingStepCompletionOrder = this.state.stepCompletionOrder.slice();
/**
* A promise that's used to ensure that step reporting cannot be run more than
* once in a given asynchronous time span.
*/
let foundStepsReportPromise;
/**
* A flag used to ensure that we only warn about parallel indexing once per
* execution to avoid spamming the console.
*/
let warnOfParallelIndexing = false;
/**
* Counts the number of times we've extended this tick.
*/
let tickExtensionCount = 0;
/**
* Given a colliding step ID, maybe warn the user about parallel indexing.
*/
const maybeWarnOfParallelIndexing = (userlandCollisionId) => {
if (warnOfParallelIndexing) return;
const hashedCollisionId = _internals.hashId(userlandCollisionId);
if (this.state.steps.has(hashedCollisionId)) {
if (!foundStepsToReport.has(hashedCollisionId)) {
warnOfParallelIndexing = true;
this.options.client["warnMetadata"]({ run_id: this.fnArg.runId }, ErrCode.AUTOMATIC_PARALLEL_INDEXING, {
message: `Duplicate step ID "${userlandCollisionId}" detected across parallel chains`,
explanation: "Using the same ID for steps in different parallel chains can cause unexpected behaviour. Your function is still running.",
action: "Use a unique ID for each step, especially those in parallel.",
code: ErrCode.AUTOMATIC_PARALLEL_INDEXING
});
}
}
};
/**
* A helper used to report steps to the core loop. Used after adding an item
* to `foundStepsToReport`.
*/
const reportNextTick = () => {
if (foundStepsReportPromise) return;
let extensionPromise;
if (++tickExtensionCount >= 10) {
tickExtensionCount = 0;
extensionPromise = resolveNextTick();
} else extensionPromise = resolveAfterPending();
foundStepsReportPromise = extensionPromise.then(() => {
foundStepsReportPromise = void 0;
for (let i = 0; i < remainingStepCompletionOrder.length; i++) {
const nextStepId = remainingStepCompletionOrder[i];
if (!nextStepId) continue;
if (unhandledFoundStepsToReport.get(nextStepId)?.handle()) {
remainingStepCompletionOrder.splice(i, 1);
unhandledFoundStepsToReport.delete(nextStepId);
reportNextTick();
return;
}
}
const steps = [...foundStepsToReport.values()];
foundStepsToReport.clear();
unhandledFoundStepsToReport.clear();
if (!isNonEmpty(steps)) return;
this.state.setCheckpoint({
type: "steps-found",
steps
});
});
};
/**
* A helper used to push a step to the list of steps to report.
*/
const pushStepToReport = (step) => {
foundStepsToReport.set(step.hashedId, step);
unhandledFoundStepsToReport.set(step.hashedId, step);
reportNextTick();
};
const stepHandler = async ({ args, matchOp, opts }) => {
const opId = matchOp(getStepOptions(args[0]), ...args.slice(1));
if (isLazyOp(opts, opId)) {
const hashedId$1 = _internals.hashId(opId.id);
if (this.state.lazyOps.hasId(hashedId$1)) {
this.options.client[internalLoggerSymbol].warn({
runId: this.fnArg.runId,
id: opId.userland?.id ?? opId.id
}, "defer skipped: duplicate ID within run");
return;
}
if (this.state.priorDefers[hashedId$1]) {
this.state.lazyOps.markSeen(hashedId$1);
return;
}
this.state.lazyOps.push({
id: hashedId$1,
op: opId.op,
name: opId.name,
displayName: opId.displayName ?? opId.id,
opts: opId.opts,
userland: opId.userland,
data: null
});
return;
}
if (this.state.executingStep)
/**
* If a step is found after asynchronous actions during another step's
* execution, everything is fine. The problem here is if we've found
* that a step nested inside another a step, which is something we don't
* support at the time of writing.
*
* In this case, we could use something like Async Hooks to understand
* how the step is being triggered, though this isn't available in all
* environments.
*
* Therefore, we'll only show a warning here to indicate that this is
* potentially an issue.
*/
this.options.client["warnMetadata"]({ run_id: this.fnArg.runId }, ErrCode.NESTING_STEPS, {
message: `Nested step tooling detected in "${opId.displayName ?? opId.id}"`,
explanation: "Nesting step.* calls is not supported. This warning may also appear if steps are separated by regular async calls, which is fine.",
action: "Avoid using step.* inside other step.* calls. Use a separate async function or promise chaining to compose steps.",
code: ErrCode.NESTING_STEPS
});
const { hashedId, isFulfilled, setActualHandler, stepInfo, stepState, wrappedHandler } = await this.applyMiddlewareToStep(opId, expectedNextStepIndexes, maybeWarnOfParallelIndexing);
const { promise, resolve, reject } = createDeferredPromise();
let extraOpts;
let fnArgs = [...args];
if (typeof stepState?.input !== "undefined" && Array.isArray(stepState.input)) switch (opId.op) {
case StepOpCode.StepPlanned:
fnArgs = [...args.slice(0, 2), ...stepState.input];
extraOpts = { input: [...stepState.input] };
break;
case StepOpCode.AiGateway:
extraOpts = { body: {
...typeof opId.opts?.body === "object" ? { ...opId.opts.body } : {},
...stepState.input[0]
} };
break;
}
if (!extraOpts && Array.isArray(stepInfo.input)) fnArgs = [...args.slice(0, 2), ...stepInfo.input];
const step = {
...opId,
opts: {
...opId.opts,
...extraOpts
},
rawArgs: fnArgs,
hashedId,
input: stepState?.input,
fn: opts?.fn ? () => opts.fn?.(this.fnArg, ...fnArgs) : void 0,
promise,
fulfilled: isFulfilled,
hasStepState: Boolean(stepState),
displayName: opId.displayName ?? opId.id,
handled: false,
middleware: {
wrappedHandler,
stepInfo,
setActualHandler
},
handle: () => {
if (step.handled) return false;
this.devDebug(`handling step "${hashedId}"`);
step.handled = true;
const result = this.state.stepState[hashedId];
if (step.fulfilled && result) {
result.fulfilled = true;
Promise.all([
result.data,
result.error,
result.input
]).then(async () => {
if (step.transformedResultPromise) {
if (step.memoizationDeferred) if (typeof result.data !== "undefined") step.memoizationDeferred.resolve(await result.data);
else {
const stepError = new StepError(opId.id, result.error);
this.state.recentlyRejectedStepError = stepError;
step.memoizationDeferre