agents/dist/retries.js

A home for your AI agents

//#region src/retries.ts
/**
* Validate retry options eagerly so invalid config fails at enqueue/schedule time
* rather than at execution time. Checks individual field ranges, enforces integer
* maxAttempts, and validates cross-field constraints after resolving against
* defaults when provided.
*/
/** Fill unset per-call retry options from a required set of defaults. */
function resolveRetryConfig(retry, defaults) {
	return {
		maxAttempts: retry?.maxAttempts ?? defaults.maxAttempts,
		baseDelayMs: retry?.baseDelayMs ?? defaults.baseDelayMs,
		maxDelayMs: retry?.maxDelayMs ?? defaults.maxDelayMs
	};
}
function validateRetryOptions(options, defaults) {
	if (options.maxAttempts !== void 0) {
		if (!Number.isFinite(options.maxAttempts) || options.maxAttempts < 1) throw new Error("retry.maxAttempts must be >= 1");
		if (!Number.isInteger(options.maxAttempts)) throw new Error("retry.maxAttempts must be an integer");
	}
	if (options.baseDelayMs !== void 0) {
		if (!Number.isFinite(options.baseDelayMs) || options.baseDelayMs <= 0) throw new Error("retry.baseDelayMs must be > 0");
	}
	if (options.maxDelayMs !== void 0) {
		if (!Number.isFinite(options.maxDelayMs) || options.maxDelayMs <= 0) throw new Error("retry.maxDelayMs must be > 0");
	}
	const resolvedBase = options.baseDelayMs ?? defaults?.baseDelayMs;
	const resolvedMax = options.maxDelayMs ?? defaults?.maxDelayMs;
	if (resolvedBase !== void 0 && resolvedMax !== void 0 && resolvedBase > resolvedMax) throw new Error("retry.baseDelayMs must be <= retry.maxDelayMs");
}
/**
* Returns the number of milliseconds to wait before retrying a request.
* Uses the "Full Jitter" approach from
* https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/
*
* @param attempt The current attempt number (1-indexed).
* @param baseDelayMs Base delay multiplier in ms.
* @param maxDelayMs Maximum delay cap in ms.
* @returns Milliseconds to wait before retrying.
*/
function jitterBackoff(attempt, baseDelayMs, maxDelayMs) {
	const upperBoundMs = Math.min(2 ** attempt * baseDelayMs, maxDelayMs);
	return Math.floor(Math.random() * upperBoundMs);
}
/**
* Retry an async function up to `n` total attempts with jittered exponential backoff.
*
* @param n Total number of attempts (must be a finite integer >= 1).
* @param fn The async function to retry. Receives the current attempt number (1-indexed).
* @param options Retry configuration.
* @returns The result of `fn` on success.
* @throws The last error if all attempts fail or `shouldRetry` returns false.
*/
async function tryN(n, fn, options) {
	if (!Number.isFinite(n) || n < 1) throw new Error("retry.maxAttempts must be >= 1");
	n = Math.floor(n);
	const rawBase = options?.baseDelayMs ?? 100;
	const rawMax = options?.maxDelayMs ?? 3e3;
	if (!Number.isFinite(rawBase) || rawBase <= 0) throw new Error("retry.baseDelayMs must be > 0");
	if (!Number.isFinite(rawMax) || rawMax <= 0) throw new Error("retry.maxDelayMs must be > 0");
	const baseDelayMs = Math.floor(rawBase);
	const maxDelayMs = Math.floor(rawMax);
	if (baseDelayMs > maxDelayMs) throw new Error("retry.baseDelayMs must be <= retry.maxDelayMs");
	let attempt = 1;
	while (true) try {
		return await fn(attempt);
	} catch (err) {
		const nextAttempt = attempt + 1;
		if (nextAttempt > n || options?.shouldRetry && !options.shouldRetry(err, nextAttempt)) throw err;
		const delay = jitterBackoff(attempt, baseDelayMs, maxDelayMs);
		await new Promise((resolve) => setTimeout(resolve, delay));
		attempt = nextAttempt;
	}
}
/**
* Returns true if the given error is retryable according to Durable Object error handling.
* See https://developers.cloudflare.com/durable-objects/best-practices/error-handling/
*
* An error is retryable if it has `retryable: true` but is NOT an overloaded error.
*/
function isErrorRetryable(err) {
	if (typeof err !== "object" || err === null) return false;
	const msg = String(err);
	const typed = err;
	return Boolean(typed.retryable) && !typed.overloaded && !msg.includes("Durable Object is overloaded");
}
/**
* The "superseded isolate" platform messages — the invocation is running on an
* isolate the platform has replaced with a new version (a deploy / code
* update). For the rest of that invocation every operation throws the same
* error (code never reloads mid-invocation), so in-process retries are futile;
* but the next fresh invocation runs the new code and succeeds.
*
* workerd surfaces this as a plain `Error` with one of a few messages, all the
* same failure class — a message match is the only signal:
*   - "Durable Object reset because its code was updated."  (DO storage op on a
*     superseded isolate / deploy bounce)
*   - "This script has been upgraded. Please send a new request to connect to
*     the new version."  (a stub/connection to a superseded script; the message
*     literally instructs the caller to retry on the new version)
*
* The match stays close to the verbatim platform strings (rather than a loose
* "upgraded"/"reset" substring) so an ordinary application error that happens
* to mention those words is NOT misclassified as a supersede.
*/
const SUPERSEDED_ISOLATE_PATTERN = /reset because its code was updated|this script has been upgraded/i;
/**
* The "Network connection lost." platform transient — the connection between
* the isolate and its storage (or another DO) dropped. Unlike a supersede this
* MAY succeed on an in-process retry (a momentary blip), so it must not skip
* the in-process retry budget — but during a deploy-reset window it never
* succeeds in-process and surfaces interleaved with the supersede messages
* (SQL ops throw `SqlError: SQL query failed: Network connection lost.` while
* KV ops throw the reset message), so on retry exhaustion it must be treated
* as the platform's failure, not the callback's.
*/
const CONNECTION_LOST_PATTERN = /network connection lost/i;
/**
* The exact Durable Object storage-reset platform signal. Keep this narrow:
* ordinary SQL and generic internal errors are application failures. This is a
* transient storage reset, not a memory-limit poison pill.
*/
const STORAGE_RESET_PATTERN = /Internal error in Durable Object storage caused object to be reset/i;
/**
* The Durable Object memory-limit reset — the isolate exceeded its 128 MB limit
* and was reset by the platform (workerd surfaces this verbatim as
* "Durable Object's isolate exceeded its memory limit and was reset."; the D1
* sibling is "D1 DB's isolate exceeded its memory limit and was reset.").
*
* The match is the broad shared fragment "exceeded its memory limit" rather than
* the full "...and was reset" sentence: real-world surfacings truncate or reword
* the tail (some log pipelines clip the message; D1/storage wrappers re-prefix
* it), and a customer-reported loop (#1825) showed lines carrying only the
* "exceeded its memory limit" fragment. Missing a surfacing here means the
* circuit breaker never engages, so we err toward the broader match — and even a
* false positive is fail-safe (a tightly-bounded retry-then-seal, not data loss).
*
* This is DELIBERATELY a separate class from `SUPERSEDED_ISOLATE_PATTERN` /
* {@link isPlatformTransientError}, and is NOT folded into them. A supersede or
* connection-lost transient means "re-run the same work and it succeeds on a
* healthy isolate" — those classes can be deferred and retried *indefinitely*. A
* memory-limit reset is the opposite: re-running the SAME memory-heavy work
* deterministically re-OOMs (the footprint, not the platform, is the cause), so
* deferring it indefinitely would PRESERVE the one-shot row and re-run the
* doomed work forever (amplifying the loop and cost — see #1825). It is a
* poison-pill signal: callers must bound retries tightly and then SEAL.
*
* Accordingly the schedule executor (`_executeScheduleCallback`) and the
* alarm-boundary circuit breaker (`Agent.alarm`) treat it as its OWN class: a
* memory-limit reset is re-thrown (row preserved) so it reaches the breaker,
* which tolerates a few strikes (`maxAlarmMemoryLimitStrikes`) and then seals +
* purges the looping row — i.e. *bounded* deferral, never the unbounded deferral
* the transient classes get.
*/
const MEMORY_LIMIT_RESET_PATTERN = /exceeded its memory limit/i;
function errorMessageOf(error) {
	return error instanceof Error ? error.message : typeof error === "string" ? error : "";
}
/**
* Iterate an error and its `cause` chain (depth-limited so a cyclic chain
* can't spin). Wrappers like `SqlError` carry the original platform error in
* `cause` and may not propagate signal properties (e.g. the CF `retryable`
* flag), so classification must look through them.
*/
function* selfAndCauses(error) {
	let current = error;
	for (let depth = 0; depth < 8 && current != null; depth++) {
		yield current;
		current = typeof current === "object" ? current.cause : void 0;
	}
}
/**
* Whether an error (or anything in its `cause` chain) is a transient
* "superseded isolate" failure — see `SUPERSEDED_ISOLATE_PATTERN`. In-process
* retries are futile for this class; the work must be deferred to a fresh
* invocation, which runs the new code and succeeds.
*/
function isDurableObjectCodeUpdateReset(error) {
	for (const e of selfAndCauses(error)) if (SUPERSEDED_ISOLATE_PATTERN.test(errorMessageOf(e))) return true;
	return false;
}
/**
* Whether an error (or anything in its `cause` chain) carries the exact
* Durable Object storage-reset platform fragment. Generic SQL/internal errors
* deliberately do not qualify.
*/
function isDurableObjectStorageReset(error) {
	for (const e of selfAndCauses(error)) if (STORAGE_RESET_PATTERN.test(errorMessageOf(e))) return true;
	return false;
}
/**
* Whether an error (or anything in its `cause` chain, or a raw error-message
* string) is a Durable Object memory-limit reset — see
* {@link MEMORY_LIMIT_RESET_PATTERN}. Unlike {@link isPlatformTransientError},
* re-running the same work re-OOMs deterministically, so callers must NOT defer
* it like a transient; they should bound retries tightly and then seal (#1825).
*/
function isDurableObjectMemoryLimitReset(error) {
	for (const e of selfAndCauses(error)) if (MEMORY_LIMIT_RESET_PATTERN.test(errorMessageOf(e))) return true;
	return false;
}
/**
* Whether an error (or anything in its `cause` chain) is a transient failure
* of the PLATFORM rather than of the code that threw it:
*
*   - a superseded-isolate reset ("reset because its code was updated" /
*     "this script has been upgraded") — a deploy replaced the isolate;
*   - an error the platform itself flags `retryable: true` (excluding
*     overloaded errors, where retrying the same object won't help) — see
*     `isErrorRetryable`;
*   - "Network connection lost." — the storage/stub connection dropped. The
*     CF `retryable` flag does not survive error wrappers (e.g. `SqlError`
*     copies only the message + `cause`) and is absent in some local-dev
*     shapes, so the verbatim message is matched as well;
*   - the exact "Internal error in Durable Object storage caused object to be
*     reset" platform fragment. Generic internal and SQL errors remain fatal.
*
* Used to decide whether failed work should be RE-RUN LATER (platform
* transient — the same work succeeds once the platform recovers, typically
* seconds after a deploy) versus ABANDONED as genuinely failing (application
* error — re-running yields the same failure). A genuine application error
* carries none of these signals, so it is never misclassified by this check.
*/
/**
* Whether a failure is the PLATFORM's rather than the application's — any
* platform transient (see {@link isPlatformTransientError}, which includes
* superseded-isolate resets) or a memory-limit reset. Failed work in this
* class must be PRESERVED and deferred, never completed as an application
* failure. The two sub-classes defer differently: transients re-run
* indefinitely (the platform recovers), while memory-limit deferral is
* bounded by the alarm circuit breaker (#1825).
*/
function isPlatformFailure(error) {
	return isPlatformTransientError(error) || isDurableObjectMemoryLimitReset(error);
}
function isPlatformTransientError(error) {
	for (const e of selfAndCauses(error)) {
		const message = errorMessageOf(e);
		if (SUPERSEDED_ISOLATE_PATTERN.test(message)) return true;
		if (CONNECTION_LOST_PATTERN.test(message)) return true;
		if (STORAGE_RESET_PATTERN.test(message)) return true;
		if (isErrorRetryable(e)) return true;
	}
	return false;
}
//#endregion
export { isDurableObjectCodeUpdateReset, isDurableObjectMemoryLimitReset, isDurableObjectStorageReset, isErrorRetryable, isPlatformFailure, isPlatformTransientError, jitterBackoff, resolveRetryConfig, tryN, validateRetryOptions };

//# sourceMappingURL=retries.js.map