agentfootprint
Version:
The explainable agent framework — backtrack a wrong answer to the exact context that caused it (evidence, not guesses). Built on footprintjs.
300 lines • 15 kB
JavaScript
/**
* withRetry — provider decorator that retries failed calls.
*
* Pattern: Decorator (GoF) — wraps an `LLMProvider` and adds retry
* policy without changing its interface.
* Role: Outer ring (Hexagonal). Composable: `withRetry(withFallback(...))`.
*
* Retries `complete()` on transient failures with exponential backoff.
* `stream()` is retried under the SAME policy, but only until its first
* chunk arrives: a stream that fails before it has produced anything
* (HTTP 429, a 5xx, a reset at connect) is exactly as safe to re-open as a
* `complete()` is to re-send. Once the first chunk is in hand the stream is
* committed — the rest is plain delegation with no retry anywhere around it
* (`openStream` owns the only retry loop and returns WITH that chunk), so a
* mid-stream failure surfaces exactly as it did before: re-sending would
* replay tokens the caller has already shown.
*
* Default policy:
* • maxAttempts: 3 (initial + 2 retries)
* • backoff: exponential — 200ms, 400ms, 800ms
* • stated wait: when the failure says how long to wait (`retryAfterMs`,
* or `retryAfterSeconds` — see statedWait.ts), the wait is
* max(the schedule's delay, the stated wait), capped at
* `maxDelayMs` so a hostile header cannot stall a run. The
* adapter that knows its wire reads the header or the
* gateway's wording; this decorator never parses prose.
* • shouldRetry: rejects 4xx-class errors (client mistakes don't
* benefit from retry), AbortError, and any error that
* declares `retryable: false`; retries 5xx, network
* errors, and unknown shapes.
*
* **Status: contract-shaped and tested — independently reproduced against a
* local harness, 2026-08-13.** Somebody who is not this library's author ran
* the behaviour end to end and it held: this decorator absorbed two HTTP
* 503-shaped failures and recovered on call three, with both
* `agentfootprint.error.retried` and `agentfootprint.error.recovered` on the
* typed stream, and it made exactly ONE streaming attempt — the limit as it
* was then designed; a stream is now retried before its first chunk (see
* above), still never after. What was exercised was a SCRIPTED failing provider — the trial's
* own note is that those tests *"were local and deterministic, so they consumed
* no GCP credit"*. **No live provider outage has been retried from this
* repository**, so this is not the field-validated rung: a real 503 from a real
* vendor, with a real Retry-After and real latency, remains unexercised.
*/
import { sleep } from '../lib/sleep.js';
import { throwIfAborted } from '../lib/abort.js';
import { statedRetryAfterMs } from './statedWait.js';
import { replayAttempts } from './replayAttempt.js';
import { requestAttempt } from '../provider-contract/requestObserver.js';
/**
* Wrap a provider so its `complete()` — and its `stream()`, up to the first
* chunk — retries transient failures.
*
* @example
* import { withRetry } from 'agentfootprint/resilience';
* import { anthropic } from 'agentfootprint/providers';
*
* const robust = withRetry(anthropic({ apiKey }), {
* maxAttempts: 5,
* onRetry: (err, attempt, ms) => console.warn(`retry ${attempt} in ${ms}ms`, err),
* });
*/
export function withRetry(provider, options = {}) {
const maxAttempts = Math.max(1, options.maxAttempts ?? 3);
const initialDelayMs = options.initialDelayMs ?? 200;
const backoffFactor = options.backoffFactor ?? 2;
const maxDelayMs = options.maxDelayMs ?? 10_000;
const shouldRetry = options.shouldRetry ?? defaultShouldRetry;
const onRetry = options.onRetry;
const wrapped = {
name: `${provider.name}+retry`,
// A retry calls the SAME wire, so it carries exactly what that wire
// carries. Forwarded rather than inherited: this object is rebuilt from
// scratch, and a dropped capability silently narrows to the
// user/assistant floor — a wrapped OpenAI provider would start refusing
// system-role delivery it can perfectly well do.
...(provider.carriesInMessages !== undefined && {
carriesInMessages: provider.carriesInMessages,
}),
// Same wire, same forced-choice answer. Dropping this one degrades to
// "not declared", which is a REFUSAL — a wrapped Anthropic provider would
// start refusing `strategy: 'tool-forced'` it can perfectly well do.
...(provider.carriesForcedToolChoice !== undefined && {
carriesForcedToolChoice: provider.carriesForcedToolChoice,
}),
...(provider.supportsForcedToolChoice !== undefined && {
supportsForcedToolChoice: provider.supportsForcedToolChoice,
}),
// Same wire, same structured-output format (`strategy: 'native'`) — the
// same refusal-if-dropped reasoning as the forced choice above.
...(provider.carriesNativeOutputSchema !== undefined && {
carriesNativeOutputSchema: provider.carriesNativeOutputSchema,
}),
...(provider.supportsNativeOutputSchema !== undefined && {
supportsNativeOutputSchema: provider.supportsNativeOutputSchema,
}),
...(provider.supportsAutoNativeOutputSchema !== undefined && {
supportsAutoNativeOutputSchema: provider.supportsAutoNativeOutputSchema,
}),
// Same wire, same cache. The agent chooses its cache strategy from this
// declaration — it used to look the strategy up by `name`, and the
// `+retry` suffix above sent every retried Anthropic call out with no
// cache markers at all.
...(provider.promptCaching !== undefined && { promptCaching: provider.promptCaching }),
// And the thinking it returns: the agent used to look the handler up by
// `name`, so a retried Anthropic provider dropped its signed thinking.
...(provider.thinkingHandler !== undefined && { thinkingHandler: provider.thinkingHandler }),
// And which thinking each model takes — dropped, the agent could no
// longer refuse `.thinking()` for a model that cannot think.
...(provider.thinkingMode !== undefined && { thinkingMode: provider.thinkingMode }),
async complete(req, hooks) {
const attempts = replayAttempts(req, hooks);
// t0 for the `recovered` report's totalDurationMs. New
// instrumentation (the decorator did not measure this before v7.8),
// not a recovered fact.
const startedMs = Date.now();
for (let attempt = 1;; attempt++) {
const invocation = requestAttempt(attempts.hooks, attempts.request);
try {
const res = await provider.complete(attempts.request, invocation.hooks);
reportRecovered(hooks, attempt, startedMs);
return attempts.response(res);
}
catch (err) {
invocation.close();
await backOffOrThrow(err, attempt, req, hooks);
}
finally {
invocation.close();
}
}
},
};
/**
* The ONE retry decision both doors share: throw `err` when the budget is
* spent or the policy says no; otherwise report the retry (consumer hook +
* in-run hooks channel) and sleep the backoff, honouring `req.signal`.
*/
async function backOffOrThrow(err, attempt, req, hooks) {
// A cancelled call is never asked again, whatever `shouldRetry` says — a
// retry would re-send (and re-bill) a request nobody is waiting for. The
// failure is the cancellation itself (lib/abort.ts).
throwIfAborted(req.signal);
if (attempt >= maxAttempts || !shouldRetry(err, attempt))
throw err;
const scheduled = Math.min(maxDelayMs, initialDelayMs * Math.pow(backoffFactor, attempt - 1));
// The server's word beats our guess — waiting less than it asked only
// spends an attempt on a refusal already announced. Capped by the same
// policy maximum, so a hostile or garbled wait cannot stall the run.
const stated = statedRetryAfterMs(err);
const delay = stated === undefined ? scheduled : Math.min(maxDelayMs, Math.max(scheduled, stated));
onRetry?.(err, attempt + 1, delay);
hooks?.onResilience?.({
kind: 'retried',
attempt: attempt + 1,
maxAttempts,
lastError: err instanceof Error ? err.message : String(err),
backoffMs: delay,
reason: classifyRetryReason(err),
// Absent when nothing was stated — the event is byte-identical to the
// schedule-only one.
...(stated !== undefined && { statedWaitMs: stated }),
});
await sleep(delay, req.signal);
}
if (provider.stream) {
// Guarded by `if (provider.stream)`; bound once so the closures below
// need no assertion. `hooks` is FORWARDED on every attempt: a stacked
// inner decorator (withRetry(withFallback(...))) would otherwise go dark
// on the stream path.
const innerStream = provider.stream.bind(provider);
/**
* Open the inner stream and wait for its FIRST step — the only span a
* retry can cover. Returns with that step in hand, so by the time the
* caller sees a chunk this loop has already been left for good.
*/
const openStream = async (req, hooks) => {
const attempts = replayAttempts(req, hooks);
const startedMs = Date.now();
for (let attempt = 1;; attempt++) {
const invocation = requestAttempt(attempts.hooks, attempts.request);
try {
const iterator = attempts.iterator(innerStream(attempts.request, invocation.hooks)[Symbol.asyncIterator]());
const first = await iterator.next();
// The stream got through: its first step arrived. Reported here —
// not at the terminal chunk — because this is where retrying ends
// (a consumer that stops reading at `done: true` would otherwise
// never see it), and so `recovered` lands before the first token.
reportRecovered(hooks, attempt, startedMs);
return { iterator, first, close: invocation.close };
}
catch (err) {
invocation.close();
await backOffOrThrow(err, attempt, req, hooks);
}
}
};
wrapped.stream = async function* retryingStream(req, hooks) {
const { iterator, first, close } = await openStream(req, hooks);
if (first.done === true) {
close();
return;
}
// From here on there is no catch: nothing below can start a retry.
let delegated = false;
try {
yield first.value;
delegated = true;
// `yield*` forwards the consumer's return()/throw() to the inner
// stream, exactly as the old pass-through did.
yield* { [Symbol.asyncIterator]: () => iterator };
}
finally {
close();
// A consumer that stopped at the first chunk still closes the wire.
if (!delegated)
await iterator.return?.();
}
};
}
return wrapped;
}
/**
* Report a recovery — only a success that FOLLOWED a failure is one; a
* first-try success reports nothing (mirrors the rules loop's guard in
* reliabilityExecution.ts).
*/
function reportRecovered(hooks, attempt, startedMs) {
if (attempt === 1)
return;
hooks?.onResilience?.({
kind: 'recovered',
attempt,
totalDurationMs: Date.now() - startedMs,
});
}
/**
* Classify the ERROR that caused a retry, for `ResilienceReport.reason`.
*
* Reads exactly the fields `defaultShouldRetry` inspects, so the label
* always describes the same signal the default policy acted on. Note this
* is a classification of the error, NOT of the predicate's reasoning —
* `shouldRetry` returns a bare boolean, so a custom predicate's rationale
* is unknowable here. `'http-4xx'` is only reachable via a custom
* predicate (the default rejects non-429 4xx), which itself is a useful
* signal.
*/
function classifyRetryReason(err) {
const status = err?.status ?? err?.statusCode;
if (typeof status !== 'number')
return 'no-status';
if (status === 429)
return 'http-429';
if (status >= 500)
return 'http-5xx';
if (status >= 400)
return 'http-4xx';
return `http-${status}`;
}
// ── Defaults ────────────────────────────────────────────────────────
/**
* Skip retry for AbortError, 4xx-class errors, and an error that declares
* `retryable: false`. Retry on everything else (network errors, 5xx,
* unknown shapes). Provider adapters that surface HTTP status should set
* `error.status` for this to work; the predicate falls back to retrying
* when status is unknown (better to retry once than to surface a flaky
* failure).
*
* `retryable: false` is how an adapter says a failure with no HTTP status
* cannot recover by asking again — a refusal raised before any request
* (no key, no model), or a 2xx answer it could not read, where a re-send
* may run and bill the model a second time. Only the literal `false` is
* read: an error that does not declare the field is judged exactly as
* before, and `retryable: true` does not override a 4xx status.
*
* Exported (module-level, NOT on the resilience barrel) as the single
* source of truth for the decorators' transience policy — shared by
* `withCredentialRetry` (identity) so "what counts as transient" never
* drifts between the LLM and credential retry wrappers.
*/
export function defaultShouldRetry(err, _attempt) {
if (isAbortError(err))
return false;
if (err?.retryable === false)
return false;
const status = err?.status ??
err?.statusCode;
if (typeof status === 'number' && status >= 400 && status < 500) {
// 429 Too Many Requests is the one 4xx that benefits from retry.
return status === 429;
}
return true;
}
function isAbortError(err) {
if (!err || typeof err !== 'object')
return false;
const e = err;
return e.name === 'AbortError' || e.code === 'ABORT_ERR';
}
//# sourceMappingURL=withRetry.js.map