@juspay/neurolink/dist/utils/providerRetry.js

The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002

/**
 * Provider-level retry utility for AI SDK calls (NL11)
 *
 * The Vercel AI SDK's `generateText()` and `streamText()` have built-in retry
 * logic (`_retryWithExponentialBackoff()` with default `maxRetries: 2`) that
 * retries on HTTP 429/500/503. These retries are completely invisible to OTel
 * because they happen inside the AI SDK.
 *
 * This module provides an instrumented retry wrapper that:
 * 1. Disables the AI SDK's internal retries (via `maxRetries: 0`)
 * 2. Implements our own retry loop with full OTel span events
 * 3. Records retry attempts, delays, status codes, and total attempt count
 *
 * @module utils/providerRetry
 */
import { NeuroLinkError, isAbortError } from "./errorHandling.js";
import { logger } from "./logger.js";
import { APICallError } from "./generationErrors.js";
import { parseRetryAfterMs } from "./retryAfter.js";
/** Maximum number of retry attempts after the initial call (total = 1 + MAX_PROVIDER_RETRIES). */
export const MAX_PROVIDER_RETRIES = 2;
/** Base delay in ms for exponential backoff between retries. */
export const BASE_RETRY_DELAY_MS = 1000;
/** Minimum delay in ms when a retryable response provides no retry timing. */
export const NO_HINT_FLOOR_MS = 10_000;
/**
 * Maximum server-requested retry delay honored by provider retries.
 *
 * A hint above this cap is not clamped-and-slept: the server declared an
 * unavailability window we are not willing to wait out, so the error is
 * surfaced immediately instead — fallback orchestration (providerFallback /
 * modelChain) can route to another provider, and callers without fallback
 * get a prompt rate-limit error rather than a silent multi-minute stall.
 */
export const MAX_RETRY_AFTER_MS = 60_000;
const sleepWithTimeout = (delayMs) => new Promise((resolve) => setTimeout(resolve, delayMs));
/**
 * Check whether an error thrown by the AI SDK is retryable.
 *
 * Uses `APICallError.isInstance()` for proper type-safe detection (the class
 * uses a branded symbol marker, so `instanceof` doesn't work across package
 * boundaries). Falls back to duck-typing for non-APICallError cases.
 */
/**
 * Duck-typed HTTP status extraction for errors of unknown provenance.
 * NeuroLink's hand-rolled clients stamp `.statusCode`; official SDK errors
 * (e.g. @anthropic-ai/sdk APIError) expose `.status`. Shared with
 * baseProvider's handleProviderError metadata preservation.
 */
export function duckTypedStatusCode(error) {
    if (!error || typeof error !== "object") {
        return undefined;
    }
    const err = error;
    if (typeof err.statusCode === "number") {
        return err.statusCode;
    }
    if (typeof err.status === "number") {
        return err.status;
    }
    // AWS SDK v3 puts the HTTP status on `$metadata.httpStatusCode` and nowhere
    // else, so without this branch no AWS error carries a status code as far as
    // anything here is concerned. That is not only a retry question: this
    // function also feeds `classifyProviderError` and the retry telemetry, so a
    // Bedrock or SageMaker 429 was not recognisable as a rate limit, and a 5xx
    // not recognisable as transient, by any status-based path. Name-based
    // classification (ThrottlingException, AccessDeniedException) still worked,
    // which is why this stayed hidden.
    const metadata = err.$metadata;
    if (typeof metadata === "object" && metadata !== null) {
        const httpStatusCode = metadata
            .httpStatusCode;
        if (typeof httpStatusCode === "number") {
            return httpStatusCode;
        }
    }
    return undefined;
}
/**
 * Duck-typed Retry-After extraction for errors of unknown provenance.
 * Reads a pre-parsed `.retryAfterMs` (stamped by metadata-preserving
 * wrappers like handleProviderError), then `.headers` (Headers instance or
 * record — e.g. @anthropic-ai/sdk APIError), then `.responseHeaders` (the
 * hand-rolled OpenAI-compatible client's record). Shared with baseProvider.
 */
export function extractRetryAfterMsFromError(error) {
    if (!error || typeof error !== "object") {
        return undefined;
    }
    const err = error;
    // Number.isFinite rejects NaN/Infinity: typeof NaN === "number", and a NaN
    // hint would survive Math.max() into sleep(NaN) — an immediate retry
    // instead of the no-hint floor.
    if (typeof err.retryAfterMs === "number" &&
        Number.isFinite(err.retryAfterMs)) {
        return err.retryAfterMs;
    }
    for (const candidate of [err.headers, err.responseHeaders]) {
        if (candidate && typeof candidate === "object") {
            const parsed = parseRetryAfterMs(candidate);
            if (parsed !== undefined) {
                return parsed;
            }
        }
    }
    return undefined;
}
/**
 * OpenAI's chat transport keeps the type inside the JSON response body,
 * rather than stamping it onto the raw error.
 */
export function readOpenAIBodyErrorType(responseBody) {
    if (typeof responseBody !== "string") {
        return undefined;
    }
    try {
        const parsed = JSON.parse(responseBody);
        const inner = parsed?.error;
        return typeof inner?.type === "string" ? inner.type : undefined;
    }
    catch {
        return undefined;
    }
}
/**
 * An OpenAI-wire `insufficient_quota` error: the account is out of credit or
 * has hit a spend cap.
 *
 * Exported and shared with the OpenAI error table so the retry predicate and
 * the message a caller sees cannot disagree about what a permanent billing
 * state looks like. The type is NOT a field on the error — OpenAI puts it
 * inside the JSON response body, which the SDK keeps as a string — so any
 * caller that needs it has to parse, and none should do so on its own.
 *
 * Deliberately keyed on the error TYPE, never on the word "quota". OpenAI and
 * xAI send 429 + `type: "insufficient_quota"` for a permanent billing state,
 * but other providers use "quota" for ordinary throttling — Google's 429 reads
 * "Quota exceeded for quota metric ...", which IS retryable and must stay that
 * way. Matching free text here would make Gemini throttling non-retryable and
 * quietly remove a layer of resilience.
 */
export function isOpenAIQuotaExhaustedError(error) {
    if (typeof error !== "object" || error === null) {
        return false;
    }
    const err = error;
    if (err.type === "insufficient_quota") {
        return true;
    }
    return readOpenAIBodyErrorType(err.responseBody) === "insufficient_quota";
}
/**
 * True when `error` is an abort, or wraps one at any depth via `cause`.
 *
 * `isAbortError` alone is not enough here because it inspects one value, and
 * by the time a provider failure reaches the retry wrapper it has usually been
 * re-thrown inside a provider-specific error whose `name` no longer says
 * "AbortError". The depth bound stops a self-referential or maliciously deep
 * `cause` chain from hanging the classifier; nothing legitimate nests further.
 */
function isAbortLike(error, depth = 0) {
    if (depth > 8 || error === null || typeof error !== "object") {
        return isAbortError(error);
    }
    if (isAbortError(error)) {
        return true;
    }
    const { cause } = error;
    return cause === undefined || cause === error
        ? false
        : isAbortLike(cause, depth + 1);
}
export function isRetryableProviderError(error) {
    // Ahead of everything, for the same reason as the quota branch below but a
    // worse failure: the caller has already walked away. Retrying a cancelled
    // turn cannot succeed — the signal stays aborted, so every further attempt
    // rejects without reaching the network — and it costs the full ladder,
    // 2 retries at the NO_HINT_FLOOR_MS floor, before the turn ends.
    //
    // Measured on SageMaker before this guard: abort at 400ms, provider failed
    // at 232ms with "Request aborted", then two more attempts 10s apart each
    // failing in 1ms without a request leaving the process, and the turn finally
    // threw at 21,996ms — 55x the time the cancellation should have taken, with
    // `name` by then reading "Error" rather than "AbortError".
    //
    // The cause chain matters and is not defensive padding. Providers wrap
    // transport failures in their own error class on the way up: SageMaker's
    // handleSageMakerError() stamps an unrecognised error `statusCode: 500`
    // while keeping the original as `cause`. That fabricated 500 is what made an
    // abort look retryable here — the duck-typed status branch below reads it
    // and returns true, outranking the wrapper's own `retryable: false`. Reading
    // through `cause` finds the abort whatever wrapped it.
    if (isAbortLike(error)) {
        return false;
    }
    // Before every other branch, including the SDK's own flag: a 429 carrying
    // `insufficient_quota` is marked retryable by the AI SDK because it only
    // looks at the status code. Retrying it burns the full ladder — 3 attempts
    // and ~20s of backoff, since no Retry-After accompanies these — on a state
    // that cannot change until someone tops up the account.
    if (isOpenAIQuotaExhaustedError(error)) {
        return false;
    }
    // Preferred path: use the AI SDK's own branded type check + isRetryable flag
    if (APICallError.isInstance(error)) {
        return error.isRetryable;
    }
    if (error instanceof NeuroLinkError) {
        return error.retriable;
    }
    // Fallback: duck-type for status codes on errors that aren't APICallError
    const statusCode = duckTypedStatusCode(error);
    if (statusCode !== undefined) {
        return statusCode === 429 || statusCode >= 500;
    }
    return false;
}
/**
 * Extract the HTTP status code from an AI SDK error, if available.
 */
export function getErrorStatusCode(error) {
    if (APICallError.isInstance(error)) {
        return error.statusCode;
    }
    return duckTypedStatusCode(error);
}
function getRetryAfterMs(error) {
    if (APICallError.isInstance(error) && error.responseHeaders) {
        const parsedDelay = parseRetryAfterMs(error.responseHeaders);
        if (parsedDelay !== undefined) {
            return parsedDelay;
        }
    }
    if (error instanceof NeuroLinkError &&
        error.retryAfterMs !== undefined &&
        Number.isFinite(error.retryAfterMs)) {
        return error.retryAfterMs;
    }
    return extractRetryAfterMsFromError(error);
}
/**
 * Execute a provider call with instrumented retry logic.
 *
 * @param operation  - The async operation to execute (should already use `maxRetries: 0`)
 * @param span       - The OTel span to annotate with retry events and attributes
 * @param label      - A human-readable label for log messages (e.g. "generateText", "streamText")
 * @returns The result of the operation
 */
export async function withProviderRetry(operation, span, label, sleep = sleepWithTimeout) {
    for (let attempt = 0; attempt <= MAX_PROVIDER_RETRIES; attempt++) {
        try {
            const result = await operation();
            // Record how many attempts it took on the span
            span?.setAttribute("gen_ai.provider.total_attempts", attempt + 1);
            if (attempt > 0) {
                logger.info(`[providerRetry] ${label} succeeded after ${attempt + 1} attempts`);
            }
            return result;
        }
        catch (error) {
            const retryable = isRetryableProviderError(error);
            const statusCode = getErrorStatusCode(error);
            const errorMessage = error instanceof Error ? error.message : String(error);
            if (!retryable || attempt === MAX_PROVIDER_RETRIES) {
                // Record failure details before re-throwing
                span?.setAttribute("gen_ai.provider.total_attempts", attempt + 1);
                if (attempt > 0) {
                    span?.setAttribute("gen_ai.provider.retries_exhausted", true);
                }
                logger.warn(`[providerRetry] ${label} failed (non-retryable or retries exhausted)`, {
                    attempt: attempt + 1,
                    retryable,
                    statusCode,
                    error: errorMessage,
                });
                throw error;
            }
            const retryAfterMs = getRetryAfterMs(error);
            if (retryAfterMs !== undefined && retryAfterMs > MAX_RETRY_AFTER_MS) {
                span?.setAttribute("gen_ai.provider.total_attempts", attempt + 1);
                span?.addEvent("gen_ai.provider.retry_suppressed", {
                    "retry.after_ms": retryAfterMs,
                    ...(statusCode !== undefined && { "retry.status_code": statusCode }),
                });
                // Best-effort stamp so upper retry layers (the MCP generation loop's
                // isRetryable check) also surface instead of re-running a
                // rate-limited provider.
                try {
                    Object.assign(error, {
                        isRetryable: false,
                        retryAfterMs,
                    });
                }
                catch {
                    /* frozen error — the throw below still surfaces it */
                }
                logger.warn(`[providerRetry] ${label} not retrying — server requested a ` +
                    `${Math.round(retryAfterMs / 1000)}s wait (cap ${MAX_RETRY_AFTER_MS / 1000}s); surfacing for fallback`, { attempt: attempt + 1, retryAfterMs, statusCode });
                throw error;
            }
            const delay = (retryAfterMs === undefined ? undefined : Math.max(0, retryAfterMs)) ??
                Math.max(BASE_RETRY_DELAY_MS * Math.pow(2, attempt), NO_HINT_FLOOR_MS);
            // Record retry event on the OTel span
            span?.addEvent("gen_ai.provider.retry", {
                "retry.attempt": attempt + 1,
                "retry.delay_ms": delay,
                ...(statusCode !== undefined && { "retry.status_code": statusCode }),
                "retry.error": errorMessage.slice(0, 256),
            });
            logger.warn(`[providerRetry] ${label} retrying after ${statusCode || "unknown"} error`, {
                attempt: attempt + 1,
                maxRetries: MAX_PROVIDER_RETRIES,
                delayMs: delay,
                statusCode,
                error: errorMessage,
            });
            await sleep(delay);
        }
    }
    // This should never be reached due to the throw inside the loop,
    // but TypeScript requires it for exhaustiveness.
    throw new Error(`[providerRetry] ${label} exhausted all retries`);
}