openclaw/dist/failover-error-BkNxlp8A.js

Multi-channel AI gateway with extensible messaging integrations

import { C as parseStrictNonNegativeInteger } from "./number-coercion-CLj0HTDM.js";
import { i as resolveGlobalSingleton } from "./global-singleton-Dc_stLtU.js";
import { n as collectErrorGraphCandidates, s as readErrorName } from "./error-coercion-D_-xJ90S.js";
import { t as formatCliCommand } from "./command-format-C7YfyMTd.js";
import { t as formatErrorMessage } from "./errors-Db3Ymjlb.js";
import { b as isAgentRunStaleLifecycleError } from "./agent-events-CoxiItUi.js";
import { l as failoverReasonFromClassification, n as classifyFailoverSignal, p as isUnclassifiedNoBodyHttpSignal } from "./classify-GlkuT1ur.js";
import { c as isProviderRequestSizeCeilingError } from "./message-patterns-Cg93oCGB.js";
import { t as extractFailoverSignalDetails } from "./signal-details-CG2lfr-u.js";
import { a as isFailoverError, c as readDirectErrorCode, l as readDirectErrorMessage, n as findErrorProperty, r as getErrorMessage, s as isTimeoutError, t as FailoverError } from "./error-Bb_OF8ag.js";
import { n as copyErrorDiagnostic } from "./error-diagnostics-BM-F2vsk.js";
import { i as isAgentHarnessPreflightError, n as AgentHarnessSessionSupersededError } from "./errors-70ml6R0Z.js";
//#region src/agents/failover-error.ts
/**
* Provider/model failover error classification.
* Converts nested provider, transport, timeout, auth, and local coordination
* failures into structured failover reasons and remediation metadata.
*/
const MAX_FAILOVER_CAUSE_DEPTH = 25;
const MISSING_TOOL_RESULT_REASON = "missing_tool_result";
const MISSING_TOOL_RESULT_TEXT_RE = /native Codex tool\.call without a matching tool\.result/i;
const RUNTIME_COORDINATION_ERROR_NAMES = /* @__PURE__ */ new Set([
	"GatewayDrainingError",
	"WorkerRunnerUnavailableError",
	"WorkerRunnerCapacityError",
	"WorkerWorkspaceReconciliationError",
	"ActiveTurnClaimError"
]);
const modelFallbackStops = resolveGlobalSingleton(Symbol.for("openclaw.modelFallbackStops"), () => /* @__PURE__ */ new WeakSet());
function recordModelFallbackStop(error) {
	modelFallbackStops.add(error);
}
function hasModelFallbackStop(error) {
	return collectErrorGraphCandidates(error, resolveNestedErrors).some((candidate) => candidate instanceof Error && modelFallbackStops.has(candidate) || isFailoverError(candidate) && isCliTerminalStopCode(candidate.code));
}
function resolveNestedErrors(candidate) {
	const errors = candidate.errors;
	return [
		candidate.error,
		candidate.cause,
		...Array.isArray(errors) ? errors : []
	];
}
/**
* True when the provider refused the request for its own size rather than for context pressure or
* bucket state.  An error that never became a `FailoverError` still carries the provider's text in
* its message, so it is read directly.
*/
function hasProviderRequestSizeCeiling(err) {
	return collectErrorGraphCandidates(err, resolveNestedErrors).some((candidate) => isFailoverError(candidate) ? candidate.requestSizeCeiling : isProviderRequestSizeCeilingError(formatErrorMessage(candidate)));
}
function findCliFailoverError(err, match, seen) {
	const direct = isFailoverError(err) ? match(err) : void 0;
	if (direct) return direct;
	if (!err || typeof err !== "object" || seen.has(err)) return;
	seen.add(err);
	for (const value of resolveNestedErrors(err)) {
		const found = findCliFailoverError(value, match, seen);
		if (found) return found;
	}
}
const CLI_TERMINAL_STOP_CODES = /* @__PURE__ */ new Set(["cli_max_turns", "cli_turn_stopped"]);
function isCliTerminalStopCode(code) {
	return code !== void 0 && CLI_TERMINAL_STOP_CODES.has(code);
}
function findCliTerminalStopError(err) {
	return findCliFailoverError(err, (error) => isCliTerminalStopCode(error.code) ? error : void 0, /* @__PURE__ */ new Set());
}
function hasCliTimeoutContext(error) {
	const context = error.cliTimeout;
	return Boolean(context && (context.mode === "overall" || context.mode === "no-output") && Number.isFinite(context.timeoutSeconds) && context.timeoutSeconds >= 0 && typeof context.observedActivity === "boolean" && Number.isInteger(context.activeToolCount) && context.activeToolCount >= 0 && Number.isInteger(context.backgroundTaskCount) && context.backgroundTaskCount >= 0);
}
function findCliTimeoutError(err) {
	return findCliFailoverError(err, (error) => hasCliTimeoutContext(error) ? error : void 0, /* @__PURE__ */ new Set());
}
/** Map a failover reason to the closest HTTP-like status code. */
function resolveFailoverStatus(reason) {
	switch (reason) {
		case "billing": return 402;
		case "server_error": return 500;
		case "rate_limit": return 429;
		case "overloaded": return 503;
		case "auth": return 401;
		case "auth_permanent": return 403;
		case "timeout": return 408;
		case "tls_certificate": return 502;
		case "context_overflow": return 413;
		case "format": return 400;
		case "model_not_found": return 404;
		case "session_expired": return 410;
		default: return;
	}
}
function readDirectStatusCode(err) {
	if (!err || typeof err !== "object") return;
	const candidate = err.status ?? err.statusCode;
	if (typeof candidate === "number") return candidate;
	if (typeof candidate === "string") return parseStrictNonNegativeInteger(candidate);
}
function getStatusCode(err) {
	return findErrorProperty(err, readDirectStatusCode);
}
function isStableProviderErrorType(value) {
	if (/^(?:api|authentication|invalid_request|not_found|overloaded|permission|rate_limit|server)_error$/i.test(value)) return false;
	return /^[A-Z][A-Z0-9_:-]*$/.test(value);
}
function readDirectErrorType(err) {
	if (!err || typeof err !== "object") return;
	const directType = err.errorType;
	if (typeof directType === "string") {
		const trimmed = directType.trim();
		return trimmed && isStableProviderErrorType(trimmed) ? trimmed : void 0;
	}
	const detailType = err.detail?.type;
	if (typeof detailType === "string") {
		const trimmed = detailType.trim();
		return trimmed && isStableProviderErrorType(trimmed) ? trimmed : void 0;
	}
	const type = err.type;
	if (typeof type === "string") {
		const trimmed = type.trim();
		if (!trimmed || /^(?:error|exception)$/i.test(trimmed)) return;
		return isStableProviderErrorType(trimmed) ? trimmed : void 0;
	}
}
function getErrorType(err) {
	return findErrorProperty(err, readDirectErrorType);
}
function readDirectProvider(err) {
	if (!err || typeof err !== "object") return;
	const provider = err.provider;
	if (typeof provider !== "string") return;
	return provider.trim() || void 0;
}
function getProvider(err) {
	return findErrorProperty(err, readDirectProvider);
}
function readDirectErrorDetails(err) {
	if (!err || typeof err !== "object") return;
	const candidate = err;
	return extractFailoverSignalDetails(candidate.param, candidate.errorBody, candidate.body, candidate.detail, candidate.error);
}
function normalizeDirectErrorSignal(err) {
	const message = readDirectErrorMessage(err);
	return {
		status: readDirectStatusCode(err),
		code: readDirectErrorCode(err),
		errorType: readDirectErrorType(err),
		message: message || void 0,
		provider: readDirectProvider(err),
		details: readDirectErrorDetails(err)
	};
}
function hasSessionTranscriptWriterClaimRebound(err, seen = /* @__PURE__ */ new Set()) {
	if (err && typeof err === "object" && readErrorName(err) === "SessionTranscriptWriterClaimReboundError") return true;
	if (!err || typeof err !== "object") return false;
	if (seen.has(err)) return false;
	seen.add(err);
	const candidate = err;
	return hasSessionTranscriptWriterClaimRebound(candidate.error, seen) || hasSessionTranscriptWriterClaimRebound(candidate.cause, seen) || hasSessionTranscriptWriterClaimRebound(candidate.reason, seen);
}
function readField(value, key) {
	if (!value || typeof value !== "object") return;
	return value[key];
}
function readErrorStringField(value, key) {
	const field = readField(value, key);
	return typeof field === "string" ? field : void 0;
}
function isMissingToolResultMessage(value) {
	return MISSING_TOOL_RESULT_TEXT_RE.test(value);
}
function isMissingToolResultMarker(value) {
	return value.trim() === MISSING_TOOL_RESULT_REASON;
}
function readMissingToolResultMarker(err) {
	const message = readDirectErrorMessage(err);
	if (message && isMissingToolResultMessage(message)) return true;
	for (const key of [
		"code",
		"reason",
		"status"
	]) {
		const value = readErrorStringField(err, key);
		if (value && isMissingToolResultMarker(value)) return true;
	}
	const output = readErrorStringField(err, "output");
	if (output && isMissingToolResultMessage(output)) return true;
	const resultReason = readErrorStringField(readField(err, "result"), "reason");
	const detailReason = readErrorStringField(readField(err, "detail"), "reason");
	if (resultReason === MISSING_TOOL_RESULT_REASON || detailReason === MISSING_TOOL_RESULT_REASON) return true;
}
function hasMissingToolResultFailure(err) {
	return findErrorProperty(err, readMissingToolResultMarker) === true;
}
function hasStaleAgentRunLifecycleFailure(err) {
	return findErrorProperty(err, (candidate) => isAgentRunStaleLifecycleError(candidate) ? true : void 0) === true;
}
function hasRuntimeCoordinationFailure(err) {
	return collectErrorGraphCandidates(err, resolveNestedErrors).some((candidate) => RUNTIME_COORDINATION_ERROR_NAMES.has(readErrorName(candidate)));
}
function hasDirectProviderFailureIdentity(err) {
	if (isFailoverError(err)) return true;
	const signal = normalizeDirectErrorSignal(err);
	return Boolean(signal.status || signal.code || signal.errorType || signal.provider);
}
/**
* True when the error is a local runtime coordination/tool-execution error
* rather than a provider/model failure. The model fallback chain must abort on
* these instead of consuming candidate slots — retrying any model would hit the
* same local condition. See #83510 and #95474.
*/
function isNonProviderRuntimeCoordinationError(err) {
	return resolveModelFallbackError(err).kind === "coordination";
}
function normalizeErrorSignal(err, providerHint) {
	const message = getErrorMessage(err);
	return {
		status: getStatusCode(err),
		code: findErrorProperty(err, readDirectErrorCode),
		errorType: getErrorType(err),
		message: message || void 0,
		provider: getProvider(err) ?? providerHint,
		details: readDirectErrorDetails(err)
	};
}
function getNestedErrorCandidates(err) {
	if (!err || typeof err !== "object") return [];
	const candidate = err;
	return [candidate.error, candidate.cause].filter((value) => value !== void 0 && value !== err);
}
function isFormatClassification(classification) {
	return classification?.kind === "reason" && classification.reason === "format";
}
function decideNestedFormatOverride(candidate, inheritedStatus, seen, depth) {
	if (depth > MAX_FAILOVER_CAUSE_DEPTH) return null;
	if (candidate && typeof candidate === "object") {
		if (seen.has(candidate)) return null;
		seen.add(candidate);
	}
	const directSignal = normalizeDirectErrorSignal(candidate);
	const nestedCandidates = getNestedErrorCandidates(candidate);
	const nestedStatus = directSignal.status ?? inheritedStatus;
	const hasDirectMessage = Boolean(directSignal.message?.trim());
	if (hasDirectMessage && isUnclassifiedNoBodyHttpSignal({
		...directSignal,
		status: nestedStatus
	})) return true;
	if (hasDirectMessage && (nestedCandidates.length === 0 || classifyFailoverSignal(directSignal))) return false;
	for (const nestedCandidate of nestedCandidates) {
		const decision = decideNestedFormatOverride(nestedCandidate, nestedStatus, seen, depth + 1);
		if (decision !== null) return decision;
	}
	return null;
}
function resolveFailoverClassificationFromErrorInternal(err, seen, depth, providerHint) {
	if (depth > MAX_FAILOVER_CAUSE_DEPTH) return null;
	if (err && typeof err === "object") {
		if (seen.has(err)) return null;
		seen.add(err);
	}
	if (isFailoverError(err)) return {
		kind: "reason",
		reason: err.reason
	};
	const signal = normalizeErrorSignal(err, providerHint);
	const classification = classifyFailoverSignal(signal);
	const nestedCandidates = getNestedErrorCandidates(err);
	if (!classification || classification.kind === "context_overflow") for (const candidate of nestedCandidates) {
		const nestedClassification = resolveFailoverClassificationFromErrorInternal(candidate, seen, depth + 1, providerHint);
		if (nestedClassification) return nestedClassification;
	}
	if (isFormatClassification(classification)) for (const candidate of nestedCandidates) {
		const shouldClearFormat = decideNestedFormatOverride(candidate, signal.status, seen, depth + 1);
		if (shouldClearFormat === true) return null;
		if (shouldClearFormat === false) break;
	}
	if (classification) return classification;
	if (isTimeoutError(err)) return {
		kind: "reason",
		reason: "timeout"
	};
	return null;
}
function resolveFailoverClassificationFromError(err, providerHint) {
	if (isAgentHarnessPreflightError(err)) return null;
	return resolveFailoverClassificationFromErrorInternal(err, /* @__PURE__ */ new Set(), 0, providerHint);
}
/** Resolve the failover reason represented by an unknown provider/runtime error. */
function resolveFailoverReasonFromError(err, providerHint) {
	return failoverReasonFromClassification(resolveFailoverClassificationFromError(err, providerHint));
}
/**
* Build an actionable remediation hint for a failover error when the failure
* reason is `auth` / `auth_permanent` and we have enough provider attribution
* to suggest a re-authentication command. Returns `undefined` for any other
* failure shape so callers can opportunistically append the hint without
* branching on every reason themselves.
*
* Keep the string short and copy-pasteable — operators see it in fallback
* summary errors and TUI status lines.
*/
function buildFailoverRemediationHint(err) {
	if (!isFailoverError(err)) return;
	if (err.reason !== "auth" && err.reason !== "auth_permanent") return;
	const provider = err.provider?.trim();
	if (!provider) return;
	if (provider === "google-gemini-cli") return `Authenticate in Gemini CLI directly, or configure a supported Google API key with: ${formatCliCommand("openclaw configure")}`;
	const command = buildProviderReauthCommand(provider);
	return command ? `Re-authenticate with: ${command}` : void 0;
}
function quotePosixShellArg(value) {
	return `'${value.replaceAll("'", "'\\''")}'`;
}
/** Build the operator command for reauthenticating one provider. */
function buildProviderReauthCommand(provider, env = process.env) {
	const trimmed = provider.trim();
	if (!trimmed || hasControlCharacter(trimmed)) return;
	return formatCliCommand(`openclaw models auth login --provider ${quotePosixShellArg(trimmed)} --force`, env);
}
function hasControlCharacter(value) {
	for (let i = 0; i < value.length; i += 1) {
		const code = value.charCodeAt(i);
		if (code < 32 || code === 127) return true;
	}
	return false;
}
/** Convert a failover or raw error into structured fields for logs/UI. */
function describeFailoverError(err) {
	if (isAgentHarnessPreflightError(err)) return { message: err.message };
	if (isFailoverError(err)) return {
		message: err.message,
		rawError: err.rawError,
		reason: err.reason,
		status: err.status,
		code: err.code,
		provider: err.provider,
		model: err.model,
		profileId: err.profileId,
		authMode: err.authMode,
		sessionId: err.sessionId,
		lane: err.lane
	};
	const signal = normalizeErrorSignal(err);
	return {
		message: signal.message ?? String(err),
		reason: resolveFailoverReasonFromError(err) ?? void 0,
		status: signal.status,
		code: signal.code,
		provider: signal.provider
	};
}
/** Convert a classified raw error into a FailoverError with optional request context. */
function coerceToFailoverError(err, context) {
	if (isFailoverError(err)) {
		if (context?.authMode && !err.authMode || context?.timeout && !err.timeout) {
			const message = typeof err.message === "string" ? err.message : String(err);
			const enriched = new FailoverError(message, {
				reason: err.reason,
				provider: err.provider,
				model: err.model,
				profileId: err.profileId,
				authMode: err.authMode ?? context.authMode,
				status: err.status,
				code: err.code,
				rawError: err.rawError,
				authProfileFailure: err.authProfileFailure,
				sessionId: err.sessionId,
				lane: err.lane,
				cause: err.cause,
				suspend: err.suspend,
				cliTimeout: err.cliTimeout,
				timeout: err.timeout ?? context.timeout,
				attempts: err.attempts,
				soonestCooldownExpiry: err.soonestCooldownExpiry
			});
			copyErrorDiagnostic(err, enriched);
			return enriched;
		}
		return err;
	}
	const reason = resolveFailoverReasonFromError(err, context?.provider);
	if (!reason) return null;
	const signal = normalizeErrorSignal(err);
	const message = signal.message ?? String(err);
	const status = signal.status ?? resolveFailoverStatus(reason);
	const code = signal.code;
	const shouldSuspend = Boolean(context?.sessionId) && (reason === "rate_limit" || reason === "billing");
	return new FailoverError(message, {
		reason,
		provider: context?.provider ?? signal.provider,
		model: context?.model,
		profileId: context?.profileId,
		authMode: context?.authMode,
		sessionId: context?.sessionId,
		lane: context?.lane,
		status,
		code,
		rawError: message,
		cause: err instanceof Error ? err : void 0,
		timeout: context?.timeout,
		suspend: shouldSuspend
	});
}
/** Classify one candidate failure once so fallback routing and diagnostics share it. */
function resolveModelFallbackError(err, context) {
	if (err instanceof AgentHarnessSessionSupersededError) return {
		kind: "coordination",
		error: err
	};
	if (hasRuntimeCoordinationFailure(err)) return {
		kind: "coordination",
		error: err
	};
	const staleLifecycleFailure = hasStaleAgentRunLifecycleFailure(err);
	if (staleLifecycleFailure && (isAgentRunStaleLifecycleError(err) || !hasDirectProviderFailureIdentity(err))) return {
		kind: "coordination",
		error: err
	};
	if (hasSessionTranscriptWriterClaimRebound(err)) return {
		kind: "coordination",
		error: err
	};
	if (hasModelFallbackStop(err)) return {
		kind: "terminal",
		error: err
	};
	if (isAgentHarnessPreflightError(err)) return {
		kind: "coordination",
		error: err
	};
	const failoverError = coerceToFailoverError(err, context);
	if (failoverError) return {
		kind: "failover",
		error: failoverError
	};
	if (hasMissingToolResultFailure(err) || staleLifecycleFailure) return {
		kind: "coordination",
		error: err
	};
	return {
		kind: "unknown",
		error: err
	};
}
//#endregion
export { findCliTerminalStopError as a, hasProviderRequestSizeCeiling as c, recordModelFallbackStop as d, resolveFailoverReasonFromError as f, describeFailoverError as i, isCliTerminalStopCode as l, resolveModelFallbackError as m, buildProviderReauthCommand as n, findCliTimeoutError as o, resolveFailoverStatus as p, coerceToFailoverError as r, hasModelFallbackStop as s, buildFailoverRemediationHint as t, isNonProviderRuntimeCoordinationError as u };