openclaw/dist/delivery-recovery.shared-D7zFcVBa.mjs

Multi-channel AI gateway with extensible messaging integrations

import { D as resolveExpiresAtMsFromDurationMs, E as resolveDateTimestampMs, M as resolveNonNegativeIntegerOption } from "./number-coercion-CkhMiRw8.mjs";
import { a as extractErrorCode, r as collectErrorGraphCandidates } from "./error-coercion-W9BtigRm.mjs";
import { c as isRecord } from "./record-coerce-TIsquoVR.mjs";
import { r as computeBackoffSchedule } from "./src-D1BHGNTj.mjs";
import { t as sleep } from "./sleep-dMoNLoKC.mjs";
import "./errors-BvOkmQM5.mjs";
import { t as getRetryAttemptErrors } from "./retry-attempt-errors-BSlvmGqS.mjs";
import { c as isPlatformMessageNotDispatchedError, l as isPlatformMessageRejectedError, s as isOutboundDeliveryError } from "./deliver-types-D5Rd5TTU.mjs";
import { createAsyncLock } from "@openclaw/fs-safe/advanced";
//#region src/infra/delivery-recovery.shared.ts
const RECOVERY_BACKOFF_MS = [
	5e3,
	25e3,
	12e4,
	6e5
];
const RECOVERY_REPLAY_SPACING_MS = 250;
const PRE_CONNECT_ERROR_CODES = /* @__PURE__ */ new Set([
	"ECONNREFUSED",
	"ENOTFOUND",
	"EAI_AGAIN",
	"ENETDOWN",
	"ENETUNREACH",
	"EHOSTUNREACH",
	"ETIMEDOUT"
]);
const TRANSPORT_ERROR_CODE_RE = /^(?:E(?:AI_|CONN|NET|HOST|ADDR|PIPE|TIMEDOUT|SOCKET)|UND_ERR_|ERR_(?:NETWORK|HTTP2|QUIC|TLS|SSL))/;
const UNPROVEN_ERROR_BRANCH = "unproven delivery error branch";
function createEmptyDeliveryRecoverySummary() {
	return {
		recovered: 0,
		failed: 0,
		skippedMaxRetries: 0,
		deferredBackoff: 0
	};
}
function resolveDeliveryRecoveryDeadlineMs(maxRecoveryMs) {
	const durationMs = resolveNonNegativeIntegerOption(maxRecoveryMs, 6e4);
	if (durationMs <= 0) return resolveDateTimestampMs(Date.now());
	return resolveExpiresAtMsFromDurationMs(durationMs) ?? resolveDateTimestampMs(Date.now());
}
function isDeliveryRecoveryRetryEligible(entry, now) {
	if (entry.availableAt && now < entry.availableAt) return {
		eligible: false,
		remainingBackoffMs: entry.availableAt - now
	};
	const backoff = computeBackoffMs(entry.retryCount);
	if (backoff <= 0 || entry.retryCount === 0 && entry.lastAttemptAt === void 0) return { eligible: true };
	const lastAttemptAt = entry.lastAttemptAt;
	const remainingBackoffMs = (typeof lastAttemptAt === "number" && Number.isFinite(lastAttemptAt) && lastAttemptAt > 0 ? lastAttemptAt : entry.enqueuedAt) + backoff - now;
	return remainingBackoffMs > 0 ? {
		eligible: false,
		remainingBackoffMs
	} : { eligible: true };
}
function preserveProofBranches(branches) {
	return branches?.map((branch) => branch ?? UNPROVEN_ERROR_BRANCH) ?? [];
}
function isProvenPreConnectCandidate(candidate) {
	const code = extractErrorCode(candidate)?.trim().toUpperCase();
	if (code === "UND_ERR_CONNECT_TIMEOUT" || code === "UND_ERR_DNS_RESOLVE_FAILED") return true;
	if (!code || !PRE_CONNECT_ERROR_CODES.has(code) || !candidate || typeof candidate !== "object") return false;
	const syscall = candidate.syscall;
	return syscall === "connect" || syscall === "getaddrinfo";
}
function nestedErrorCandidates(current) {
	const retryBranches = preserveProofBranches(getRetryAttemptErrors(current));
	if (isPlatformMessageNotDispatchedError(current) || isProvenPreConnectCandidate(current)) return retryBranches;
	const nestedObjects = [
		current.cause,
		current.original,
		current.error,
		current.reason
	].filter((candidate) => candidate !== null && typeof candidate === "object");
	const aggregateBranches = Array.isArray(current.errors) ? preserveProofBranches(current.errors) : [];
	return [
		...retryBranches,
		...aggregateBranches,
		...nestedObjects
	];
}
function hasDeliveryNotSentProof(candidates) {
	let foundNotSentProof = false;
	for (const candidate of candidates) {
		if (isOutboundDeliveryError(candidate) && candidate.sentBeforeError) return false;
		const code = extractErrorCode(candidate)?.trim().toUpperCase();
		if (isPlatformMessageNotDispatchedError(candidate) || isProvenPreConnectCandidate(candidate)) {
			foundNotSentProof = true;
			continue;
		}
		const nested = candidate && typeof candidate === "object" ? nestedErrorCandidates(candidate) : [];
		const isPreConnectAggregateSummary = candidate !== null && typeof candidate === "object" && Array.isArray(candidate.errors) && code !== void 0 && PRE_CONNECT_ERROR_CODES.has(code);
		if (nested.length === 0 || code && !isPreConnectAggregateSummary && (PRE_CONNECT_ERROR_CODES.has(code) || TRANSPORT_ERROR_CODE_RE.test(code))) return false;
	}
	return foundNotSentProof;
}
function isProvenDeliveryNotSentError(err) {
	return hasDeliveryNotSentProof(collectErrorGraphCandidates(err, nestedErrorCandidates));
}
/** Finds a provider's permanent pre-dispatch rejection through delivery wrappers. */
function findPlatformMessageRejectedError(err) {
	return collectErrorGraphCandidates(err, nestedErrorCandidates).find(isPlatformMessageRejectedError);
}
/**
* Returns the typed no-send marker's retry decision after proving the full error graph.
* Untyped pre-connect proof stays undefined so caller-specific text policy keeps precedence.
*/
function resolveDeliveryNotSentRetryability(err) {
	return deliveryNotSentRetryability(err, true);
}
function deliveryNotSentRetryability(err, requireMarker) {
	const candidates = collectErrorGraphCandidates(err, nestedErrorCandidates);
	if (requireMarker && !candidates.some(isPlatformMessageNotDispatchedError) || !hasDeliveryNotSentProof(candidates) || hasDeliverySendEvidence(candidates)) return;
	return !candidates.some(isPlatformMessageRejectedError);
}
function hasDeliverySendEvidence(candidates) {
	return candidates.some((candidate) => isRecord(candidate) && (candidate.sentBeforeError === true || candidate.visibleReplySent === true || isRecord(candidate.deliveryResult) && candidate.deliveryResult.visibleReplySent === true));
}
/** True only when the complete error graph proves a retryable recipient no-send. */
function isRetryableDeliveryNotSentError(err) {
	return deliveryNotSentRetryability(err, false) ?? false;
}
/** True when the durable queue retained the exact failed attempt for recovery. */
function isDeliveryRecoveryOwnedRetry(err) {
	return collectErrorGraphCandidates(err, nestedErrorCandidates).some((candidate) => isOutboundDeliveryError(candidate) && candidate.queueCustody === "held");
}
function computeBackoffMs(retryCount) {
	return computeBackoffSchedule(RECOVERY_BACKOFF_MS, retryCount);
}
function getErrnoCode(err) {
	return err && typeof err === "object" && "code" in err ? String(err.code) : null;
}
function createRecoveryReplayPacer() {
	let lastReplayStartedAt = 0;
	const withLock = createAsyncLock();
	return (deadlineMs) => withLock(async () => {
		const now = Date.now();
		if (deadlineMs !== void 0 && now >= deadlineMs) return "deadline-exceeded";
		const elapsedMs = now - lastReplayStartedAt;
		const waitMs = elapsedMs < 0 ? 0 : Math.max(0, RECOVERY_REPLAY_SPACING_MS - elapsedMs);
		if (waitMs > 0) {
			const remainingBudgetMs = deadlineMs === void 0 ? waitMs : Math.max(0, deadlineMs - now);
			await sleep(Math.min(waitMs, remainingBudgetMs));
		}
		if (deadlineMs !== void 0 && Date.now() >= deadlineMs) return "deadline-exceeded";
		lastReplayStartedAt = Date.now();
		return "ready";
	});
}
function createRecoveryClaim() {
	const activeEntries = /* @__PURE__ */ new Set();
	return async (entryId, run) => {
		if (activeEntries.has(entryId)) return { status: "claimed-by-other-owner" };
		activeEntries.add(entryId);
		try {
			return {
				status: "claimed",
				value: await run()
			};
		} finally {
			activeEntries.delete(entryId);
		}
	};
}
/** Own one queue namespace's live claims, drain exclusion, and replay pacing. */
function createDeliveryRecoveryCoordinator() {
	const withClaim = createRecoveryClaim();
	const withDrainClaim = createRecoveryClaim();
	return {
		withClaim,
		async withDrain(drainKey, run) {
			return (await withDrainClaim(drainKey, run)).status === "claimed";
		},
		async scan(options) {
			for (const entry of options.entries.toSorted((a, b) => a.enqueuedAt - b.enqueuedAt)) {
				if (options.deadlineMs !== void 0 && Date.now() >= options.deadlineMs) {
					options.onDeadlineExceeded?.();
					break;
				}
				const claim = await withClaim(entry.id, async () => {
					const current = await options.loadEntry(entry.id);
					if (!current) {
						options.onMissingEntry?.(entry);
						return "continue";
					}
					return await options.onEntry(current) ?? "continue";
				});
				if (claim.status === "claimed-by-other-owner") {
					options.onClaimConflict?.(entry);
					continue;
				}
				if (claim.value === "stop") break;
			}
		},
		waitForReplay: createRecoveryReplayPacer()
	};
}
//#endregion
export { getErrnoCode as a, isProvenDeliveryNotSentError as c, resolveDeliveryRecoveryDeadlineMs as d, findPlatformMessageRejectedError as i, isRetryableDeliveryNotSentError as l, createDeliveryRecoveryCoordinator as n, isDeliveryRecoveryOwnedRetry as o, createEmptyDeliveryRecoverySummary as r, isDeliveryRecoveryRetryEligible as s, computeBackoffMs as t, resolveDeliveryNotSentRetryability as u };