openclaw/dist/server-restart-sentinel-ZII26Umm.js

Multi-channel AI gateway with extensible messaging integrations

import { l as toErrorObject } from "./error-coercion-D_-xJ90S.js";
import { n as getRuntimeConfig } from "./io.runtime-B9iJRs3w.js";
import { w as resolveStateDir } from "./paths-D2sRr1a_.js";
import { t as formatErrorMessage } from "./errors-Db3Ymjlb.js";
import { t as createSubsystemLogger } from "./subsystem-Dy2tqXOS.js";
import { l as resolveRuntimeServiceVersion, s as resolveRuntimeServiceBuildId } from "./version-v1kuAkGj.js";
import "./io-bdCzpGWJ.js";
import { u as getActivePluginRegistry } from "./runtime-BL4wZfTq.js";
import { b as runWithGatewayIndependentRootWorkAdmission } from "./gateway-work-admission-R1IpuDim.js";
import { t as INTERNAL_MESSAGE_CHANNEL } from "./message-channel-constants-2zSoJXQC.js";
import "./message-channel-BQrhwUEA.js";
import { i as getUpdateRun, l as recordUpdateRunVerification, o as recordUpdateRunPhase, r as finishUpdateRun } from "./update-run-ledger-Dse_-g6v.js";
import { s as getMediaDir } from "./store-Ph57PZ9P.js";
import { c as normalizeDeliveryContext, o as mergeDeliveryContext } from "./delivery-context.shared-CXmRgetN.js";
import { l as requestHeartbeat } from "./heartbeat-wake-dlWl8dXS.js";
import { a as enqueueSystemEvent, h as withSystemEventOwner } from "./system-events-C03jsM0j.js";
import { c as resolveSystemMainSessionTarget } from "./main-session-Br0F9dzh.js";
import { a as hasRestartRecoveryTerminalRun, n as getRestartRecoveryTerminalDeliveryEvidence } from "./restart-recovery-state-DDdTIiD6.js";
import { L as resolveDurableCompletionDeliveryMode } from "./task-registry-BP9SCk1B.js";
import { i as finalizeUpdateRestartSentinelRunningVersion, l as readRestartSentinel, o as formatRestartSentinelMessage, p as summarizeRestartSentinel, r as clearRestartSentinelIfRevision } from "./restart-sentinel-FmKS9LVR.js";
import { a as isPendingControlPlaneUpdateRestartSentinel } from "./update-control-plane-sentinel-BN5TeCL_.js";
import "./sessions-9nxpeTwt.js";
import { t as appendAssistantMessageToSessionTranscript } from "./transcript-Dv8-hmIo.js";
import { t as finalizeInboundContext } from "./inbound-context-BJXKd4x8.js";
import { t as normalizeMediaReferenceForComparison } from "./media-reference-comparison-DpA8NCy-.js";
import { l as drainPendingSessionDelivery, u as recoverPendingSessionDeliveries } from "./subagent-completion-admission.store-ieYRlFt3.js";
import { r as loadGatewaySessionEntry } from "./session-utils-store-CInT2loy.js";
import "./session-utils-Cai0_C6U.js";
import { r as hasExplicitlyVisibleAgentPayload } from "./message-visibility-DVJxwfNj.js";
import { _ as markSessionDeliverySettlement, a as SessionDeliveryDeferredError, c as advanceSessionDeliveryAgentRun, f as enqueueSessionDelivery, g as markSessionDeliveryAttemptStarted, i as SessionDeliveryDeadLetteredError, o as SessionDeliveryRetryChargedError, p as failSessionDelivery, s as SessionDeliverySafeRetryError, u as deferSessionDelivery, v as mergeSessionDeliveryPreparedMediaBlocks } from "./session-delivery-queue-storage-ClcsFC9e.js";
import { a as getAgentCommandDeliveryFailure, d as hasCompleteAutomaticMediaDeliveryOutcomeEvidence, l as hasCommittedOutboundDeliveryEvidence, n as collectAutomaticDeliveredMediaUrls, r as collectDeliveredMediaUrls, s as getGatewayAgentResult, t as collectAmbiguousAutomaticMediaUrls } from "./delivery-evidence-5HyXxfSi.js";
import { f as formatGeneratedMediaDeliveryRetryForPrompt } from "./subagent-announce-delivery-CB6b5kft.js";
import { t as removeCronRunContinuationSessionIfIdle } from "./cron-run-continuation-cleanup-D93ByBcx.js";
import { a as settleCorrelatedSubagentDelivery, r as resolveCorrelatedSubagentDelivery } from "./subagent-completion-delivery-BDvv1caT.js";
import { t as dispatchGatewayLifecycleMethod } from "./server-recovery-runtime-context-Cx7vLPdb.js";
import { t as dispatchReplyWithBufferedBlockDispatcherCore } from "./provider-dispatcher-DyFXY5kS.js";
import { t as recordInboundSession } from "./session-CGo-oQbb.js";
import { n as dispatchAssembledChannelTurn } from "./lifecycle-DQ-vfJ_s.js";
import "./get-reply-run-queue-Bmv2zHLu.js";
import { t as runStartupTasks } from "./startup-tasks-BkYwMCKQ.js";
import { i as attachManagedOutgoingMediaToMessage, s as createManagedOutgoingMediaBlocks } from "./managed-image-attachments-CwIi4Xb-.js";
import { i as updateRunReportInputFromSentinel, n as renderUpdateRunReport } from "./update-run-report-CEiRumC4.js";
import { n as deliverRestartSentinelNotice, r as enqueueRestartSentinelNotice, t as resolveUpdateRunNoticeTarget } from "./update-run-notice-target-FgJmCpEJ.js";
//#region src/gateway/server-restart-sentinel-agent-delivery.ts
const log$1 = createSubsystemLogger("gateway/restart-sentinel");
const AGENT_DELIVERY_OWNERSHIP_RETRY_MS = 1e3;
function sessionDeliveryStateDirArgs$1(stateDir) {
	return stateDir === void 0 ? [] : [stateDir];
}
async function deadLetterSessionDelivery(entry, reason, stateDir) {
	await markSessionDeliverySettlement(entry, "moved-to-failed", ...sessionDeliveryStateDirArgs$1(stateDir));
	log$1.warn("queued session delivery requires durable dead-letter settlement", { queueId: entry.id });
	throw new SessionDeliveryDeadLetteredError(reason);
}
function hasQueuedVisibleAgentPayload(result) {
	return Array.isArray(result.payloads) && result.payloads.some(hasExplicitlyVisibleAgentPayload);
}
function hasUnexpectedRecoverySideEffects(result) {
	return result.restartUnsafeSideEffectsDetected === true || result.messagingToolAggregateEvidenceUnaccounted === true || result.messagingToolSentTargetsTruncated === true || result.didSendDeterministicApprovalPrompt === true || hasCommittedOutboundDeliveryEvidence(result);
}
function resolveQueuedAgentRunId(entry) {
	const base = entry.idempotencyKey ?? entry.messageId;
	return entry.agentRunAttempt ? `${base}:attempt:${entry.agentRunAttempt}` : base;
}
function collectVisiblePayloadMediaUrls(result) {
	const urls = /* @__PURE__ */ new Set();
	const payloads = Array.isArray(result.payloads) ? result.payloads : [];
	for (const payload of payloads) {
		if (!hasExplicitlyVisibleAgentPayload(payload)) continue;
		for (const url of collectDeliveredMediaUrls({ payloads: [payload] })) urls.add(url);
	}
	return Array.from(urls);
}
function collectQueuedDeliveredMediaUrls(params) {
	if (params.route.channel === "webchat") return collectVisiblePayloadMediaUrls(params.result);
	return collectAutomaticDeliveredMediaUrls(params.result);
}
function hasAutomaticVisibleSendEvidence(result) {
	if (result.deliveryStatus?.status === "sent" || result.deliveryStatus?.status === "suppressed") return hasQueuedVisibleAgentPayload(result);
	const payloads = Array.isArray(result.payloads) ? result.payloads : [];
	return (Array.isArray(result.deliveryStatus?.payloadOutcomes) ? result.deliveryStatus.payloadOutcomes : []).some((outcome) => {
		if (!outcome || typeof outcome !== "object" || Array.isArray(outcome)) return false;
		const record = outcome;
		if (record.status !== "sent" && record.status !== "suppressed" && record.sentBeforeError !== true) return false;
		const index = typeof record.index === "number" && Number.isInteger(record.index) ? record.index : void 0;
		return index !== void 0 && hasExplicitlyVisibleAgentPayload(payloads[index]);
	});
}
function hasQueuedVisibleReplyEvidence(params) {
	if (params.route.channel === "webchat") return hasQueuedVisibleAgentPayload(params.result);
	return hasAutomaticVisibleSendEvidence(params.result);
}
async function evaluateQueuedGeneratedMediaAgentResult(params) {
	if (hasUnexpectedRecoverySideEffects(params.result)) {
		log$1.warn("queued generated-media recovery reported an unexpected committed side effect", { queueId: params.entry.id });
		await deadLetterSessionDelivery(params.entry, "queued generated-media delivery dead-lettered after an unexpected committed side effect", params.stateDir);
	}
	const expectedMediaUrls = params.entry.expectedMediaUrls ?? [];
	const deliveredMediaUrls = new Set(collectQueuedDeliveredMediaUrls(params).map(normalizeMediaReferenceForComparison));
	const isDelivered = (url) => deliveredMediaUrls.has(normalizeMediaReferenceForComparison(url));
	const missingMediaUrls = expectedMediaUrls.filter((url) => !isDelivered(url));
	const provenExpectedMediaUrls = expectedMediaUrls.filter(isDelivered);
	const ambiguousMediaUrls = new Set(collectAmbiguousAutomaticMediaUrls(params.result).map(normalizeMediaReferenceForComparison));
	const deliveryFailure = getAgentCommandDeliveryFailure(params.result);
	const replySatisfied = expectedMediaUrls.length > 0 ? missingMediaUrls.length === 0 : hasQueuedVisibleReplyEvidence(params);
	if (params.result.payloadsTruncated === true && !replySatisfied) {
		log$1.warn("queued generated-media delivery has truncated delivery evidence", { queueId: params.entry.id });
		await deadLetterSessionDelivery(params.entry, "queued generated-media delivery dead-lettered after truncated evidence", params.stateDir);
	}
	if (expectedMediaUrls.length > 0 && missingMediaUrls.length === 0) {
		await params.persistInternalMedia?.(provenExpectedMediaUrls);
		return;
	}
	const rearmAgentRun = async (reason, updates) => {
		const currentAgentRunAttempt = params.entry.agentRunAttempt ?? 0;
		if (!(params.entry.lastChargedAgentRunAttempt === currentAgentRunAttempt)) await failSessionDelivery(params.entry.id, reason, ...sessionDeliveryStateDirArgs$1(params.stateDir));
		try {
			if (updates) await advanceSessionDeliveryAgentRun(params.entry.id, updates, ...sessionDeliveryStateDirArgs$1(params.stateDir));
			else if (params.stateDir !== void 0) await advanceSessionDeliveryAgentRun(params.entry.id, void 0, params.stateDir);
			else await advanceSessionDeliveryAgentRun(params.entry.id);
			await deferSessionDelivery(params.entry.id, AGENT_DELIVERY_OWNERSHIP_RETRY_MS, ...sessionDeliveryStateDirArgs$1(params.stateDir));
		} catch (error) {
			log$1.warn("queued generated-media terminal attempt state transition remains pending", {
				queueId: params.entry.id,
				error: String(error)
			});
			throw new SessionDeliveryRetryChargedError(`${reason}; queue state transition failed after retry charge`);
		}
		throw new SessionDeliveryDeferredError(reason);
	};
	if (deliveryFailure && expectedMediaUrls.length > 0) {
		if (params.result.deliveryStatus?.status === "partial_failed" && !hasCompleteAutomaticMediaDeliveryOutcomeEvidence(params.result, missingMediaUrls) || missingMediaUrls.some((url) => ambiguousMediaUrls.has(normalizeMediaReferenceForComparison(url)))) {
			log$1.warn("queued generated-media delivery has ambiguous attachment side effects", {
				queueId: params.entry.id,
				error: deliveryFailure
			});
			await deadLetterSessionDelivery(params.entry, "queued generated-media delivery dead-lettered after ambiguous side effects", params.stateDir);
		}
	} else if (deliveryFailure) {
		if (hasQueuedVisibleReplyEvidence(params)) {
			log$1.warn("queued generated-media notice may already be visible; refusing duplicate replay", {
				queueId: params.entry.id,
				error: deliveryFailure
			});
			await deadLetterSessionDelivery(params.entry, "queued generated-media notice dead-lettered after a visible partial delivery", params.stateDir);
		}
		await rearmAgentRun(deliveryFailure);
	}
	if (provenExpectedMediaUrls.length > 0) await params.persistInternalMedia?.(provenExpectedMediaUrls);
	if (missingMediaUrls.length > 0) {
		const retryMessage = formatGeneratedMediaDeliveryRetryForPrompt(missingMediaUrls);
		await rearmAgentRun(`queued generated-media agent turn ${missingMediaUrls.length < expectedMediaUrls.length ? "partially missed" : "missed"} expected media: ${missingMediaUrls.join(", ")}`, {
			expectedMediaUrls: missingMediaUrls,
			...missingMediaUrls.length < expectedMediaUrls.length || hasQueuedVisibleReplyEvidence(params) || params.result.deliveryStatus?.status === "partial_failed" ? { suppressTextDelivery: true } : {},
			...retryMessage ? { message: retryMessage } : {}
		});
	}
	if (expectedMediaUrls.length === 0 && !hasQueuedVisibleReplyEvidence(params)) await rearmAgentRun("queued generated-media agent turn completed without a visible reply");
}
/** Runs durable generated-media handoffs through the normal owning-session agent loop. */
async function deliverQueuedGeneratedMediaAgentTurn(params) {
	if (params.entry.kind !== "agentTurn") return false;
	const entry = params.entry;
	const route = entry.route;
	if (!route || entry.inputProvenance?.kind !== "inter_session" || !entry.sourceReplyDeliveryMode) return false;
	const queuedRunId = resolveQueuedAgentRunId(entry);
	if (resolveDurableCompletionDeliveryMode(entry.sourceReplyDeliveryMode) === "host_owned" && route.channel === "webchat") return await deadLetterSessionDelivery(entry, "queued host-owned generated-media delivery requires an external route", params.stateDir);
	const persistInternalMedia = route.channel === "webchat" && (entry.expectedMediaUrls?.length ?? 0) > 0 ? async (mediaUrls) => {
		const sessionId = params.sessionEntry?.sessionId?.trim();
		if (!sessionId) throw new Error("queued internal generated-media delivery has no owning session");
		const stateDir = params.stateDir ?? resolveStateDir();
		const preparedMediaBlocks = { ...entry.preparedMediaBlocks };
		const content = [];
		for (const mediaUrl of mediaUrls) {
			let blocks = preparedMediaBlocks[mediaUrl];
			if (!blocks) {
				const attachment = entry.expectedMediaAttachments?.[mediaUrl];
				blocks = await createManagedOutgoingMediaBlocks({
					sessionKey: params.canonicalKey,
					agentId: params.agentId,
					items: [{
						url: mediaUrl,
						...attachment?.name ? { filename: attachment.name } : {},
						...attachment?.mimeType ? { mimeType: attachment.mimeType } : {},
						trustedLocal: true,
						...attachment?.durationMs !== void 0 ? { durationMs: attachment.durationMs } : {},
						...attachment?.width !== void 0 ? { width: attachment.width } : {},
						...attachment?.height !== void 0 ? { height: attachment.height } : {}
					}],
					stateDir,
					localRoots: [getMediaDir()]
				});
				if (!blocks.some((block) => block.type === "image" || block.type === "audio" || block.type === "video" || block.type === "attachment")) throw new Error("queued internal generated media could not be prepared");
				blocks = await mergeSessionDeliveryPreparedMediaBlocks(entry.id, mediaUrl, blocks, stateDir);
				preparedMediaBlocks[mediaUrl] = blocks;
			}
			content.push(...blocks);
		}
		const appended = await appendAssistantMessageToSessionTranscript({
			agentId: params.agentId,
			sessionKey: params.canonicalKey,
			storePath: params.storePath,
			expectedSessionId: sessionId,
			...params.sessionEntry?.cronRunContinuation?.lifecycleRevision ? { expectedLifecycleRevision: params.sessionEntry.cronRunContinuation.lifecycleRevision } : {},
			content: [],
			displayContent: content,
			idempotencyKey: `${queuedRunId}:generated-media-transcript`,
			updateMode: "inline"
		});
		if (!appended.ok) {
			if (appended.code === "session-rebound") await deadLetterSessionDelivery(entry, "queued internal generated-media delivery lost its owning session", params.stateDir);
			throw new Error(`queued internal generated-media transcript persistence failed: ${appended.reason}`);
		}
		if (!attachManagedOutgoingMediaToMessage({
			messageId: appended.messageId,
			blocks: content,
			stateDir
		})) throw new Error("queued internal generated-media artifact attachment failed");
	} : void 0;
	const evaluateResult = async (result) => {
		await evaluateQueuedGeneratedMediaAgentResult({
			entry,
			result,
			route,
			...params.stateDir !== void 0 ? { stateDir: params.stateDir } : {},
			...persistInternalMedia ? { persistInternalMedia } : {}
		});
		return true;
	};
	const terminalEvidence = getRestartRecoveryTerminalDeliveryEvidence(params.sessionEntry, queuedRunId);
	if (terminalEvidence) return await evaluateResult(terminalEvidence);
	if (hasRestartRecoveryTerminalRun(params.sessionEntry, queuedRunId)) await deadLetterSessionDelivery(entry, "queued generated-media agent turn dead-lettered without durable terminal evidence", params.stateDir);
	if (params.sessionEntry?.restartRecoveryDeliverySourceRunId === queuedRunId && Boolean(params.sessionEntry.restartRecoveryDeliveryRunId)) {
		await deferSessionDelivery(entry.id, AGENT_DELIVERY_OWNERSHIP_RETRY_MS, ...sessionDeliveryStateDirArgs$1(params.stateDir));
		throw new SessionDeliveryDeferredError("queued generated-media agent turn is still owned by agent recovery");
	}
	if (entry.deliveryStartedAt !== void 0) await deadLetterSessionDelivery(entry, "queued generated-media agent turn dead-lettered after an interrupted unproven attempt", params.stateDir);
	const sourceReplyDeliveryMode = "automatic";
	const cronSessionId = params.sessionEntry?.cronRunContinuation?.lifecycleRevision?.trim() ? params.sessionEntry?.sessionId?.trim() : void 0;
	await markSessionDeliveryAttemptStarted(entry, ...sessionDeliveryStateDirArgs$1(params.stateDir));
	let accepted = false;
	let response;
	try {
		response = await dispatchGatewayLifecycleMethod("agent", {
			sessionKey: params.canonicalKey,
			message: entry.message,
			deliver: route.channel !== INTERNAL_MESSAGE_CHANNEL,
			bestEffortDeliver: false,
			channel: route.channel,
			accountId: route.accountId,
			to: route.to,
			threadId: route.threadId,
			...cronSessionId ? { sessionId: cronSessionId } : {},
			inputProvenance: entry.inputProvenance,
			sourceReplyDeliveryMode,
			disableMessageTool: true,
			forceRestartSafeTools: true,
			idempotencyKey: queuedRunId
		}, {
			...cronSessionId ? { allowSyntheticCronRunContinuation: true } : {},
			expectFinal: true,
			forceSyntheticClient: true,
			internalDeliveryMediaUrls: entry.expectedMediaUrls ?? [],
			...entry.suppressTextDelivery === true ? { internalDeliverySuppressText: true } : {},
			...params.resolveGatewayContext ? { resolveGatewayContext: params.resolveGatewayContext } : {},
			onAccepted: () => {
				accepted = true;
			}
		});
	} catch (error) {
		if (!accepted) throw new SessionDeliverySafeRetryError("queued generated-media agent turn failed before gateway acceptance", { cause: error });
		throw error;
	}
	const result = getGatewayAgentResult(response);
	if (!result) {
		const responseStatus = response && typeof response === "object" ? response.status : void 0;
		const latestEntry = loadGatewaySessionEntry(entry.sessionKey).entry;
		if (responseStatus === "accepted") accepted = true;
		if (responseStatus === "accepted" || responseStatus === "in_flight" || latestEntry?.restartRecoveryDeliverySourceRunId === queuedRunId && latestEntry.restartRecoveryDeliveryRunId) {
			await deferSessionDelivery(entry.id, AGENT_DELIVERY_OWNERSHIP_RETRY_MS, ...sessionDeliveryStateDirArgs$1(params.stateDir));
			throw new SessionDeliveryDeferredError("queued generated-media agent turn is still owned by agent recovery");
		}
		if (hasRestartRecoveryTerminalRun(latestEntry, queuedRunId)) {
			const latestTerminalEvidence = getRestartRecoveryTerminalDeliveryEvidence(latestEntry, queuedRunId);
			if (latestTerminalEvidence) return await evaluateResult(latestTerminalEvidence);
			log$1.warn("queued generated-media agent turn ended without durable delivery evidence; failing closed", {
				queueId: entry.id,
				runId: queuedRunId
			});
			await deadLetterSessionDelivery(entry, "queued generated-media agent turn dead-lettered without durable terminal evidence", params.stateDir);
		}
		if (!accepted) throw new SessionDeliverySafeRetryError("queued generated-media agent turn returned no result before gateway acceptance");
		throw new Error("queued generated-media agent turn returned no delivery result");
	}
	return await evaluateResult(result);
}
//#endregion
//#region src/gateway/server-restart-update-run.ts
/** The booting Gateway records only its own observations; the CLI owns pending handoffs. */
async function finalizeRestartUpdateRun(payload, pendingExpired = false) {
	const updateRunId = payload.stats?.runId;
	let updateRun = updateRunId ? getUpdateRun(updateRunId) : void 0;
	if (updateRun) {
		if (!updateRun.origin.sessionKey && payload.sessionKey) updateRun = recordUpdateRunPhase(updateRun.runId, updateRun.phase, { origin: {
			sessionKey: payload.sessionKey,
			...payload.deliveryContext ? { deliveryContext: {
				...payload.deliveryContext,
				threadId: payload.threadId
			} } : {}
		} });
		if (updateRun.status === "running" && (updateRun.phase === "restarting" || updateRun.phase === "verifying")) updateRun = recordUpdateRunPhase(updateRun.runId, "verifying");
		const runningVersion = resolveRuntimeServiceVersion();
		const runningBuildId = resolveRuntimeServiceBuildId();
		const expectedVersion = updateRun.after.version ?? updateRun.target.version;
		const expectedBuildId = updateRun.after.buildId;
		const pluginErrors = getActivePluginRegistry()?.diagnostics.filter((entry) => entry.level === "error").map((entry) => entry.message);
		updateRun = recordUpdateRunVerification(updateRun.runId, {
			booted: true,
			serviceRunning: true,
			pid: process.pid,
			runningVersion,
			...runningBuildId ? { runningBuildId } : {},
			...expectedVersion ? { versionMatch: expectedVersion === runningVersion && (!expectedBuildId || expectedBuildId === runningBuildId) } : {},
			...pluginErrors ? { pluginErrors } : {},
			...payload.doctorHint ? { doctorHint: payload.doctorHint } : {}
		});
		if (updateRun.phase === "verifying" && updateRun.status === "running") {
			const { createUpdateRunNotifier } = await import("./update-run-notice.runtime.js");
			await createUpdateRunNotifier(updateRun)(updateRun, "verifying");
		}
		if (pendingExpired || !isPendingControlPlaneUpdateRestartSentinel(payload)) updateRun = finishUpdateRun(updateRun.runId, {
			status: pendingExpired || payload.status === "error" || updateRun.verification.versionMatch === false ? "failed" : payload.status === "ok" ? "succeeded" : "skipped",
			reason: pendingExpired || updateRun.verification.versionMatch === false ? "restart-unhealthy" : payload.stats?.reason ?? void 0,
			after: {
				version: runningVersion,
				...runningBuildId ? { buildId: runningBuildId } : {}
			}
		});
	}
	return updateRun;
}
//#endregion
//#region src/gateway/server-restart-sentinel.ts
const log = createSubsystemLogger("gateway/restart-sentinel");
const RESTART_CONTINUATION_BUSY_RETRY_DELAY_MS = process.env.VITEST ? 1 : 6e3;
const RESTART_CONTINUATION_BUSY_MAX_ATTEMPTS = 20;
const CONTROL_PLANE_UPDATE_PENDING_RETRY_DELAY_MS = process.env.VITEST ? 1 : 2e3;
const CONTROL_PLANE_UPDATE_PENDING_MAX_ATTEMPTS = 900;
const RESTART_CONTINUATION_BUSY_RETRY_ERROR = "restart continuation deferred because previous run is still shutting down";
let latestUpdateRestartSentinel = null;
/** Settles every queue entry through its durable producer before cron cleanup. */
const settleQueuedSessionDelivery = async (entry, outcome) => {
	await settleCorrelatedSubagentDelivery(entry, outcome);
	await removeCronRunContinuationSessionIfIdle(entry.sessionKey, entry.id);
};
function sessionDeliveryStateDirArgs(stateDir) {
	return stateDir === void 0 ? [] : [stateDir];
}
function cloneRestartSentinelPayload(payload) {
	return payload ? structuredClone(payload) : null;
}
function enqueueRestartSentinelWake(message, sessionKey, agentId, deliveryContext) {
	const eventOptions = {
		sessionKey,
		...deliveryContext ? { deliveryContext } : {}
	};
	enqueueSystemEvent(message, agentId ? withSystemEventOwner(eventOptions, agentId) : eventOptions);
	requestHeartbeat({
		source: "restart-sentinel",
		intent: "immediate",
		reason: "wake",
		...agentId ? { agentId } : {},
		sessionKey
	});
}
async function waitForRetry(delayMs) {
	await new Promise((resolve) => {
		setTimeout(resolve, delayMs).unref?.();
	});
}
const buildRestartContinuationMessageId = (params) => `restart-sentinel:${params.sessionKey}:${params.kind}:${params.revision}`;
function isRestartContinuationBusyPayload(payload) {
	return typeof payload.text === "string" && payload.text.trim() === "⚠️ Previous run is still shutting down. Please try again in a moment.";
}
function isRestartContinuationBusyRetry(entry) {
	return entry?.lastError === RESTART_CONTINUATION_BUSY_RETRY_ERROR;
}
function resolveQueuedRestartContinuationMessageId(entry) {
	if (isRestartContinuationBusyRetry(entry) && entry.retryCount > 0) return `${entry.messageId}:retry:${entry.retryCount}`;
	return entry.messageId;
}
function resolveQueuedSessionDeliveryContext(entry) {
	if (entry.kind === "agentTurn" && entry.route) return {
		channel: entry.route.channel,
		to: entry.route.to,
		...entry.route.accountId ? { accountId: entry.route.accountId } : {},
		...entry.route.threadId ? { threadId: entry.route.threadId } : {}
	};
	return entry.deliveryContext;
}
async function deliverQueuedSessionDelivery(params) {
	const queuedEntry = resolveCorrelatedSubagentDelivery(params.entry);
	const { cfg, agentId, entry, storePath, canonicalKey } = loadGatewaySessionEntry(queuedEntry.sessionKey);
	const deliveryContext = resolveQueuedSessionDeliveryContext(queuedEntry);
	if (queuedEntry.kind === "systemEvent") {
		const { agentId: systemEventAgentId, text } = queuedEntry;
		enqueueRestartSentinelWake(text, canonicalKey, systemEventAgentId, deliveryContext);
		return;
	}
	if (queuedEntry.expectedSessionId && (!entry?.sessionId || entry.sessionId !== queuedEntry.expectedSessionId)) {
		log.warn("restart continuation skipped: session changed", {
			sessionKey: canonicalKey,
			queueId: queuedEntry.id,
			expectedSessionId: queuedEntry.expectedSessionId,
			actualSessionId: entry?.sessionId ?? null
		});
		enqueueRestartSentinelWake(queuedEntry.message, canonicalKey, void 0, deliveryContext);
		return;
	}
	if (!queuedEntry.route) {
		enqueueRestartSentinelWake(queuedEntry.message, canonicalKey, void 0, deliveryContext);
		return;
	}
	if (await deliverQueuedGeneratedMediaAgentTurn({
		entry: queuedEntry,
		canonicalKey,
		agentId,
		storePath,
		sessionEntry: entry,
		...params.stateDir !== void 0 ? { stateDir: params.stateDir } : {},
		...params.resolveGatewayContext ? { resolveGatewayContext: params.resolveGatewayContext } : {}
	})) return;
	if (queuedEntry.deliveryStartedAt !== void 0) {
		await markSessionDeliverySettlement(queuedEntry, "moved-to-failed", ...sessionDeliveryStateDirArgs(params.stateDir));
		throw new SessionDeliveryDeadLetteredError("queued agent turn dead-lettered after an interrupted unproven attempt");
	}
	const route = queuedEntry.route;
	const messageId = resolveQueuedRestartContinuationMessageId(queuedEntry);
	const userMessage = queuedEntry.message.trim();
	let dispatchError;
	const ctxPayload = finalizeInboundContext({
		Body: userMessage,
		BodyForAgent: userMessage,
		BodyForCommands: "",
		RawBody: userMessage,
		CommandBody: "",
		SessionKey: canonicalKey,
		AccountId: route.accountId,
		MessageSid: messageId,
		Timestamp: Date.now(),
		InputProvenance: {
			kind: "internal_system",
			sourceChannel: route.channel,
			sourceTool: "restart-sentinel"
		},
		Provider: INTERNAL_MESSAGE_CHANNEL,
		Surface: INTERNAL_MESSAGE_CHANNEL,
		ChatType: route.chatType,
		CommandAuthorized: true,
		GatewayClientScopes: ["operator.admin"],
		GatewayClientCaps: [],
		ReplyToId: route.replyToId,
		OriginatingChannel: route.channel,
		OriginatingTo: route.to,
		ExplicitDeliverRoute: false,
		MessageThreadId: route.threadId
	}, {
		forceBodyForCommands: true,
		forceChatType: true
	});
	await dispatchAssembledChannelTurn({
		cfg,
		channel: route.channel,
		accountId: route.accountId,
		agentId,
		routeSessionKey: canonicalKey,
		storePath,
		ctxPayload,
		recordInboundSession,
		dispatchReplyWithBufferedBlockDispatcher: dispatchReplyWithBufferedBlockDispatcherCore,
		replyOptions: { sourceReplyDeliveryMode: "message_tool_only" },
		turnAdoptionLifecycle: {
			admission: "cancel-only",
			onAdopted: () => markSessionDeliveryAttemptStarted(queuedEntry, ...sessionDeliveryStateDirArgs(params.stateDir))
		},
		delivery: {
			preparePayload: (payload) => {
				if (isRestartContinuationBusyPayload(payload)) throw new SessionDeliverySafeRetryError(RESTART_CONTINUATION_BUSY_RETRY_ERROR);
				return payload;
			},
			durable: false,
			deliver: async () => ({ visibleReplySent: false }),
			onError: (err, info) => {
				dispatchError ??= err;
				log.warn(`restart continuation dispatch failed during ${info.kind}: ${String(err)}`, { sessionKey: canonicalKey });
			}
		},
		record: { onRecordError: (err) => {
			log.warn(`restart continuation failed to record inbound session metadata: ${String(err)}`, { sessionKey: canonicalKey });
		} }
	});
	if (dispatchError) throw toErrorObject(dispatchError, "Non-Error thrown");
}
function buildQueuedRestartContinuation(params) {
	const idempotencyKey = params.idempotencyKey ?? buildRestartContinuationMessageId({
		sessionKey: params.sessionKey,
		kind: params.continuation.kind,
		revision: params.revision
	});
	if (params.continuation.kind === "systemEvent") return {
		kind: "systemEvent",
		sessionKey: params.sessionKey,
		...params.agentId ? { agentId: params.agentId } : {},
		text: params.continuation.text,
		...params.deliveryContext ? { deliveryContext: params.deliveryContext } : {},
		idempotencyKey,
		maxRetries: RESTART_CONTINUATION_BUSY_MAX_ATTEMPTS,
		completionRetention: "permanent"
	};
	return {
		kind: "agentTurn",
		sessionKey: params.sessionKey,
		message: params.continuation.message,
		messageId: idempotencyKey,
		...params.expectedSessionId ? { expectedSessionId: params.expectedSessionId } : {},
		maxRetries: RESTART_CONTINUATION_BUSY_MAX_ATTEMPTS,
		completionRetention: "permanent",
		...params.route ? { route: params.route } : {},
		...params.deliveryContext ? { deliveryContext: params.deliveryContext } : {},
		idempotencyKey
	};
}
async function drainRestartContinuationQueue(params) {
	for (let attempt = 1; attempt <= RESTART_CONTINUATION_BUSY_MAX_ATTEMPTS; attempt += 1) {
		if (!isRestartContinuationBusyRetry(await drainPendingSessionDelivery({
			id: params.entryId,
			logLabel: "restart continuation",
			log: params.log,
			bypassBackoff: true,
			deliver: (entry, context = {}) => deliverQueuedSessionDelivery({
				deps: params.deps,
				entry,
				...context.stateDir !== void 0 ? { stateDir: context.stateDir } : {}
			}),
			onSettled: settleQueuedSessionDelivery
		}))) return;
		if (attempt >= RESTART_CONTINUATION_BUSY_MAX_ATTEMPTS) return;
		params.log.info(`restart continuation: entry ${params.entryId} still waiting for the previous run to clear; retrying in ${RESTART_CONTINUATION_BUSY_RETRY_DELAY_MS}ms`);
		await waitForRetry(RESTART_CONTINUATION_BUSY_RETRY_DELAY_MS);
	}
}
async function recoverPendingRestartContinuationDeliveries(params) {
	await recoverPendingSessionDeliveries({
		deliver: (entry, context = {}) => deliverQueuedSessionDelivery({
			deps: params.deps,
			entry,
			...context.stateDir !== void 0 ? { stateDir: context.stateDir } : {},
			...params.resolveGatewayContext ? { resolveGatewayContext: params.resolveGatewayContext } : {}
		}),
		log: params.log ?? log,
		maxEnqueuedAt: params.maxEnqueuedAt,
		onSettled: settleQueuedSessionDelivery
	});
}
async function loadRestartSentinelStartupTask(params) {
	const sentinel = await readRestartSentinel();
	if (!sentinel) return null;
	const payload = sentinel.payload;
	const sentinelRevision = sentinel.revision;
	if (payload.kind === "update") recordLatestUpdateRestartSentinel(payload);
	const sessionKey = payload.sessionKey?.trim();
	const message = formatRestartSentinelMessage(payload);
	const updateRun = payload.kind === "update" ? await finalizeRestartUpdateRun(payload) : void 0;
	const updateRunId = updateRun?.runId;
	let noticeMessage = payload.kind === "update" ? renderUpdateRunReport(updateRun ?? updateRunReportInputFromSentinel(payload)).markdown : message;
	const summary = summarizeRestartSentinel(payload);
	const wakeDeliveryContext = mergeDeliveryContext(payload.threadId != null ? {
		...payload.deliveryContext,
		threadId: payload.threadId
	} : payload.deliveryContext, void 0);
	const run = async () => {
		let routedSessionKey = sessionKey;
		let wakeAgentId;
		if (isPendingControlPlaneUpdateRestartSentinel(payload) && (!updateRun || updateRun.status === "running")) {
			const attempt = params.attempt ?? 0;
			if (attempt < CONTROL_PLANE_UPDATE_PENDING_MAX_ATTEMPTS) {
				setTimeout(() => {
					runWithGatewayIndependentRootWorkAdmission(async () => {
						await scheduleRestartSentinelWakeAttempt({
							deps: params.deps,
							attempt: attempt + 1
						});
					}, "restart-sentinel:wake").catch((err) => {
						log.warn(`restart sentinel pending update retry failed: ${formatErrorMessage(err)}`);
					});
				}, CONTROL_PLANE_UPDATE_PENDING_RETRY_DELAY_MS).unref?.();
				return {
					status: "skipped",
					reason: "update-restart-pending"
				};
			}
			log.warn(`${summary}: update restart sentinel remained pending after retry window`, {
				sessionKey,
				reason: payload.stats?.reason ?? null
			});
			if (updateRunId) {
				const expiredRun = await finalizeRestartUpdateRun(payload, true);
				if (expiredRun) noticeMessage = renderUpdateRunReport(expiredRun).markdown;
			}
		}
		if (!routedSessionKey) {
			if ((payload.kind === "config-patch" || payload.kind === "config-apply") && (typeof payload.message !== "string" || payload.message.trim().length === 0) && !payload.continuation && !payload.deliveryContext && payload.threadId == null) {
				if (!await clearRestartSentinelIfRevision(sentinelRevision)) log.info(`${summary}: newer restart sentinel preserved while consuming config restart`);
				return { status: "ran" };
			}
			const systemTarget = resolveSystemMainSessionTarget(getRuntimeConfig());
			routedSessionKey = systemTarget.sessionKey;
			wakeAgentId = systemTarget.agentId;
		}
		const continuation = sessionKey ? payload.continuation : void 0;
		const session = loadGatewaySessionEntry(routedSessionKey);
		const { cfg, entry, canonicalKey } = session;
		const target = resolveUpdateRunNoticeTarget({
			cfg,
			sessionKey,
			session,
			explicitDeliveryContext: sessionKey ? payload.deliveryContext : void 0,
			threadId: sessionKey ? payload.threadId : void 0
		});
		const route = target.kind === "route" ? target.route : void 0;
		const deliveryContext = normalizeDeliveryContext(route) ?? (sessionKey ? wakeDeliveryContext : void 0);
		let continuationQueueId;
		let wakeQueueId;
		let noticeQueueId;
		let noticeQueueCreated = false;
		const continuationRoute = continuation ? route : void 0;
		let internalNoticeWritten = false;
		if (updateRun?.verification.noticeDelivered) internalNoticeWritten = true;
		else if (sessionKey && target.kind === "internal") {
			const { agentId, entry: internalEntry, storePath } = target.session;
			const notice = await appendAssistantMessageToSessionTranscript({
				agentId,
				sessionKey: canonicalKey,
				expectedSessionId: internalEntry.sessionId,
				expectedLifecycleRevision: internalEntry.lifecycleRevision ?? null,
				storePath,
				text: noticeMessage,
				idempotencyKey: updateRunId ? `update-run-finished:${updateRunId}` : `restart-sentinel-notice:${canonicalKey}:${sentinelRevision}`
			}).catch((error) => ({
				ok: false,
				reason: formatErrorMessage(error)
			}));
			internalNoticeWritten = notice.ok;
			if (notice.ok && updateRunId) recordUpdateRunVerification(updateRunId, { noticeDelivered: true });
			if (!notice.ok) log.warn(`${summary}: internal restart notice append failed; falling back to wake: ${notice.reason}`, { sessionKey: canonicalKey });
		}
		const routedAgentTurnContinuation = continuation?.kind === "agentTurn" && continuationRoute !== void 0;
		const updateComplete = (internalNoticeWritten || updateRunId && route) && payload.kind === "update" && !continuation;
		if (!routedAgentTurnContinuation && !updateComplete) wakeQueueId = await enqueueSessionDelivery(buildQueuedRestartContinuation({
			sessionKey: canonicalKey,
			agentId: wakeAgentId,
			continuation: {
				kind: "systemEvent",
				text: message
			},
			revision: sentinelRevision,
			deliveryContext,
			idempotencyKey: `restart-sentinel-wake:${canonicalKey}:${sentinelRevision}`
		}));
		if (!sessionKey && payload.continuation) log.warn(`${summary}: continuation skipped: restart sentinel sessionKey unavailable`, {
			sessionKey: canonicalKey,
			continuationKind: payload.continuation.kind
		});
		if (continuation) continuationQueueId = await enqueueSessionDelivery(buildQueuedRestartContinuation({
			sessionKey: canonicalKey,
			continuation,
			revision: sentinelRevision,
			route: continuationRoute,
			expectedSessionId: entry?.sessionId,
			deliveryContext
		}));
		if (route && !updateRun?.verification.noticeDelivered) {
			const queuedNotice = await enqueueRestartSentinelNotice({
				cfg,
				...route,
				message: noticeMessage,
				sessionKey: canonicalKey,
				revision: sentinelRevision,
				...updateRunId ? { deliveryIntentId: `update-run-finished:${updateRunId}` } : {}
			});
			noticeQueueId = queuedNotice.id;
			noticeQueueCreated = queuedNotice.created;
		}
		if (!await clearRestartSentinelIfRevision(sentinelRevision)) log.info(`${summary}: newer restart sentinel preserved while draining durable work`, { sessionKey: canonicalKey });
		if (wakeQueueId) await drainRestartContinuationQueue({
			deps: params.deps,
			entryId: wakeQueueId,
			log
		});
		if (route && noticeQueueId && noticeQueueCreated) {
			if (await deliverRestartSentinelNotice({
				deps: params.deps,
				cfg,
				sessionKey: canonicalKey,
				summary,
				message: noticeMessage,
				...route,
				queueId: noticeQueueId
			}) && updateRunId) recordUpdateRunVerification(updateRunId, { noticeDelivered: true });
		} else if (noticeQueueId && !noticeQueueCreated) log.info(`${summary}: durable restart notice already owned`, { sessionKey: canonicalKey });
		if (continuationQueueId) await drainRestartContinuationQueue({
			deps: params.deps,
			entryId: continuationQueueId,
			log
		});
		return { status: "ran" };
	};
	return {
		source: "restart-sentinel",
		...sessionKey ? { sessionKey } : {},
		run
	};
}
async function scheduleRestartSentinelWakeAttempt(params) {
	const task = await loadRestartSentinelStartupTask(params);
	if (!task) return;
	await runStartupTasks({
		tasks: [task],
		log
	});
}
async function scheduleRestartSentinelWake(params) {
	await scheduleRestartSentinelWakeAttempt({
		...params,
		attempt: 0
	});
}
async function refreshLatestUpdateRestartSentinel() {
	const current = await readRestartSentinel();
	if (current?.payload.kind === "update" && isPendingControlPlaneUpdateRestartSentinel(current.payload)) {
		latestUpdateRestartSentinel = cloneRestartSentinelPayload(current.payload);
		return cloneRestartSentinelPayload(latestUpdateRestartSentinel);
	}
	const sentinel = await finalizeUpdateRestartSentinelRunningVersion() ?? current;
	if (sentinel?.payload.kind === "update") latestUpdateRestartSentinel = cloneRestartSentinelPayload(sentinel.payload);
	return cloneRestartSentinelPayload(latestUpdateRestartSentinel);
}
function getLatestUpdateRestartSentinel() {
	return cloneRestartSentinelPayload(latestUpdateRestartSentinel);
}
function recordLatestUpdateRestartSentinel(payload) {
	latestUpdateRestartSentinel = cloneRestartSentinelPayload(payload);
}
//#endregion
export { refreshLatestUpdateRestartSentinel as a, recoverPendingRestartContinuationDeliveries as i, getLatestUpdateRestartSentinel as n, scheduleRestartSentinelWake as o, recordLatestUpdateRestartSentinel as r, settleQueuedSessionDelivery as s, deliverQueuedSessionDelivery as t };