openclaw/dist/server-maintenance-CahI1H6W.js

Multi-channel AI gateway with extensible messaging integrations

import { g as isFutureDateTimestampMs } from "./number-coercion-CLj0HTDM.js";
import { r as createLazyPromiseLoader } from "./lazy-promise-DGqyc4Y4.js";
import { t as pruneMapToMaxSize } from "./map-size-CNcWiFKu.js";
import { t as formatErrorMessage } from "./errors-Db3Ymjlb.js";
import { r as generateSecureInt } from "./secure-random-Ds4AFLgz.js";
import { w as sweepStaleRunContexts } from "./agent-run-registry-CKYKdfNd.js";
import { O as tryBeginGatewaySuspendAdmission, f as isGatewayWorkAdmissionClosed } from "./gateway-work-admission-R1IpuDim.js";
import { c as pruneOutboundMedia, l as prunePlaybackTranscodeCache, r as cleanOldMedia } from "./store-Ph57PZ9P.js";
import { r as HEALTH_REFRESH_INTERVAL_MS, s as TICK_INTERVAL_MS, t as DEDUPE_MAX } from "./server-constants-BrVEC7RW.js";
import "./agent-run-terminal-outcome-CigeY75d.js";
import { t as createGatewayActiveWorkSnapshot } from "./gateway-active-work-DdwRky6I.js";
import { i as tryResolveSessionCompatibilityOwnerAgentId } from "./session-request-agent-CCRSEGCB.js";
import { h as pruneExpiredDeliveryQueueTombstones } from "./delivery-queue-sqlite-Bpn2g0Bx.js";
import { n as pruneOrphanedDeliveryQueueMedia } from "./delivery-queue-media-spool-D2PN5qRE.js";
import { a as registerSkillUsageTracking } from "./curator-DE46Wqg0.js";
import { c as removeChatAbortControllerEntry, t as abortChatRunById } from "./chat-abort-qLn3eFOg.js";
import { t as chatAbortMarkerTimestampMs } from "./server-chat-state-CwKZZaYd.js";
import { l as pruneExpiredDevicePairSetupCompletions } from "./device-bootstrap-DRFMRCcr.js";
import "./server-utils-BUZaV07G.js";
import { n as hasRegisteredChatRunForSessionKey } from "./session-active-runs-CCNuJW7B.js";
import "./server-shared-C-7Ahu3n.js";
import { i as WORKTREE_GC_INTERVAL_MS, l as managedWorktrees, u as resolveWorktreeCleanupLimits } from "./service-B1SvAvxa.js";
import { r as checkTelemetryUpdate } from "./telemetry-DmzlffL1.js";
import { t as createManagedWorktreeOwnerPolicy } from "./owner-protection-DS4QvFc2.js";
import { r as pruneStaleControlPlaneBuckets } from "./control-plane-rate-limit-B1YxvgrA.js";
import { i as waitForMediaCleanupDrainsToSettle, n as registerMediaCleanupDrain, r as waitForMediaCleanupDrains, t as MEDIA_CLEANUP_STOP_TIMEOUT_MS } from "./server-media-cleanup-lifecycle-dOPPbnLL.js";
import { s as setBroadcastHealthUpdate } from "./health-state-BQ5pY3AF.js";
function createHostThawRecovery(deps) {
	let lastTickAtMs = deps.nowMs();
	let pendingFrozenMs;
	let pendingChannelRestart = false;
	let activeRecovery;
	const runStep = async (label, step) => {
		try {
			await step();
		} catch (error) {
			deps.logger.error(`host thaw ${label} failed: ${String(error)}`);
		}
	};
	const restartChannels = async (mode) => {
		try {
			const outcome = await deps.restartChannelsIfIdle(mode);
			pendingChannelRestart = outcome.status === "retry";
			if (outcome.status === "retry") {
				const detail = outcome.reason === "active-work" ? "gateway still has active work" : outcome.reason === "admission-closed" ? "gateway admission is closed" : "one or more channel accounts remain pending";
				deps.logger.info(`host thaw channel restart deferred: ${detail}`);
			}
		} catch (error) {
			pendingChannelRestart = true;
			deps.logger.error(`host thaw channel restart failed: ${String(error)}`);
		}
	};
	const recover = async (frozenMs) => {
		deps.logger.info(`host timing gap detected: process was frozen ~${Math.round(frozenMs)}ms; restarting channels when idle and refreshing health`);
		const deferForAdmission = () => {
			if (deps.isAdmissionClosed()) {
				pendingFrozenMs = Math.max(pendingFrozenMs ?? 0, frozenMs);
				deps.logger.info("host thaw recovery deferred: gateway suspension began mid-recovery");
				return true;
			}
			return false;
		};
		if (deferForAdmission()) return;
		await runStep("event-loop reset", deps.resetEventLoopHealth);
		if (deferForAdmission()) return;
		await restartChannels("new-thaw");
		for (const [label, step] of [["health refresh", deps.refreshHealth], ["presence refresh", deps.refreshPresence]]) {
			if (deferForAdmission()) return;
			await runStep(label, step);
		}
	};
	return { tick: async () => {
		const nowMs = deps.nowMs();
		const gapMs = nowMs - lastTickAtMs;
		lastTickAtMs = nowMs;
		if (gapMs >= 75e3) pendingFrozenMs = Math.max(pendingFrozenMs ?? 0, gapMs - TICK_INTERVAL_MS);
		if (pendingFrozenMs === void 0 && !pendingChannelRestart || deps.isAdmissionClosed() || activeRecovery) return;
		const frozenMs = pendingFrozenMs;
		pendingFrozenMs = void 0;
		activeRecovery = frozenMs === void 0 ? restartChannels("deferred-retry") : recover(frozenMs);
		try {
			await activeRecovery;
		} finally {
			activeRecovery = void 0;
		}
	} };
}
//#endregion
//#region src/gateway/server-maintenance.ts
const DELIVERY_QUEUE_MEDIA_GC_INTERVAL_MS = 36e5;
const TELEMETRY_MAINTENANCE_INTERVAL_MS = 3e5;
function startGatewayMaintenanceTimers(params) {
	setBroadcastHealthUpdate((snap) => {
		params.broadcast("health", snap, { stateVersion: {
			presence: params.getPresenceVersion(),
			health: params.getHealthVersion()
		} });
		params.nodeSendToAllSubscribed("health", snap);
	});
	const restartChannelsIfIdle = async (mode) => {
		let invalidated = false;
		const admission = tryBeginGatewaySuspendAdmission(() => {
			invalidated = true;
		});
		if (!admission) return {
			status: "retry",
			reason: "admission-closed"
		};
		let snapshot;
		try {
			snapshot = createGatewayActiveWorkSnapshot(params.activeWorkInspectors, { ignoreTerminalSessions: true });
		} catch (error) {
			admission.rollback();
			throw error;
		}
		if (!snapshot.idle) {
			admission.rollback();
			return {
				status: "retry",
				reason: "active-work"
			};
		}
		if (!admission.commit()) return {
			status: "retry",
			reason: "admission-closed"
		};
		try {
			return await params.restartRunningChannels(mode, () => !invalidated) ? { status: "completed" } : {
				status: "retry",
				reason: "channel-restart-incomplete"
			};
		} finally {
			admission.release();
		}
	};
	const hostThawRecovery = createHostThawRecovery({
		nowMs: Date.now,
		restartChannelsIfIdle,
		refreshHealth: async () => {
			await params.refreshGatewayHealthSnapshot({ probe: true });
		},
		refreshPresence: params.refreshPresence,
		resetEventLoopHealth: params.resetEventLoopHealth,
		isAdmissionClosed: isGatewayWorkAdmissionClosed,
		logger: params.logHealth
	});
	let nextTelemetryCheckAtMs = Date.now() + generateSecureInt(TELEMETRY_MAINTENANCE_INTERVAL_MS);
	const tickInterval = setInterval(() => {
		hostThawRecovery.tick();
		const now = Date.now();
		if (!params.isNixMode && now >= nextTelemetryCheckAtMs) {
			nextTelemetryCheckAtMs = now + TELEMETRY_MAINTENANCE_INTERVAL_MS + generateSecureInt(TELEMETRY_MAINTENANCE_INTERVAL_MS);
			checkTelemetryUpdate(params.getRuntimeConfig(), { surface: "gateway" }).catch(() => {});
		}
		const payload = { ts: now };
		params.broadcast("tick", payload);
		params.nodeSendToAllSubscribed("tick", payload);
	}, TICK_INTERVAL_MS);
	const healthInterval = setInterval(() => {
		params.refreshGatewayHealthSnapshot({ probe: false }).catch((err) => params.logHealth.error(`refresh failed: ${formatErrorMessage(err)}`));
	}, HEALTH_REFRESH_INTERVAL_MS);
	params.refreshGatewayHealthSnapshot({ probe: false }).catch((err) => params.logHealth.error(`initial refresh failed: ${formatErrorMessage(err)}`));
	const runWorktreeGc = params.runWorktreeGc ?? (() => {
		const cfg = params.getRuntimeConfig();
		return managedWorktrees.gc({
			...createManagedWorktreeOwnerPolicy(cfg),
			limits: resolveWorktreeCleanupLimits()
		});
	});
	const performWorktreeGc = () => runWorktreeGc().catch((err) => {
		params.logHealth.error(`managed worktree cleanup failed: ${formatErrorMessage(err)}`);
	});
	const worktreeCleanup = setInterval(() => void performWorktreeGc(), WORKTREE_GC_INTERVAL_MS);
	performWorktreeGc();
	const runDeliveryQueueMediaGc = params.runDeliveryQueueMediaGc ?? (async () => {
		try {
			pruneExpiredDeliveryQueueTombstones();
		} finally {
			await pruneOrphanedDeliveryQueueMedia();
		}
	});
	let deliveryQueueMediaGcStartedAtMs = 0;
	const deliveryQueueMediaGcLoader = createLazyPromiseLoader(async () => {
		try {
			await runDeliveryQueueMediaGc();
		} catch (error) {
			params.logHealth.error(`delivery queue maintenance failed: ${formatErrorMessage(error)}`);
		} finally {
			deliveryQueueMediaGcLoader.clear();
		}
	});
	const performDeliveryQueueMediaGc = () => {
		if (!deliveryQueueMediaGcLoader.peek()) deliveryQueueMediaGcStartedAtMs = Date.now();
		return deliveryQueueMediaGcLoader.load();
	};
	performDeliveryQueueMediaGc();
	let devicePairSetupCompletionGcInFlight = null;
	const performDevicePairSetupCompletionGc = (nowMs) => {
		if (devicePairSetupCompletionGcInFlight) return devicePairSetupCompletionGcInFlight;
		devicePairSetupCompletionGcInFlight = pruneExpiredDevicePairSetupCompletions({ nowMs }).then(() => void 0).catch((error) => {
			params.logHealth.error(`device pair setup cleanup failed: ${formatErrorMessage(error)}`);
		}).finally(() => {
			devicePairSetupCompletionGcInFlight = null;
		});
		return devicePairSetupCompletionGcInFlight;
	};
	performDevicePairSetupCompletionGc(Date.now());
	const skillUsageCleanup = registerSkillUsageTracking();
	const dedupeCleanup = setInterval(() => {
		const AGENT_RUN_SEQ_MAX = 1e4;
		const now = Date.now();
		performDevicePairSetupCompletionGc(now);
		if (now - deliveryQueueMediaGcStartedAtMs >= DELIVERY_QUEUE_MEDIA_GC_INTERVAL_MS) performDeliveryQueueMediaGc();
		const resolveDedupeRunId = (key, entry) => {
			if (!key.startsWith("agent:") && !key.startsWith("chat:")) return;
			const keyRunId = key.slice(key.indexOf(":") + 1);
			if (keyRunId) {
				if (params.chatAbortControllers.has(keyRunId) || params.chatQueuedTurns.has(keyRunId)) return keyRunId;
			}
			const payload = entry.payload;
			return payload && typeof payload === "object" && !Array.isArray(payload) ? typeof payload.runId === "string" ? payload.runId.trim() || void 0 : void 0 : void 0;
		};
		const isPendingAcceptedRunDedupeKey = (key, dedupeEntry) => {
			if (!key.startsWith("agent:") && !key.startsWith("pending-chat:")) return false;
			const payload = dedupeEntry.payload;
			if (!payload || typeof payload !== "object" || Array.isArray(payload)) return false;
			if (payload.status !== "accepted") return false;
			const expiresAtMs = payload.expiresAtMs;
			return isFutureDateTimestampMs(expiresAtMs, { nowMs: now });
		};
		const isActiveRunDedupeKey = (key, dedupeEntry) => {
			const isAgentKey = key.startsWith("agent:");
			const isChatKey = key.startsWith("chat:");
			if (!isAgentKey && !isChatKey) return false;
			const runId = resolveDedupeRunId(key, dedupeEntry);
			const entry = runId ? params.chatAbortControllers.get(runId) : void 0;
			if (entry) return isAgentKey ? entry.kind === "agent" : entry.kind !== "agent";
			return Boolean(isChatKey && runId && params.chatQueuedTurns.has(runId));
		};
		for (const [k, v] of params.dedupe) {
			if (isActiveRunDedupeKey(k, v) || isPendingAcceptedRunDedupeKey(k, v)) continue;
			if (now - v.ts > 3e5) params.dedupe.delete(k);
		}
		if (params.dedupe.size > 1e3) {
			const excess = params.dedupe.size - DEDUPE_MAX;
			const oldestKeys = [...params.dedupe.entries()].filter(([key, entry]) => !isActiveRunDedupeKey(key, entry) && !isPendingAcceptedRunDedupeKey(key, entry)).toSorted(([, left], [, right]) => left.ts - right.ts).slice(0, excess).map(([key]) => key);
			for (const key of oldestKeys) params.dedupe.delete(key);
		}
		pruneMapToMaxSize(params.agentRunSeq, AGENT_RUN_SEQ_MAX);
		for (const [runId, entry] of params.chatAbortControllers) {
			const terminalClearOverdue = typeof entry.projectSessionTerminalObservedAt === "number" && now - entry.projectSessionTerminalObservedAt > 15e3;
			if (entry.projectSessionTerminalPending === true && !terminalClearOverdue) continue;
			if (isFutureDateTimestampMs(entry.expiresAtMs, { nowMs: now })) continue;
			if (entry.projectSessionTerminalPersistence) {
				const lifecycleGeneration = entry.lifecycleGeneration?.trim();
				const sessionKey = entry.sessionKey.trim();
				const sessionId = entry.sessionId.trim();
				if (entry.controlUiVisible !== false && lifecycleGeneration && sessionKey && sessionId) params.restartRecoveryCandidates.set(runId, {
					runId,
					lifecycleGeneration,
					sessionKey,
					sessionId,
					observedAt: entry.projectSessionTerminalObservedAt
				});
				removeChatAbortControllerEntry(params.chatAbortControllers, runId, entry);
				continue;
			}
			if (entry.projectSessionActive === false) {
				removeChatAbortControllerEntry(params.chatAbortControllers, runId, entry);
				continue;
			}
			if (!abortChatRunById(params, {
				runId,
				sessionKey: entry.sessionKey,
				stopReason: "timeout"
			}).aborted) removeChatAbortControllerEntry(params.chatAbortControllers, runId, entry);
		}
		const ABORTED_RUN_TTL_MS = 36e5;
		pruneStaleControlPlaneBuckets(now);
		for (const [runId, record] of params.chatRunState.runs) {
			if (record.abortMarker !== void 0) {
				if (now - chatAbortMarkerTimestampMs(record.abortMarker) > ABORTED_RUN_TTL_MS) {
					params.chatRunState.deleteAbortMarker(runId);
					params.chatRunState.clearRun(runId);
				}
				continue;
			}
			if (params.chatAbortControllers.has(runId)) continue;
			if ([
				record.deltaSentAt,
				record.bufferUpdatedAt,
				record.agentText?.assistant?.lastSentAt,
				record.agentText?.thinking?.lastSentAt
			].some((timestamp) => timestamp !== void 0 && now - timestamp > ABORTED_RUN_TTL_MS)) params.chatRunState.clearRun(runId);
		}
		sweepStaleRunContexts();
	}, 6e4);
	const playbackTranscodeCacheCleanupLoader = createLazyPromiseLoader(async () => {
		try {
			await prunePlaybackTranscodeCache();
		} catch (err) {
			params.logHealth.error(`playback transcode cache cleanup failed: ${formatErrorMessage(err)}`);
		} finally {
			playbackTranscodeCacheCleanupLoader.clear();
		}
	});
	const runManagedOutgoingMediaGc = params.runManagedOutgoingMediaGc ?? (async () => {
		const { cleanupManagedOutgoingMediaRecords } = await import("./managed-image-attachments-D18sTbBJ.js");
		return await cleanupManagedOutgoingMediaRecords({ hasActiveSessionRun: (sessionKey, agentId) => {
			const cfg = params.getRuntimeConfig();
			return hasRegisteredChatRunForSessionKey({
				context: { chatAbortControllers: params.chatAbortControllers },
				sessionKey,
				agentId,
				defaultAgentId: tryResolveSessionCompatibilityOwnerAgentId(cfg, sessionKey)
			});
		} });
	});
	const managedOutgoingCleanupLoader = createLazyPromiseLoader(async () => {
		try {
			await runManagedOutgoingMediaGc();
		} catch (err) {
			params.logHealth.error(`managed outgoing media cleanup failed: ${formatErrorMessage(err)}`);
		} finally {
			managedOutgoingCleanupLoader.clear();
		}
	});
	let mediaCleanupInFlight = null;
	const runMediaCleanup = () => {
		if (mediaCleanupInFlight) return mediaCleanupInFlight;
		const ttlHours = params.getRuntimeConfig().attachments?.ttlHours;
		mediaCleanupInFlight = (ttlHours !== void 0 ? cleanOldMedia(ttlHours * 60 * 6e4, {
			recursive: true,
			pruneEmptyDirs: true
		}) : pruneOutboundMedia()).catch((err) => {
			params.logHealth.error(`media cleanup failed: ${formatErrorMessage(err)}`);
		}).finally(() => {
			mediaCleanupInFlight = null;
		});
		return mediaCleanupInFlight;
	};
	let mediaCleanupInterval;
	let mediaCleanupStopped = false;
	const runMediaMaintenance = () => {
		if (mediaCleanupStopped) return;
		playbackTranscodeCacheCleanupLoader.load();
		managedOutgoingCleanupLoader.load();
		runMediaCleanup();
	};
	let mediaCleanupStartPromise;
	const startMediaCleanup = () => {
		if (mediaCleanupStopped || mediaCleanupInterval || mediaCleanupStartPromise) return;
		mediaCleanupStartPromise = waitForMediaCleanupDrainsToSettle().then(() => {
			mediaCleanupStartPromise = void 0;
			if (mediaCleanupStopped || mediaCleanupInterval) return;
			mediaCleanupInterval = setInterval(runMediaMaintenance, 36e5);
			runMediaMaintenance();
		});
	};
	let stopMediaCleanupPromise;
	const stopMediaCleanup = () => {
		stopMediaCleanupPromise ??= (async () => {
			mediaCleanupStopped = true;
			if (mediaCleanupInterval) {
				clearInterval(mediaCleanupInterval);
				mediaCleanupInterval = void 0;
			}
			const pending = [
				playbackTranscodeCacheCleanupLoader.peek(),
				managedOutgoingCleanupLoader.peek(),
				mediaCleanupInFlight
			].filter((promise) => promise !== void 0 && promise !== null);
			if (pending.length > 0) registerMediaCleanupDrain(Promise.allSettled(pending).then(() => void 0));
			return await waitForMediaCleanupDrains({
				timeoutMs: MEDIA_CLEANUP_STOP_TIMEOUT_MS,
				onTimeout: () => {
					params.logHealth.error(`media cleanup drain exceeded ${MEDIA_CLEANUP_STOP_TIMEOUT_MS}ms; retaining shared state until cleanup settles`);
				}
			});
		})();
		return stopMediaCleanupPromise;
	};
	return {
		tickInterval,
		healthInterval,
		dedupeCleanup,
		startMediaCleanup,
		stopMediaCleanup,
		worktreeCleanup,
		skillUsageCleanup
	};
}
//#endregion
export { startGatewayMaintenanceTimers };