UNPKG

openclaw

Version:

Multi-channel AI gateway with extensible messaging integrations

559 lines (558 loc) 22.6 kB
import { c as normalizeOptionalString } from "./string-coerce-mnp54Vah.js"; import { y as resolveStateDir } from "./paths-mvMm5bYV.js"; import { t as createSubsystemLogger } from "./subsystem-BzXSmsuh.js"; import { a as isSubagentSessionKey, i as isCronSessionKey, n as isAcpSessionKey } from "./session-key-utils-Bx3apsJ3.js"; import { o as callGateway } from "./call-B5-GYOlf.js"; import { n as deliveryContextFromSession, o as normalizeDeliveryContext } from "./delivery-context.shared-CpAdEETO.js"; import { i as resolveSessionFilePath, s as resolveSessionTranscriptPathInDir } from "./paths-NEwU8m3X.js"; import { t as loadSessionStore } from "./store-load-C4uq6Lrq.js"; import { d as updateSessionStore } from "./store-Qsgtu-0y.js"; import { t as isDeliverableMessageChannel } from "./message-channel-normalize-BLLf0ubu.js"; import "./message-channel-BiOeMu0l.js"; import { t as resolveAgentSessionDirs } from "./session-dirs-COV6A9cq.js"; import { r as resolveAllAgentSessionStoreTargetsSync } from "./targets-BrMDn2yj.js"; import "./sessions-D6zZvoxX.js"; import { d as listActiveEmbeddedRunSessionIds, f as listActiveEmbeddedRunSessionKeys } from "./run-state-BYGFyjne.js"; import { d as readSessionMessagesAsync } from "./session-utils.fs-D_8-X4jl.js"; import { t as sanitizePendingFinalDeliveryText } from "./pending-final-delivery-BsRuDnJV.js"; import { g as resolveGatewaySessionStoreTarget } from "./session-utils-CHNCuY8a.js"; import { n as resolveSendPolicy } from "./send-policy-CfC5ATxp.js"; import fs from "node:fs"; import path from "node:path"; import crypto from "node:crypto"; //#region src/agents/main-session-restart-recovery.ts /** * Post-restart recovery for main sessions interrupted while holding a transcript lock. */ const log = createSubsystemLogger("main-session-restart-recovery"); const DEFAULT_RECOVERY_DELAY_MS = 5e3; const MAX_RECOVERY_RETRIES = 3; const RETRY_BACKOFF_MULTIPLIER = 2; const UNRESUMABLE_SESSION_NOTICE = "I was interrupted by a gateway restart and couldn't safely resume the previous turn. Please send that last request again and I'll pick it up cleanly."; function shouldSkipMainRecovery(entry, sessionKey) { if (typeof entry.spawnDepth === "number" && entry.spawnDepth > 0) return true; if (entry.subagentRole != null) return true; return isSubagentSessionKey(sessionKey) || isCronSessionKey(sessionKey) || isAcpSessionKey(sessionKey); } function normalizeStringSet(values) { const normalized = /* @__PURE__ */ new Set(); for (const value of values ?? []) { const trimmed = value.trim(); if (trimmed) normalized.add(trimmed); } return normalized; } function normalizeFiniteTimestamp(value) { return typeof value === "number" && Number.isFinite(value) ? value : void 0; } function hasCurrentProcessOwner(params) { if (params.activeSessionIds.has(params.entry.sessionId)) return true; return params.activeSessionIds.size === 0 && params.activeSessionKeys.has(params.sessionKey); } function normalizeTranscriptLockPath(lockPath) { const trimmed = lockPath.trim(); if (!path.basename(trimmed).endsWith(".jsonl.lock")) return; const resolved = path.resolve(trimmed); try { return path.join(fs.realpathSync(path.dirname(resolved)), path.basename(resolved)); } catch { return resolved; } } function resolveEntryTranscriptLockPaths(params) { const paths = /* @__PURE__ */ new Set(); const push = (resolvePath) => { try { paths.add(path.resolve(`${resolvePath()}.lock`)); } catch {} }; push(() => resolveSessionFilePath(params.entry.sessionId, params.entry, { sessionsDir: params.sessionsDir })); push(() => resolveSessionTranscriptPathInDir(params.entry.sessionId, params.sessionsDir)); return [...paths]; } async function markRestartAbortedMainSessions(params) { const sessionKeys = normalizeStringSet(params.sessionKeys); const sessionIds = normalizeStringSet(params.sessionIds); const preferSessionIdMatch = sessionIds.size > 0; const result = { marked: 0, skipped: 0 }; if (sessionKeys.size === 0 && sessionIds.size === 0) return result; const storePaths = /* @__PURE__ */ new Set(); const env = params.stateDir === void 0 ? process.env : { ...process.env, OPENCLAW_STATE_DIR: params.stateDir }; const stateDir = resolveStateDir(env); const configs = [params.cfg, ...params.additionalCfgs ?? []].filter((cfg) => Boolean(cfg)); for (const cfg of configs) { try { for (const target of resolveAllAgentSessionStoreTargetsSync(cfg, { env })) storePaths.add(path.resolve(target.storePath)); } catch (err) { log.warn(`failed to resolve configured session stores for restart marker: ${String(err)}`); } for (const sessionKey of sessionKeys) try { const target = resolveGatewaySessionStoreTarget({ cfg, key: sessionKey, scanLegacyKeys: true }); storePaths.add(path.resolve(target.storePath)); for (const storeKey of target.storeKeys) { const trimmed = storeKey.trim(); if (trimmed) sessionKeys.add(trimmed); } } catch (err) { log.warn(`failed to resolve session store for restart marker ${sessionKey}: ${String(err)}`); } } for (const sessionsDir of await resolveAgentSessionDirs(stateDir)) storePaths.add(path.join(sessionsDir, "sessions.json")); for (const storePath of storePaths) await updateSessionStore(storePath, (store) => { for (const [sessionKey, entry] of Object.entries(store)) { if (!entry || entry.status !== "running") continue; if (!(typeof entry.sessionId === "string" && sessionIds.has(entry.sessionId) ? true : !preferSessionIdMatch && sessionKeys.has(sessionKey))) continue; if (shouldSkipMainRecovery(entry, sessionKey)) { result.skipped++; continue; } entry.abortedLastRun = true; entry.updatedAt = Date.now(); store[sessionKey] = entry; result.marked++; } }, { skipMaintenance: true }); if (result.marked > 0) log.warn(`marked ${result.marked} interrupted main session(s) for restart recovery${params.reason ? ` (${params.reason})` : ""}`); return result; } async function markStartupOrphanedMainSessionsForRecovery(params) { const result = { marked: 0, skipped: 0 }; const providedActiveSessionIds = params.activeSessionIds === void 0 ? void 0 : normalizeStringSet(params.activeSessionIds); const providedActiveSessionKeys = params.activeSessionKeys === void 0 ? void 0 : normalizeStringSet(params.activeSessionKeys); const updatedBeforeMs = normalizeFiniteTimestamp(params.updatedBeforeMs); const resolveActiveSessionIds = () => providedActiveSessionIds ?? normalizeStringSet(listActiveEmbeddedRunSessionIds()); const resolveActiveSessionKeys = () => providedActiveSessionKeys ?? normalizeStringSet(listActiveEmbeddedRunSessionKeys()); for (const storePath of await resolveRestartRecoveryStorePaths(params)) await updateSessionStore(storePath, (store) => { for (const [sessionKey, entry] of Object.entries(store)) { if (!entry || entry.status !== "running" || entry.abortedLastRun === true) continue; if (shouldSkipMainRecovery(entry, sessionKey)) { result.skipped++; continue; } const updatedAt = normalizeFiniteTimestamp(entry.updatedAt); if (updatedBeforeMs !== void 0 && updatedAt !== void 0 && updatedAt > updatedBeforeMs) continue; if (hasCurrentProcessOwner({ activeSessionIds: resolveActiveSessionIds(), activeSessionKeys: resolveActiveSessionKeys(), entry, sessionKey })) continue; entry.abortedLastRun = true; entry.updatedAt = Date.now(); store[sessionKey] = entry; result.marked++; } }, { skipMaintenance: true }); if (result.marked > 0) log.warn(`marked ${result.marked} startup-orphaned main session(s) for restart recovery`); return result; } function getMessageRole(message) { if (!message || typeof message !== "object") return; const role = message.role; return typeof role === "string" ? role : void 0; } function isMeaningfulTailMessage(message) { const role = getMessageRole(message); if (!role || role === "system") return false; return true; } function isResumableTailMessage(message) { const role = getMessageRole(message); return role === "user" || role === "tool" || role === "toolResult"; } function isApprovalPendingToolResult(message) { if (!message || typeof message !== "object" || getMessageRole(message) !== "toolResult") return false; const details = message.details; if (!details || typeof details !== "object") return false; return details.status === "approval-pending"; } function resolveMainSessionResumeBlockReason(messages) { const lastMeaningful = messages.toReversed().find(isMeaningfulTailMessage); if (!lastMeaningful || !isResumableTailMessage(lastMeaningful)) return "transcript tail is not resumable"; if (isApprovalPendingToolResult(lastMeaningful)) return "transcript tail is a stale approval-pending tool result"; return null; } function buildResumeMessage(pendingFinalDeliveryText) { const base = "[System] Your previous turn was interrupted by a gateway restart while OpenClaw was waiting on tool/model work. Continue from the existing transcript and finish the interrupted response."; const sanitizedPendingText = typeof pendingFinalDeliveryText === "string" ? sanitizePendingFinalDeliveryText(pendingFinalDeliveryText) : ""; if (sanitizedPendingText) return `${base}\n\nNote: The interrupted final reply was captured: "${sanitizedPendingText}"`; return base; } async function markSessionFailed(params) { await updateSessionStore(params.storePath, (store) => { const entry = store[params.sessionKey]; if (!entry || entry.status !== "running") return; entry.status = "failed"; entry.abortedLastRun = true; entry.endedAt = Date.now(); entry.updatedAt = entry.endedAt; entry.pendingFinalDelivery = void 0; entry.pendingFinalDeliveryText = void 0; entry.pendingFinalDeliveryCreatedAt = void 0; entry.pendingFinalDeliveryLastAttemptAt = void 0; entry.pendingFinalDeliveryAttemptCount = void 0; entry.pendingFinalDeliveryLastError = void 0; entry.pendingFinalDeliveryContext = void 0; entry.restartRecoveryDeliveryContext = void 0; entry.restartRecoveryDeliveryRunId = void 0; store[params.sessionKey] = entry; }, { skipMaintenance: true }); log.warn(`marked interrupted main session failed: ${params.sessionKey} (${params.reason})`); } async function sendUnresumableSessionNotice(params) { const deliveryContext = resolveRestartRecoveryDeliveryContext({ cfg: params.cfg, entry: params.entry, includeSessionDeliveryFallback: true, sessionKey: params.sessionKey }); if (!deliveryContext) return false; const messageParams = { to: deliveryContext.to, message: UNRESUMABLE_SESSION_NOTICE, bestEffort: true }; if (deliveryContext?.threadId != null) messageParams.threadId = deliveryContext.threadId; const actionParams = { channel: deliveryContext.channel, action: "send", sessionKey: params.sessionKey, sessionId: params.entry.sessionId, idempotencyKey: `main-session-restart-recovery:${params.entry.sessionId}:failed-notice`, params: messageParams }; const accountId = normalizeOptionalString(deliveryContext?.accountId); if (accountId) actionParams.accountId = accountId; try { await callGateway({ method: "message.action", params: actionParams, timeoutMs: 1e4 }); log.info(`sent interrupted main session recovery notice: ${params.sessionKey} (${params.reason})`); return true; } catch (err) { log.warn(`failed to send interrupted main session recovery notice ${params.sessionKey}: ${String(err)}`); return false; } } function resolveRestartRecoveryDeliveryContext(params) { const deliveryContext = normalizeDeliveryContext(params.entry.pendingFinalDeliveryContext) ?? normalizeDeliveryContext(params.entry.restartRecoveryDeliveryContext) ?? (params.includeSessionDeliveryFallback ? deliveryContextFromSession(params.entry) : void 0); const channel = normalizeOptionalString(deliveryContext?.channel); const to = normalizeOptionalString(deliveryContext?.to); if (!channel || !to || !isDeliverableMessageChannel(channel)) return; if (params.cfg && resolveSendPolicy({ cfg: params.cfg, entry: params.entry, sessionKey: params.sessionKey, channel, chatType: params.entry.chatType }) === "deny") return; return { ...deliveryContext, channel, to }; } async function resumeMainSession(params) { const sanitizedPendingText = typeof params.pendingFinalDeliveryText === "string" ? sanitizePendingFinalDeliveryText(params.pendingFinalDeliveryText) : ""; const deliveryContext = resolveRestartRecoveryDeliveryContext({ cfg: params.cfg, entry: params.entry, sessionKey: params.sessionKey }); try { const agentParams = { message: buildResumeMessage(sanitizedPendingText), sessionKey: params.sessionKey, idempotencyKey: crypto.randomUUID(), deliver: Boolean(deliveryContext), lane: "main" }; if (deliveryContext) { agentParams.channel = deliveryContext.channel; agentParams.to = deliveryContext.to; agentParams.bestEffortDeliver = true; if (deliveryContext.accountId) agentParams.accountId = deliveryContext.accountId; if (deliveryContext.threadId != null) agentParams.threadId = String(deliveryContext.threadId); } await callGateway({ method: "agent", params: agentParams, timeoutMs: 1e4 }); await updateSessionStore(params.storePath, (store) => { const entry = store[params.sessionKey]; if (!entry) return; const now = Date.now(); entry.abortedLastRun = false; entry.updatedAt = now; if (entry.pendingFinalDelivery || entry.pendingFinalDeliveryText) if (sanitizedPendingText) { entry.pendingFinalDeliveryLastAttemptAt = now; entry.pendingFinalDeliveryAttemptCount = (entry.pendingFinalDeliveryAttemptCount ?? 0) + 1; entry.pendingFinalDeliveryLastError = null; entry.pendingFinalDeliveryText = sanitizedPendingText; } else { entry.pendingFinalDelivery = void 0; entry.pendingFinalDeliveryText = void 0; entry.pendingFinalDeliveryCreatedAt = void 0; entry.pendingFinalDeliveryLastAttemptAt = void 0; entry.pendingFinalDeliveryAttemptCount = void 0; entry.pendingFinalDeliveryLastError = void 0; entry.pendingFinalDeliveryContext = void 0; } store[params.sessionKey] = entry; }, { skipMaintenance: true }); log.info(`resumed interrupted main session: ${params.sessionKey}${sanitizedPendingText ? " (with pending payload)" : ""}`); return true; } catch (err) { log.warn(`failed to resume interrupted main session ${params.sessionKey}: ${String(err)}`); return false; } } async function markRestartAbortedMainSessionsFromLocks(params) { const result = { marked: 0, skipped: 0 }; const sessionsDir = path.resolve(params.sessionsDir); const interruptedLockPaths = new Set(params.cleanedLocks.map((lock) => normalizeTranscriptLockPath(lock.lockPath)).filter((lockPath) => Boolean(lockPath))); if (interruptedLockPaths.size === 0) return result; await updateSessionStore(path.join(sessionsDir, "sessions.json"), (store) => { for (const [sessionKey, entry] of Object.entries(store)) { if (!entry || entry.status !== "running") continue; if (shouldSkipMainRecovery(entry, sessionKey)) { result.skipped++; continue; } if (!resolveEntryTranscriptLockPaths({ entry, sessionsDir }).some((lockPath) => interruptedLockPaths.has(lockPath))) continue; entry.abortedLastRun = true; store[sessionKey] = entry; result.marked++; } }, { skipMaintenance: true }); if (result.marked > 0) log.warn(`marked ${result.marked} interrupted main session(s) from stale transcript locks`); return result; } function isRoutableRecoveryStore(params) { if (!params.cfg) return true; if (!params.cfg.session?.store) return true; try { const target = resolveGatewaySessionStoreTarget({ cfg: params.cfg, key: params.sessionKey, scanLegacyKeys: true }); return path.resolve(target.storePath) === path.resolve(params.storePath); } catch (err) { log.warn(`failed to resolve recovery store for ${params.sessionKey}: ${String(err)}`); return false; } } async function recoverStore(params) { const result = { recovered: 0, failed: 0, skipped: 0 }; const providedActiveSessionIds = params.activeSessionIds === void 0 ? void 0 : normalizeStringSet(params.activeSessionIds); const providedActiveSessionKeys = params.activeSessionKeys === void 0 ? void 0 : normalizeStringSet(params.activeSessionKeys); const resolveActiveSessionIds = () => providedActiveSessionIds ?? normalizeStringSet(listActiveEmbeddedRunSessionIds()); const resolveActiveSessionKeys = () => providedActiveSessionKeys ?? normalizeStringSet(listActiveEmbeddedRunSessionKeys()); let store; try { store = loadSessionStore(params.storePath); } catch (err) { log.warn(`failed to load session store ${params.storePath}: ${String(err)}`); result.failed++; return result; } for (const [sessionKey, entry] of Object.entries(store).toSorted(([a], [b]) => a.localeCompare(b))) { if (!entry || entry.status !== "running" || entry.abortedLastRun !== true) continue; if (shouldSkipMainRecovery(entry, sessionKey)) { result.skipped++; continue; } if (!isRoutableRecoveryStore({ cfg: params.cfg, sessionKey, storePath: params.storePath })) { result.skipped++; continue; } if (hasCurrentProcessOwner({ activeSessionIds: resolveActiveSessionIds(), activeSessionKeys: resolveActiveSessionKeys(), entry, sessionKey })) { result.skipped++; continue; } const resumeDedupeKey = sessionKey; if (params.resumedSessionKeys.has(resumeDedupeKey)) { result.skipped++; continue; } if (entry.pendingFinalDelivery === true && entry.pendingFinalDeliveryText) { if (await resumeMainSession({ cfg: params.cfg, entry, storePath: params.storePath, sessionKey, pendingFinalDeliveryText: entry.pendingFinalDeliveryText })) { params.resumedSessionKeys.add(resumeDedupeKey); result.recovered++; } else result.failed++; continue; } let messages; try { messages = await readSessionMessagesAsync(entry.sessionId, params.storePath, entry.sessionFile, { mode: "recent", maxMessages: 20, maxBytes: 256 * 1024 }); } catch (err) { log.warn(`failed to read transcript for ${sessionKey}: ${String(err)}`); result.failed++; continue; } const resumeBlockReason = resolveMainSessionResumeBlockReason(messages); if (resumeBlockReason) { await sendUnresumableSessionNotice({ cfg: params.cfg, entry, sessionKey, reason: resumeBlockReason }); await markSessionFailed({ storePath: params.storePath, sessionKey, reason: resumeBlockReason }); result.failed++; continue; } if (await resumeMainSession({ cfg: params.cfg, entry, storePath: params.storePath, sessionKey, pendingFinalDeliveryText: entry.pendingFinalDeliveryText })) { params.resumedSessionKeys.add(resumeDedupeKey); result.recovered++; } else result.failed++; } return result; } async function resolveRestartRecoveryStorePaths(params) { const storePaths = /* @__PURE__ */ new Set(); const stateDir = params.stateDir ?? resolveStateDir(process.env); for (const sessionsDir of await resolveAgentSessionDirs(stateDir)) storePaths.add(path.join(sessionsDir, "sessions.json")); if (params.cfg) { const env = { ...process.env, OPENCLAW_STATE_DIR: stateDir }; for (const target of resolveAllAgentSessionStoreTargetsSync(params.cfg, { env })) storePaths.add(path.resolve(target.storePath)); } return [...storePaths].toSorted((a, b) => a.localeCompare(b)); } async function recoverRestartAbortedMainSessions(params = {}) { const result = { recovered: 0, failed: 0, skipped: 0 }; const resumedSessionKeys = params.resumedSessionKeys ?? /* @__PURE__ */ new Set(); for (const storePath of await resolveRestartRecoveryStorePaths(params)) { const storeResult = await recoverStore({ cfg: params.cfg, storePath, resumedSessionKeys, activeSessionIds: params.activeSessionIds, activeSessionKeys: params.activeSessionKeys }); result.recovered += storeResult.recovered; result.failed += storeResult.failed; result.skipped += storeResult.skipped; } if (result.recovered > 0 || result.failed > 0) log.info(`main-session restart recovery complete: recovered=${result.recovered} failed=${result.failed} skipped=${result.skipped}`); return result; } async function recoverStartupOrphanedMainSessions(params = {}) { const startupRecoveryCutoffMs = params.updatedBeforeMs ?? Date.now(); const marked = await markStartupOrphanedMainSessionsForRecovery({ cfg: params.cfg, stateDir: params.stateDir, activeSessionIds: params.activeSessionIds, activeSessionKeys: params.activeSessionKeys, updatedBeforeMs: startupRecoveryCutoffMs }); const recovered = await recoverRestartAbortedMainSessions({ cfg: params.cfg, stateDir: params.stateDir, resumedSessionKeys: params.resumedSessionKeys, activeSessionIds: params.activeSessionIds, activeSessionKeys: params.activeSessionKeys }); return { marked: marked.marked, recovered: recovered.recovered, failed: recovered.failed, skipped: marked.skipped + recovered.skipped }; } function scheduleRestartAbortedMainSessionRecovery(params = {}) { const initialDelay = params.delayMs ?? DEFAULT_RECOVERY_DELAY_MS; const maxRetries = params.maxRetries ?? MAX_RECOVERY_RETRIES; const resumedSessionKeys = /* @__PURE__ */ new Set(); const startupRecoveryCutoffMs = Date.now(); const runRecoveryAttempt = (attempt, delay) => { recoverStartupOrphanedMainSessions({ cfg: params.cfg, stateDir: params.stateDir, resumedSessionKeys, updatedBeforeMs: startupRecoveryCutoffMs }).then((result) => { if (result.failed > 0 && attempt < maxRetries) scheduleAttempt(attempt + 1, delay * RETRY_BACKOFF_MULTIPLIER); }).catch((err) => { if (attempt < maxRetries) { log.warn(`main-session restart recovery failed: ${String(err)}`); scheduleAttempt(attempt + 1, delay * RETRY_BACKOFF_MULTIPLIER); } else log.warn(`main-session restart recovery gave up: ${String(err)}`); }); }; const scheduleAttempt = (attempt, delay) => { if (delay <= 0) { runRecoveryAttempt(attempt, delay); return; } setTimeout(() => { runRecoveryAttempt(attempt, delay); }, delay).unref?.(); }; scheduleAttempt(1, initialDelay); } //#endregion export { recoverStartupOrphanedMainSessions as a, recoverRestartAbortedMainSessions as i, markRestartAbortedMainSessionsFromLocks as n, scheduleRestartAbortedMainSessionRecovery as o, markStartupOrphanedMainSessionsForRecovery as r, markRestartAbortedMainSessions as t };