openclaw
Version:
Multi-channel AI gateway with extensible messaging integrations
559 lines (558 loc) • 22.6 kB
JavaScript
import { c as normalizeOptionalString } from "./string-coerce-mnp54Vah.js";
import { y as resolveStateDir } from "./paths-mvMm5bYV.js";
import { t as createSubsystemLogger } from "./subsystem-BzXSmsuh.js";
import { a as isSubagentSessionKey, i as isCronSessionKey, n as isAcpSessionKey } from "./session-key-utils-Bx3apsJ3.js";
import { o as callGateway } from "./call-B5-GYOlf.js";
import { n as deliveryContextFromSession, o as normalizeDeliveryContext } from "./delivery-context.shared-CpAdEETO.js";
import { i as resolveSessionFilePath, s as resolveSessionTranscriptPathInDir } from "./paths-NEwU8m3X.js";
import { t as loadSessionStore } from "./store-load-C4uq6Lrq.js";
import { d as updateSessionStore } from "./store-Qsgtu-0y.js";
import { t as isDeliverableMessageChannel } from "./message-channel-normalize-BLLf0ubu.js";
import "./message-channel-BiOeMu0l.js";
import { t as resolveAgentSessionDirs } from "./session-dirs-COV6A9cq.js";
import { r as resolveAllAgentSessionStoreTargetsSync } from "./targets-BrMDn2yj.js";
import "./sessions-D6zZvoxX.js";
import { d as listActiveEmbeddedRunSessionIds, f as listActiveEmbeddedRunSessionKeys } from "./run-state-BYGFyjne.js";
import { d as readSessionMessagesAsync } from "./session-utils.fs-D_8-X4jl.js";
import { t as sanitizePendingFinalDeliveryText } from "./pending-final-delivery-BsRuDnJV.js";
import { g as resolveGatewaySessionStoreTarget } from "./session-utils-CHNCuY8a.js";
import { n as resolveSendPolicy } from "./send-policy-CfC5ATxp.js";
import fs from "node:fs";
import path from "node:path";
import crypto from "node:crypto";
//#region src/agents/main-session-restart-recovery.ts
/**
* Post-restart recovery for main sessions interrupted while holding a transcript lock.
*/
const log = createSubsystemLogger("main-session-restart-recovery");
const DEFAULT_RECOVERY_DELAY_MS = 5e3;
const MAX_RECOVERY_RETRIES = 3;
const RETRY_BACKOFF_MULTIPLIER = 2;
const UNRESUMABLE_SESSION_NOTICE = "I was interrupted by a gateway restart and couldn't safely resume the previous turn. Please send that last request again and I'll pick it up cleanly.";
function shouldSkipMainRecovery(entry, sessionKey) {
if (typeof entry.spawnDepth === "number" && entry.spawnDepth > 0) return true;
if (entry.subagentRole != null) return true;
return isSubagentSessionKey(sessionKey) || isCronSessionKey(sessionKey) || isAcpSessionKey(sessionKey);
}
function normalizeStringSet(values) {
const normalized = /* @__PURE__ */ new Set();
for (const value of values ?? []) {
const trimmed = value.trim();
if (trimmed) normalized.add(trimmed);
}
return normalized;
}
function normalizeFiniteTimestamp(value) {
return typeof value === "number" && Number.isFinite(value) ? value : void 0;
}
function hasCurrentProcessOwner(params) {
if (params.activeSessionIds.has(params.entry.sessionId)) return true;
return params.activeSessionIds.size === 0 && params.activeSessionKeys.has(params.sessionKey);
}
function normalizeTranscriptLockPath(lockPath) {
const trimmed = lockPath.trim();
if (!path.basename(trimmed).endsWith(".jsonl.lock")) return;
const resolved = path.resolve(trimmed);
try {
return path.join(fs.realpathSync(path.dirname(resolved)), path.basename(resolved));
} catch {
return resolved;
}
}
function resolveEntryTranscriptLockPaths(params) {
const paths = /* @__PURE__ */ new Set();
const push = (resolvePath) => {
try {
paths.add(path.resolve(`${resolvePath()}.lock`));
} catch {}
};
push(() => resolveSessionFilePath(params.entry.sessionId, params.entry, { sessionsDir: params.sessionsDir }));
push(() => resolveSessionTranscriptPathInDir(params.entry.sessionId, params.sessionsDir));
return [...paths];
}
async function markRestartAbortedMainSessions(params) {
const sessionKeys = normalizeStringSet(params.sessionKeys);
const sessionIds = normalizeStringSet(params.sessionIds);
const preferSessionIdMatch = sessionIds.size > 0;
const result = {
marked: 0,
skipped: 0
};
if (sessionKeys.size === 0 && sessionIds.size === 0) return result;
const storePaths = /* @__PURE__ */ new Set();
const env = params.stateDir === void 0 ? process.env : {
...process.env,
OPENCLAW_STATE_DIR: params.stateDir
};
const stateDir = resolveStateDir(env);
const configs = [params.cfg, ...params.additionalCfgs ?? []].filter((cfg) => Boolean(cfg));
for (const cfg of configs) {
try {
for (const target of resolveAllAgentSessionStoreTargetsSync(cfg, { env })) storePaths.add(path.resolve(target.storePath));
} catch (err) {
log.warn(`failed to resolve configured session stores for restart marker: ${String(err)}`);
}
for (const sessionKey of sessionKeys) try {
const target = resolveGatewaySessionStoreTarget({
cfg,
key: sessionKey,
scanLegacyKeys: true
});
storePaths.add(path.resolve(target.storePath));
for (const storeKey of target.storeKeys) {
const trimmed = storeKey.trim();
if (trimmed) sessionKeys.add(trimmed);
}
} catch (err) {
log.warn(`failed to resolve session store for restart marker ${sessionKey}: ${String(err)}`);
}
}
for (const sessionsDir of await resolveAgentSessionDirs(stateDir)) storePaths.add(path.join(sessionsDir, "sessions.json"));
for (const storePath of storePaths) await updateSessionStore(storePath, (store) => {
for (const [sessionKey, entry] of Object.entries(store)) {
if (!entry || entry.status !== "running") continue;
if (!(typeof entry.sessionId === "string" && sessionIds.has(entry.sessionId) ? true : !preferSessionIdMatch && sessionKeys.has(sessionKey))) continue;
if (shouldSkipMainRecovery(entry, sessionKey)) {
result.skipped++;
continue;
}
entry.abortedLastRun = true;
entry.updatedAt = Date.now();
store[sessionKey] = entry;
result.marked++;
}
}, { skipMaintenance: true });
if (result.marked > 0) log.warn(`marked ${result.marked} interrupted main session(s) for restart recovery${params.reason ? ` (${params.reason})` : ""}`);
return result;
}
async function markStartupOrphanedMainSessionsForRecovery(params) {
const result = {
marked: 0,
skipped: 0
};
const providedActiveSessionIds = params.activeSessionIds === void 0 ? void 0 : normalizeStringSet(params.activeSessionIds);
const providedActiveSessionKeys = params.activeSessionKeys === void 0 ? void 0 : normalizeStringSet(params.activeSessionKeys);
const updatedBeforeMs = normalizeFiniteTimestamp(params.updatedBeforeMs);
const resolveActiveSessionIds = () => providedActiveSessionIds ?? normalizeStringSet(listActiveEmbeddedRunSessionIds());
const resolveActiveSessionKeys = () => providedActiveSessionKeys ?? normalizeStringSet(listActiveEmbeddedRunSessionKeys());
for (const storePath of await resolveRestartRecoveryStorePaths(params)) await updateSessionStore(storePath, (store) => {
for (const [sessionKey, entry] of Object.entries(store)) {
if (!entry || entry.status !== "running" || entry.abortedLastRun === true) continue;
if (shouldSkipMainRecovery(entry, sessionKey)) {
result.skipped++;
continue;
}
const updatedAt = normalizeFiniteTimestamp(entry.updatedAt);
if (updatedBeforeMs !== void 0 && updatedAt !== void 0 && updatedAt > updatedBeforeMs) continue;
if (hasCurrentProcessOwner({
activeSessionIds: resolveActiveSessionIds(),
activeSessionKeys: resolveActiveSessionKeys(),
entry,
sessionKey
})) continue;
entry.abortedLastRun = true;
entry.updatedAt = Date.now();
store[sessionKey] = entry;
result.marked++;
}
}, { skipMaintenance: true });
if (result.marked > 0) log.warn(`marked ${result.marked} startup-orphaned main session(s) for restart recovery`);
return result;
}
function getMessageRole(message) {
if (!message || typeof message !== "object") return;
const role = message.role;
return typeof role === "string" ? role : void 0;
}
function isMeaningfulTailMessage(message) {
const role = getMessageRole(message);
if (!role || role === "system") return false;
return true;
}
function isResumableTailMessage(message) {
const role = getMessageRole(message);
return role === "user" || role === "tool" || role === "toolResult";
}
function isApprovalPendingToolResult(message) {
if (!message || typeof message !== "object" || getMessageRole(message) !== "toolResult") return false;
const details = message.details;
if (!details || typeof details !== "object") return false;
return details.status === "approval-pending";
}
function resolveMainSessionResumeBlockReason(messages) {
const lastMeaningful = messages.toReversed().find(isMeaningfulTailMessage);
if (!lastMeaningful || !isResumableTailMessage(lastMeaningful)) return "transcript tail is not resumable";
if (isApprovalPendingToolResult(lastMeaningful)) return "transcript tail is a stale approval-pending tool result";
return null;
}
function buildResumeMessage(pendingFinalDeliveryText) {
const base = "[System] Your previous turn was interrupted by a gateway restart while OpenClaw was waiting on tool/model work. Continue from the existing transcript and finish the interrupted response.";
const sanitizedPendingText = typeof pendingFinalDeliveryText === "string" ? sanitizePendingFinalDeliveryText(pendingFinalDeliveryText) : "";
if (sanitizedPendingText) return `${base}\n\nNote: The interrupted final reply was captured: "${sanitizedPendingText}"`;
return base;
}
async function markSessionFailed(params) {
await updateSessionStore(params.storePath, (store) => {
const entry = store[params.sessionKey];
if (!entry || entry.status !== "running") return;
entry.status = "failed";
entry.abortedLastRun = true;
entry.endedAt = Date.now();
entry.updatedAt = entry.endedAt;
entry.pendingFinalDelivery = void 0;
entry.pendingFinalDeliveryText = void 0;
entry.pendingFinalDeliveryCreatedAt = void 0;
entry.pendingFinalDeliveryLastAttemptAt = void 0;
entry.pendingFinalDeliveryAttemptCount = void 0;
entry.pendingFinalDeliveryLastError = void 0;
entry.pendingFinalDeliveryContext = void 0;
entry.restartRecoveryDeliveryContext = void 0;
entry.restartRecoveryDeliveryRunId = void 0;
store[params.sessionKey] = entry;
}, { skipMaintenance: true });
log.warn(`marked interrupted main session failed: ${params.sessionKey} (${params.reason})`);
}
async function sendUnresumableSessionNotice(params) {
const deliveryContext = resolveRestartRecoveryDeliveryContext({
cfg: params.cfg,
entry: params.entry,
includeSessionDeliveryFallback: true,
sessionKey: params.sessionKey
});
if (!deliveryContext) return false;
const messageParams = {
to: deliveryContext.to,
message: UNRESUMABLE_SESSION_NOTICE,
bestEffort: true
};
if (deliveryContext?.threadId != null) messageParams.threadId = deliveryContext.threadId;
const actionParams = {
channel: deliveryContext.channel,
action: "send",
sessionKey: params.sessionKey,
sessionId: params.entry.sessionId,
idempotencyKey: `main-session-restart-recovery:${params.entry.sessionId}:failed-notice`,
params: messageParams
};
const accountId = normalizeOptionalString(deliveryContext?.accountId);
if (accountId) actionParams.accountId = accountId;
try {
await callGateway({
method: "message.action",
params: actionParams,
timeoutMs: 1e4
});
log.info(`sent interrupted main session recovery notice: ${params.sessionKey} (${params.reason})`);
return true;
} catch (err) {
log.warn(`failed to send interrupted main session recovery notice ${params.sessionKey}: ${String(err)}`);
return false;
}
}
function resolveRestartRecoveryDeliveryContext(params) {
const deliveryContext = normalizeDeliveryContext(params.entry.pendingFinalDeliveryContext) ?? normalizeDeliveryContext(params.entry.restartRecoveryDeliveryContext) ?? (params.includeSessionDeliveryFallback ? deliveryContextFromSession(params.entry) : void 0);
const channel = normalizeOptionalString(deliveryContext?.channel);
const to = normalizeOptionalString(deliveryContext?.to);
if (!channel || !to || !isDeliverableMessageChannel(channel)) return;
if (params.cfg && resolveSendPolicy({
cfg: params.cfg,
entry: params.entry,
sessionKey: params.sessionKey,
channel,
chatType: params.entry.chatType
}) === "deny") return;
return {
...deliveryContext,
channel,
to
};
}
async function resumeMainSession(params) {
const sanitizedPendingText = typeof params.pendingFinalDeliveryText === "string" ? sanitizePendingFinalDeliveryText(params.pendingFinalDeliveryText) : "";
const deliveryContext = resolveRestartRecoveryDeliveryContext({
cfg: params.cfg,
entry: params.entry,
sessionKey: params.sessionKey
});
try {
const agentParams = {
message: buildResumeMessage(sanitizedPendingText),
sessionKey: params.sessionKey,
idempotencyKey: crypto.randomUUID(),
deliver: Boolean(deliveryContext),
lane: "main"
};
if (deliveryContext) {
agentParams.channel = deliveryContext.channel;
agentParams.to = deliveryContext.to;
agentParams.bestEffortDeliver = true;
if (deliveryContext.accountId) agentParams.accountId = deliveryContext.accountId;
if (deliveryContext.threadId != null) agentParams.threadId = String(deliveryContext.threadId);
}
await callGateway({
method: "agent",
params: agentParams,
timeoutMs: 1e4
});
await updateSessionStore(params.storePath, (store) => {
const entry = store[params.sessionKey];
if (!entry) return;
const now = Date.now();
entry.abortedLastRun = false;
entry.updatedAt = now;
if (entry.pendingFinalDelivery || entry.pendingFinalDeliveryText) if (sanitizedPendingText) {
entry.pendingFinalDeliveryLastAttemptAt = now;
entry.pendingFinalDeliveryAttemptCount = (entry.pendingFinalDeliveryAttemptCount ?? 0) + 1;
entry.pendingFinalDeliveryLastError = null;
entry.pendingFinalDeliveryText = sanitizedPendingText;
} else {
entry.pendingFinalDelivery = void 0;
entry.pendingFinalDeliveryText = void 0;
entry.pendingFinalDeliveryCreatedAt = void 0;
entry.pendingFinalDeliveryLastAttemptAt = void 0;
entry.pendingFinalDeliveryAttemptCount = void 0;
entry.pendingFinalDeliveryLastError = void 0;
entry.pendingFinalDeliveryContext = void 0;
}
store[params.sessionKey] = entry;
}, { skipMaintenance: true });
log.info(`resumed interrupted main session: ${params.sessionKey}${sanitizedPendingText ? " (with pending payload)" : ""}`);
return true;
} catch (err) {
log.warn(`failed to resume interrupted main session ${params.sessionKey}: ${String(err)}`);
return false;
}
}
async function markRestartAbortedMainSessionsFromLocks(params) {
const result = {
marked: 0,
skipped: 0
};
const sessionsDir = path.resolve(params.sessionsDir);
const interruptedLockPaths = new Set(params.cleanedLocks.map((lock) => normalizeTranscriptLockPath(lock.lockPath)).filter((lockPath) => Boolean(lockPath)));
if (interruptedLockPaths.size === 0) return result;
await updateSessionStore(path.join(sessionsDir, "sessions.json"), (store) => {
for (const [sessionKey, entry] of Object.entries(store)) {
if (!entry || entry.status !== "running") continue;
if (shouldSkipMainRecovery(entry, sessionKey)) {
result.skipped++;
continue;
}
if (!resolveEntryTranscriptLockPaths({
entry,
sessionsDir
}).some((lockPath) => interruptedLockPaths.has(lockPath))) continue;
entry.abortedLastRun = true;
store[sessionKey] = entry;
result.marked++;
}
}, { skipMaintenance: true });
if (result.marked > 0) log.warn(`marked ${result.marked} interrupted main session(s) from stale transcript locks`);
return result;
}
function isRoutableRecoveryStore(params) {
if (!params.cfg) return true;
if (!params.cfg.session?.store) return true;
try {
const target = resolveGatewaySessionStoreTarget({
cfg: params.cfg,
key: params.sessionKey,
scanLegacyKeys: true
});
return path.resolve(target.storePath) === path.resolve(params.storePath);
} catch (err) {
log.warn(`failed to resolve recovery store for ${params.sessionKey}: ${String(err)}`);
return false;
}
}
async function recoverStore(params) {
const result = {
recovered: 0,
failed: 0,
skipped: 0
};
const providedActiveSessionIds = params.activeSessionIds === void 0 ? void 0 : normalizeStringSet(params.activeSessionIds);
const providedActiveSessionKeys = params.activeSessionKeys === void 0 ? void 0 : normalizeStringSet(params.activeSessionKeys);
const resolveActiveSessionIds = () => providedActiveSessionIds ?? normalizeStringSet(listActiveEmbeddedRunSessionIds());
const resolveActiveSessionKeys = () => providedActiveSessionKeys ?? normalizeStringSet(listActiveEmbeddedRunSessionKeys());
let store;
try {
store = loadSessionStore(params.storePath);
} catch (err) {
log.warn(`failed to load session store ${params.storePath}: ${String(err)}`);
result.failed++;
return result;
}
for (const [sessionKey, entry] of Object.entries(store).toSorted(([a], [b]) => a.localeCompare(b))) {
if (!entry || entry.status !== "running" || entry.abortedLastRun !== true) continue;
if (shouldSkipMainRecovery(entry, sessionKey)) {
result.skipped++;
continue;
}
if (!isRoutableRecoveryStore({
cfg: params.cfg,
sessionKey,
storePath: params.storePath
})) {
result.skipped++;
continue;
}
if (hasCurrentProcessOwner({
activeSessionIds: resolveActiveSessionIds(),
activeSessionKeys: resolveActiveSessionKeys(),
entry,
sessionKey
})) {
result.skipped++;
continue;
}
const resumeDedupeKey = sessionKey;
if (params.resumedSessionKeys.has(resumeDedupeKey)) {
result.skipped++;
continue;
}
if (entry.pendingFinalDelivery === true && entry.pendingFinalDeliveryText) {
if (await resumeMainSession({
cfg: params.cfg,
entry,
storePath: params.storePath,
sessionKey,
pendingFinalDeliveryText: entry.pendingFinalDeliveryText
})) {
params.resumedSessionKeys.add(resumeDedupeKey);
result.recovered++;
} else result.failed++;
continue;
}
let messages;
try {
messages = await readSessionMessagesAsync(entry.sessionId, params.storePath, entry.sessionFile, {
mode: "recent",
maxMessages: 20,
maxBytes: 256 * 1024
});
} catch (err) {
log.warn(`failed to read transcript for ${sessionKey}: ${String(err)}`);
result.failed++;
continue;
}
const resumeBlockReason = resolveMainSessionResumeBlockReason(messages);
if (resumeBlockReason) {
await sendUnresumableSessionNotice({
cfg: params.cfg,
entry,
sessionKey,
reason: resumeBlockReason
});
await markSessionFailed({
storePath: params.storePath,
sessionKey,
reason: resumeBlockReason
});
result.failed++;
continue;
}
if (await resumeMainSession({
cfg: params.cfg,
entry,
storePath: params.storePath,
sessionKey,
pendingFinalDeliveryText: entry.pendingFinalDeliveryText
})) {
params.resumedSessionKeys.add(resumeDedupeKey);
result.recovered++;
} else result.failed++;
}
return result;
}
async function resolveRestartRecoveryStorePaths(params) {
const storePaths = /* @__PURE__ */ new Set();
const stateDir = params.stateDir ?? resolveStateDir(process.env);
for (const sessionsDir of await resolveAgentSessionDirs(stateDir)) storePaths.add(path.join(sessionsDir, "sessions.json"));
if (params.cfg) {
const env = {
...process.env,
OPENCLAW_STATE_DIR: stateDir
};
for (const target of resolveAllAgentSessionStoreTargetsSync(params.cfg, { env })) storePaths.add(path.resolve(target.storePath));
}
return [...storePaths].toSorted((a, b) => a.localeCompare(b));
}
async function recoverRestartAbortedMainSessions(params = {}) {
const result = {
recovered: 0,
failed: 0,
skipped: 0
};
const resumedSessionKeys = params.resumedSessionKeys ?? /* @__PURE__ */ new Set();
for (const storePath of await resolveRestartRecoveryStorePaths(params)) {
const storeResult = await recoverStore({
cfg: params.cfg,
storePath,
resumedSessionKeys,
activeSessionIds: params.activeSessionIds,
activeSessionKeys: params.activeSessionKeys
});
result.recovered += storeResult.recovered;
result.failed += storeResult.failed;
result.skipped += storeResult.skipped;
}
if (result.recovered > 0 || result.failed > 0) log.info(`main-session restart recovery complete: recovered=${result.recovered} failed=${result.failed} skipped=${result.skipped}`);
return result;
}
async function recoverStartupOrphanedMainSessions(params = {}) {
const startupRecoveryCutoffMs = params.updatedBeforeMs ?? Date.now();
const marked = await markStartupOrphanedMainSessionsForRecovery({
cfg: params.cfg,
stateDir: params.stateDir,
activeSessionIds: params.activeSessionIds,
activeSessionKeys: params.activeSessionKeys,
updatedBeforeMs: startupRecoveryCutoffMs
});
const recovered = await recoverRestartAbortedMainSessions({
cfg: params.cfg,
stateDir: params.stateDir,
resumedSessionKeys: params.resumedSessionKeys,
activeSessionIds: params.activeSessionIds,
activeSessionKeys: params.activeSessionKeys
});
return {
marked: marked.marked,
recovered: recovered.recovered,
failed: recovered.failed,
skipped: marked.skipped + recovered.skipped
};
}
function scheduleRestartAbortedMainSessionRecovery(params = {}) {
const initialDelay = params.delayMs ?? DEFAULT_RECOVERY_DELAY_MS;
const maxRetries = params.maxRetries ?? MAX_RECOVERY_RETRIES;
const resumedSessionKeys = /* @__PURE__ */ new Set();
const startupRecoveryCutoffMs = Date.now();
const runRecoveryAttempt = (attempt, delay) => {
recoverStartupOrphanedMainSessions({
cfg: params.cfg,
stateDir: params.stateDir,
resumedSessionKeys,
updatedBeforeMs: startupRecoveryCutoffMs
}).then((result) => {
if (result.failed > 0 && attempt < maxRetries) scheduleAttempt(attempt + 1, delay * RETRY_BACKOFF_MULTIPLIER);
}).catch((err) => {
if (attempt < maxRetries) {
log.warn(`main-session restart recovery failed: ${String(err)}`);
scheduleAttempt(attempt + 1, delay * RETRY_BACKOFF_MULTIPLIER);
} else log.warn(`main-session restart recovery gave up: ${String(err)}`);
});
};
const scheduleAttempt = (attempt, delay) => {
if (delay <= 0) {
runRecoveryAttempt(attempt, delay);
return;
}
setTimeout(() => {
runRecoveryAttempt(attempt, delay);
}, delay).unref?.();
};
scheduleAttempt(1, initialDelay);
}
//#endregion
export { recoverStartupOrphanedMainSessions as a, recoverRestartAbortedMainSessions as i, markRestartAbortedMainSessionsFromLocks as n, scheduleRestartAbortedMainSessionRecovery as o, markStartupOrphanedMainSessionsForRecovery as r, markRestartAbortedMainSessions as t };