openclaw
Version:
Multi-channel AI gateway with extensible messaging integrations
146 lines (145 loc) • 7.81 kB
JavaScript
import { r as truncateUtf16Safe } from "./utf16-slice-D_ngcYKd.js";
import { t as formatCliCommand } from "./command-format-C7YfyMTd.js";
import { t as createSubsystemLogger } from "./subsystem-Dy2tqXOS.js";
import { a as getNodeSqliteKysely, i as executeSqliteQueryTakeFirstSync, r as executeSqliteQuerySync } from "./kysely-sync-COmh4HWh.js";
import { i as openOpenClawStateDatabase, s as runOpenClawStateWriteTransaction } from "./openclaw-state-db-BRTnL-D8.js";
import "./openclaw-agent-db-contract-CGTyjij4.js";
import { i as formatLegacyAgentMediaMigrationRequiredMessage, t as GATEWAY_AGENT_MEDIA_MIGRATION_REQUIRED_REASON } from "./openclaw-agent-db-migration-required-BHhCEvlh.js";
import { randomUUID } from "node:crypto";
//#region src/infra/gateway-boot-lifecycle.ts
const GATEWAY_BOOT_LOOP_UNCLEAN_THRESHOLD = 3;
const GATEWAY_BOOT_LOOP_WINDOW_MS = 3e5;
const GATEWAY_BOOT_LIFECYCLE_RETENTION_MS = 864e5;
const GATEWAY_CRASH_LOOP_BREAKER_REASON = "gateway.crash_loop_breaker";
const GATEWAY_CRASH_LOOP_RECOVERED_REASON = "gateway.crash_loop_recovered";
/**
* The breaker only self-clears after the full window drains. Operator surfaces name the manual
* override command, not the internal RPC. Account hints carry accountId to avoid starting a
* different default account than the warning named.
*/
function formatGatewayCrashLoopManualChannelStartHint(target) {
const params = JSON.stringify({
channel: target?.channelId ?? "<id>",
...target?.accountId ? { accountId: target.accountId } : {}
});
return `Start a channel manually with: ${formatCliCommand("openclaw gateway call channels.start")} --params '${params}'`;
}
const gatewayLifecycleLog = createSubsystemLogger("gateway/lifecycle");
function buildGatewayCrashLoopBreakerDecision(params) {
const windowMs = params.windowMs ?? GATEWAY_BOOT_LOOP_WINDOW_MS;
const tripped = params.uncleanBoots >= GATEWAY_BOOT_LOOP_UNCLEAN_THRESHOLD;
const hasUnrecoveredBreakerMarker = typeof params.latestBreakerStartedAtMs === "number" && (typeof params.latestRecoveryStartedAtMs !== "number" || params.latestRecoveryStartedAtMs < params.latestBreakerStartedAtMs);
return {
tripped,
uncleanBoots: params.uncleanBoots,
windowMs,
shouldWriteStabilityBundle: tripped && !hasUnrecoveredBreakerMarker,
recovered: !tripped && hasUnrecoveredBreakerMarker
};
}
function inspectGatewayCrashLoopBreaker(env = process.env, nowMs = Date.now()) {
try {
const { db } = openOpenClawStateDatabase({ env });
const kysely = getNodeSqliteKysely(db);
const windowStartMs = nowMs - GATEWAY_BOOT_LOOP_WINDOW_MS;
const uncleanRow = executeSqliteQueryTakeFirstSync(db, kysely.selectFrom("gateway_boot_lifecycle").select((eb) => eb.fn.countAll().as("count")).where((eb) => eb.or([eb.and([eb("completed_at_ms", "is", null), eb("started_at_ms", ">=", windowStartMs)]), eb.and([eb("outcome", "=", "startup_failed"), eb("completed_at_ms", ">=", windowStartMs)])])));
const latestBreaker = executeSqliteQueryTakeFirstSync(db, kysely.selectFrom("gateway_boot_lifecycle").select("started_at_ms as startedAtMs").where("startup_reason", "=", GATEWAY_CRASH_LOOP_BREAKER_REASON).orderBy("started_at_ms", "desc").limit(1));
const latestRecovery = executeSqliteQueryTakeFirstSync(db, kysely.selectFrom("gateway_boot_lifecycle").select("started_at_ms as startedAtMs").where("startup_reason", "=", GATEWAY_CRASH_LOOP_RECOVERED_REASON).orderBy("started_at_ms", "desc").limit(1));
return buildGatewayCrashLoopBreakerDecision({
uncleanBoots: uncleanRow?.count ?? 0,
latestBreakerStartedAtMs: latestBreaker?.startedAtMs,
latestRecoveryStartedAtMs: latestRecovery?.startedAtMs
});
} catch (err) {
gatewayLifecycleLog.warn(`crash-loop breaker state unavailable; fail-open: ${String(err)}`);
return buildGatewayCrashLoopBreakerDecision({ uncleanBoots: 0 });
}
}
function recordGatewayBootStart(env = process.env, nowMs = Date.now(), reason) {
const bootId = randomUUID();
try {
runOpenClawStateWriteTransaction(({ db }) => {
const kysely = getNodeSqliteKysely(db);
executeSqliteQuerySync(db, kysely.deleteFrom("gateway_boot_lifecycle").where("started_at_ms", "<", nowMs - GATEWAY_BOOT_LIFECYCLE_RETENTION_MS));
executeSqliteQuerySync(db, kysely.insertInto("gateway_boot_lifecycle").values({
boot_id: bootId,
pid: process.pid,
started_at_ms: nowMs,
completed_at_ms: null,
outcome: null,
startup_reason: reason ?? null,
reason: null
}));
}, { env });
return bootId;
} catch (err) {
gatewayLifecycleLog.warn(`failed to persist gateway boot start; fail-open: ${String(err)}`);
return;
}
}
/**
* Split a stable safe-mode lifetime before channel autostart resumes. A fresh
* open row makes a process death during recovered channel startup count toward
* the next breaker decision instead of aging out with the original boot.
*/
function recordGatewayCrashLoopRecovery(bootId, env = process.env, nowMs = Date.now()) {
const recoveredBootId = randomUUID();
try {
runOpenClawStateWriteTransaction(({ db }) => {
const kysely = getNodeSqliteKysely(db);
if (bootId) executeSqliteQuerySync(db, kysely.updateTable("gateway_boot_lifecycle").set({
completed_at_ms: nowMs,
outcome: "safe_mode_stable",
reason: null
}).where("boot_id", "=", bootId));
executeSqliteQuerySync(db, kysely.insertInto("gateway_boot_lifecycle").values({
boot_id: recoveredBootId,
pid: process.pid,
started_at_ms: nowMs,
completed_at_ms: null,
outcome: null,
startup_reason: GATEWAY_CRASH_LOOP_RECOVERED_REASON,
reason: null
}));
}, { env });
return recoveredBootId;
} catch (err) {
gatewayLifecycleLog.warn(`failed to persist gateway crash-loop recovery; fail-safe: ${String(err)}`);
return;
}
}
function completeGatewayBootLifecycle(bootId, completion, env = process.env, nowMs = Date.now()) {
if (!bootId) return;
try {
runOpenClawStateWriteTransaction(({ db }) => {
const kysely = getNodeSqliteKysely(db);
executeSqliteQuerySync(db, kysely.updateTable("gateway_boot_lifecycle").set({
completed_at_ms: nowMs,
outcome: completion.outcome,
...completion.startupReason ? { startup_reason: completion.startupReason } : {},
reason: completion.reason ?? null
}).where("boot_id", "=", bootId));
}, { env });
} catch (err) {
gatewayLifecycleLog.warn(`failed to persist gateway boot outcome; fail-open: ${String(err)}`);
}
}
function repairGatewayAgentMediaMigrationStartupFailures(params) {
if (params.databasePaths.length === 0) return 0;
try {
return runOpenClawStateWriteTransaction(({ db }) => {
const kysely = getNodeSqliteKysely(db);
const legacyMessages = [...new Set(params.databasePaths.flatMap((pathname) => Array.from({ length: 19 }, (_, schemaVersion) => {
const message = formatLegacyAgentMediaMigrationRequiredMessage(pathname, schemaVersion);
return [message, truncateUtf16Safe(message, 500)];
}).flat()))];
const result = executeSqliteQuerySync(db, kysely.updateTable("gateway_boot_lifecycle").set({ outcome: "startup_failure_repaired" }).where("outcome", "=", "startup_failed").where((eb) => eb.or([eb("startup_reason", "=", GATEWAY_AGENT_MEDIA_MIGRATION_REQUIRED_REASON), eb("reason", "in", legacyMessages)])));
return Number(result.numAffectedRows ?? 0);
}, { env: params.env ?? process.env });
} catch (err) {
gatewayLifecycleLog.warn(`failed to repair media-migration startup history; fail-open: ${String(err)}`);
return 0;
}
}
//#endregion
export { inspectGatewayCrashLoopBreaker as a, repairGatewayAgentMediaMigrationStartupFailures as c, formatGatewayCrashLoopManualChannelStartHint as i, GATEWAY_CRASH_LOOP_RECOVERED_REASON as n, recordGatewayBootStart as o, completeGatewayBootLifecycle as r, recordGatewayCrashLoopRecovery as s, GATEWAY_CRASH_LOOP_BREAKER_REASON as t };