openclaw
Version:
Multi-channel AI gateway with extensible messaging integrations
140 lines (139 loc) • 6.07 kB
JavaScript
import { j as resolveTimerTimeoutMs } from "./number-coercion-CJQ8TR--.js";
import "./number-coercion-Z7n6tXLk.js";
import { t as createSubsystemLogger } from "./subsystem-BzXSmsuh.js";
import { i as resolveChannelRestartReason, r as evaluateChannelHealth } from "./channel-health-policy-D_eDwUBm.js";
//#region src/gateway/server-runtime-service-shared.ts
/** Creates a heartbeat runner placeholder for minimal/test gateway service state. */
function createNoopHeartbeatRunner() {
return {
stop: () => {},
updateConfig: (_cfg) => {}
};
}
//#endregion
//#region src/gateway/channel-health-monitor.ts
const log = createSubsystemLogger("gateway/health-monitor");
const DEFAULT_CHECK_INTERVAL_MS = 5 * 6e4;
const DEFAULT_MONITOR_STARTUP_GRACE_MS = 6e4;
const DEFAULT_COOLDOWN_CYCLES = 2;
const DEFAULT_MAX_RESTARTS_PER_HOUR = 10;
const ONE_HOUR_MS = 60 * 6e4;
function resolveTimingPolicy(deps) {
return {
monitorStartupGraceMs: deps.timing?.monitorStartupGraceMs ?? deps.startupGraceMs ?? DEFAULT_MONITOR_STARTUP_GRACE_MS,
channelConnectGraceMs: deps.timing?.channelConnectGraceMs ?? deps.channelStartupGraceMs ?? 12e4,
staleEventThresholdMs: deps.timing?.staleEventThresholdMs ?? deps.staleEventThresholdMs ?? 18e5
};
}
/** Start the periodic channel health monitor and return its stop handle. */
function startChannelHealthMonitor(deps) {
const { channelManager, cooldownCycles = DEFAULT_COOLDOWN_CYCLES, maxRestartsPerHour = DEFAULT_MAX_RESTARTS_PER_HOUR, abortSignal } = deps;
const checkIntervalMs = resolveTimerTimeoutMs(deps.checkIntervalMs, DEFAULT_CHECK_INTERVAL_MS);
const timing = resolveTimingPolicy(deps);
const cooldownMs = cooldownCycles * checkIntervalMs;
const restartRecords = /* @__PURE__ */ new Map();
const startedAt = Date.now();
let stopped = false;
let checkInFlight = false;
let timer = null;
const rKey = (channelId, accountId) => `${channelId}:${accountId}`;
function pruneOldRestarts(record, now) {
record.restartsThisHour = record.restartsThisHour.filter((r) => now - r.at < ONE_HOUR_MS);
}
async function runCheck() {
if (stopped || checkInFlight) return;
checkInFlight = true;
try {
const now = Date.now();
if (now - startedAt < timing.monitorStartupGraceMs) return;
const snapshot = channelManager.getRuntimeSnapshot();
for (const [channelId, accounts] of Object.entries(snapshot.channelAccounts)) {
if (!accounts) continue;
for (const [accountId, status] of Object.entries(accounts)) {
if (!status) continue;
if (!channelManager.isHealthMonitorEnabled(channelId, accountId)) continue;
if (channelManager.isManuallyStopped(channelId, accountId)) continue;
const health = evaluateChannelHealth(status, {
channelId,
now,
staleEventThresholdMs: timing.staleEventThresholdMs,
channelConnectGraceMs: timing.channelConnectGraceMs
});
if (health.healthy) continue;
const key = rKey(channelId, accountId);
const record = restartRecords.get(key) ?? {
lastRestartAt: 0,
restartsThisHour: []
};
if (now - record.lastRestartAt <= cooldownMs) continue;
pruneOldRestarts(record, now);
if (record.restartsThisHour.length >= maxRestartsPerHour) {
log.warn?.(`[${channelId}:${accountId}] health-monitor: hit ${maxRestartsPerHour} restarts/hour limit, skipping`);
continue;
}
const reason = resolveChannelRestartReason(status, health);
log.info?.(`[${channelId}:${accountId}] health-monitor: restarting (reason: ${reason})`);
record.lastRestartAt = now;
record.restartsThisHour.push({ at: now });
restartRecords.set(key, record);
try {
if (status.running) await channelManager.stopChannel(channelId, accountId, { manual: false });
channelManager.resetRestartAttempts(channelId, accountId);
await channelManager.startChannel(channelId, accountId);
} catch (err) {
log.error?.(`[${channelId}:${accountId}] health-monitor: restart failed: ${String(err)}`);
}
}
}
} catch (err) {
log.error?.(`health-monitor: check failed: ${String(err)}`);
} finally {
checkInFlight = false;
}
}
function stop() {
stopped = true;
if (timer) {
clearInterval(timer);
timer = null;
}
abortSignal?.removeEventListener("abort", stop);
}
if (abortSignal?.aborted) stopped = true;
else {
abortSignal?.addEventListener("abort", stop, { once: true });
timer = setInterval(() => void runCheck(), checkIntervalMs);
if (typeof timer === "object" && "unref" in timer) timer.unref();
log.info?.(`started (interval: ${Math.round(checkIntervalMs / 1e3)}s, startup-grace: ${Math.round(timing.monitorStartupGraceMs / 1e3)}s, channel-connect-grace: ${Math.round(timing.channelConnectGraceMs / 1e3)}s)`);
}
return { stop };
}
//#endregion
//#region src/gateway/server-runtime-startup-services.ts
/** Starts channel health monitoring when gateway config enables it. */
function startGatewayChannelHealthMonitor(params) {
const healthCheckMinutes = params.cfg.gateway?.channelHealthCheckMinutes;
if (healthCheckMinutes === 0) return null;
const staleEventThresholdMinutes = params.cfg.gateway?.channelStaleEventThresholdMinutes;
const maxRestartsPerHour = params.cfg.gateway?.channelMaxRestartsPerHour;
return startChannelHealthMonitor({
channelManager: params.channelManager,
checkIntervalMs: (healthCheckMinutes ?? 5) * 6e4,
...staleEventThresholdMinutes != null && { staleEventThresholdMs: staleEventThresholdMinutes * 6e4 },
...maxRestartsPerHour != null && { maxRestartsPerHour }
});
}
/** Starts background runtime services and returns their stop/update handles. */
function startGatewayRuntimeServices(params) {
const channelHealthMonitor = startGatewayChannelHealthMonitor({
cfg: params.cfgAtStart,
channelManager: params.channelManager
});
return {
heartbeatRunner: createNoopHeartbeatRunner(),
channelHealthMonitor,
stopModelPricingRefresh: () => {}
};
}
//#endregion
export { startGatewayRuntimeServices as n, createNoopHeartbeatRunner as r, startGatewayChannelHealthMonitor as t };