openclaw
Version:
Multi-channel AI gateway with extensible messaging integrations
544 lines (543 loc) • 22.8 kB
JavaScript
import { F as resolveTimerTimeoutMs } from "./number-coercion-CLj0HTDM.js";
import "./src-vebZIeLe.js";
import { t as expectDefined } from "./expect-CyE8FADM.js";
import { r as asNullableRecord } from "./record-coerce-DItp3I4t.js";
import { l as normalizeOptionalString } from "./string-coerce-CIXf7egm.js";
import { r as truncateUtf16Safe } from "./utf16-slice-D_ngcYKd.js";
import { g as redactToolPayloadTextWithConfig } from "./redact-BtvPPfTi.js";
import { n as normalizeAgentId } from "./agent-id-CeT3w4ap.js";
import { D as tryResolveLegacyCompatibilityAgentId, i as listAgentEntries } from "./agent-scope-config-DcbEhP0R.js";
import "./session-key-BnWWjqNc.js";
import "./legacy.default-agent-owner-BGwEdQRe.js";
import { t as formatErrorMessage } from "./errors-Db3Ymjlb.js";
import { t as createSubsystemLogger } from "./subsystem-Dy2tqXOS.js";
import { t as isDiagnosticFlagEnabled } from "./diagnostic-flags-qVPAqdVk.js";
import { o as resolveSessionStorePathCore } from "./paths-CXdaYWF_.js";
import "./agent-scope-DbtJyKUL.js";
import { t as runTasksWithConcurrency } from "./run-with-concurrency-Dtu208ef.js";
import { u as getActivePluginRegistry } from "./runtime-BL4wZfTq.js";
import { i as trackAsyncWork } from "./async-work-scope-CMQS2uTf.js";
import { l as toPublicPluginVerificationDiagnostic, o as listActiveDegradedPlugins, r as degradedPluginMatchesRoot } from "./runtime-degraded-state-B_-lbcPZ.js";
import { i as resolveHeartbeatSummariesForAgents } from "./heartbeat-summary-projection-0z8UfVxf.js";
import { a as resolvePreferredAccountId, t as buildChannelAccountBindings } from "./bindings-RJ9_sjHk.js";
import { i as resolveChannelDefaultAccountId } from "./helpers-VcK_WJ24.js";
import { n as hasConfiguredUnavailableCredentialStatus, s as redactChannelStatusSummaryBaseUrl } from "./account-snapshot-fields-a22gRqLh.js";
import { a as resolveUnavailableChannelAccountSnapshot } from "./account-state-CPvd6WGN.js";
import { a as resolveChannelAccountEnabled, i as resolveChannelAccountConfigured, t as buildChannelAccountSnapshotFromInspection } from "./account-summary-oKRoWiin.js";
import { t as listReadOnlyChannelPluginsForConfig } from "./read-only-0zbnWxaR.js";
import { t as inspectChannelAccount } from "./account-inspection-DzzBSljF.js";
import { n as awaitWithinDeadline, t as ABSOLUTE_DEADLINE_EXPIRED } from "./absolute-deadline-D0jNXqHr.js";
import { t as buildChannelAccountSnapshotFromAccount } from "./status-BQhzABCJ.js";
import { i as resolveChannelHealthState, n as DEFAULT_CHANNEL_STALE_EVENT_THRESHOLD_MS, t as DEFAULT_CHANNEL_CONNECT_GRACE_MS } from "./channel-health-policy-D20awG2O.js";
import { n as listPluginServiceHealthFailures } from "./service-health-BvEAb43_.js";
import { n as buildContextEngineHealthSummary, t as buildDeliveryQueueHealthSummary } from "./delivery-queue-BPryUKwz.js";
//#region src/gateway/health/account-context.ts
const PUBLIC_IMESSAGE_FULL_DISK_ACCESS_ERROR = "imsg cannot access ~/Library/Messages/chat.db. Grant Full Disk Access to the Gateway/launcher process and restart Gateway.";
const redactIMessageProbeErrorMessage = (message) => {
const trimmed = message.trim();
if (!trimmed) return "";
return trimmed.replaceAll(/\/Users\/[^/\s]+\/Library\/Messages\/chat\.db/g, "~/Library/Messages/chat.db");
};
function buildNonSensitiveProbeFailure(channelId, probe) {
const record = asNullableRecord(probe);
if (channelId !== "imessage" || !record || record.ok !== false) return;
if (typeof record.error !== "string") return;
const error = redactIMessageProbeErrorMessage(record.error);
if (!/\bimsg\b/i.test(error) || !error.includes("~/Library/Messages/chat.db") || !/\bFull Disk Access\b/i.test(error)) return;
return {
ok: false,
error: PUBLIC_IMESSAGE_FULL_DISK_ACCESS_ERROR
};
}
function readBooleanField(value, key) {
const record = asNullableRecord(value);
if (!record) return;
return typeof record[key] === "boolean" ? record[key] : void 0;
}
const hasAccountValue = (account) => account !== null && account !== void 0;
function resolveProbeAccountEnabled(params) {
const fallback = readBooleanField(params.account, "enabled") ?? true;
try {
return resolveChannelAccountEnabled({
plugin: params.plugin,
account: params.account,
cfg: params.cfg
});
} catch (error) {
params.diagnostics.push(`${params.plugin.id}:${params.accountId}: failed to evaluate enabled state (${formatErrorMessage(error)}).`);
return fallback;
}
}
async function resolveProbeAccountConfigured(params) {
const fallback = readBooleanField(params.account, "configured") ?? true;
try {
return await resolveChannelAccountConfigured({
plugin: params.plugin,
account: params.account,
cfg: params.cfg,
readAccountConfiguredField: true
});
} catch (error) {
params.diagnostics.push(`${params.plugin.id}:${params.accountId}: failed to evaluate configured state (${formatErrorMessage(error)}).`);
return fallback;
}
}
async function resolveHealthAccountContext(params) {
const diagnostics = [];
let inspectedAccount;
try {
inspectedAccount = await inspectChannelAccount(params);
} catch (error) {
diagnostics.push(`${params.plugin.id}:${params.accountId}: failed to inspect account (${formatErrorMessage(error)}).`);
}
const inspectedEnabled = readBooleanField(inspectedAccount, "enabled");
const inspectedConfigured = readBooleanField(inspectedAccount, "configured");
let account;
if (inspectedEnabled !== false && !hasConfiguredUnavailableCredentialStatus(inspectedAccount)) try {
account = params.plugin.config.resolveAccount(params.cfg, params.accountId);
} catch (error) {
diagnostics.push(`${params.plugin.id}:${params.accountId}: failed to resolve account (${formatErrorMessage(error)}).`);
}
if (!hasAccountValue(account)) return {
probeAccount: void 0,
inspectedAccount,
enabled: inspectedEnabled ?? false,
configured: inspectedConfigured,
diagnostics
};
const enabled = resolveProbeAccountEnabled({
plugin: params.plugin,
cfg: params.cfg,
accountId: params.accountId,
account,
diagnostics
});
const configured = await resolveProbeAccountConfigured({
plugin: params.plugin,
cfg: params.cfg,
accountId: params.accountId,
account,
diagnostics
});
return {
probeAccount: account,
inspectedAccount,
enabled,
configured,
diagnostics
};
}
//#endregion
//#region src/gateway/health/collector.ts
const HEALTH_COLLECTION_TIMEOUT_MS = 7e3;
const HEALTH_PROBE_CONCURRENCY = 5;
const HEALTH_RECENT_SESSION_LIMIT = 5;
const healthLog = createSubsystemLogger("health");
const debugHealth = (cfg, message, meta) => {
if (isDiagnosticFlagEnabled("health", cfg)) healthLog.info(message, meta);
};
function resolveHealthAgentOrder(cfg) {
const defaultAgentId = tryResolveLegacyCompatibilityAgentId(cfg);
const entries = listAgentEntries(cfg);
const seen = /* @__PURE__ */ new Set();
const ordered = [];
for (const entry of entries) {
if (!entry || typeof entry !== "object") continue;
if (typeof entry.id !== "string" || !entry.id.trim()) continue;
const id = normalizeAgentId(entry.id);
if (!id || seen.has(id)) continue;
seen.add(id);
ordered.push({
id,
name: typeof entry.name === "string" ? entry.name : void 0
});
}
if (defaultAgentId && !seen.has(defaultAgentId)) ordered.unshift({ id: defaultAgentId });
return {
defaultAgentId,
ordered
};
}
async function createHealthSessionStoreReader(agentIds) {
const { createStatusSessionStoreReader } = await import("./session-stores-DxNOn7Xa.js");
const { readSessionStoreSummaryReadOnly } = await import("./session-accessor-pyriiR4V.js");
const { isTransientSqliteError } = await import("./unhandled-rejections-BbwPmmNc.js");
return createStatusSessionStoreReader(agentIds, HEALTH_RECENT_SESSION_LIMIT, (scope, options) => {
try {
return readSessionStoreSummaryReadOnly(scope, options);
} catch (error) {
if (!isTransientSqliteError(error)) throw error;
return {
count: 0,
recent: [],
byAgent: /* @__PURE__ */ new Map()
};
}
});
}
function projectHealthSessions(path, summary) {
const recent = summary.recent.map(({ sessionKey: key, entry }) => ({
key,
updatedAt: entry.updatedAt || null,
age: entry.updatedAt ? Date.now() - entry.updatedAt : null
}));
return {
path,
count: summary.count,
recent
};
}
async function buildHealthSessionSummary(storePath, agentId) {
const store = (await createHealthSessionStoreReader(agentId ? [agentId] : [])).read(storePath, agentId);
return projectHealthSessions(store.path, store);
}
/** Shares one bounded session snapshot across every configured agent in this collection. */
async function buildHealthAgentSummaries(cfg, { defaultAgentId, ordered }) {
const agentIds = ordered.map((entry) => entry.id);
const reader = await createHealthSessionStoreReader(agentIds);
const heartbeats = resolveHeartbeatSummariesForAgents(cfg, agentIds);
return ordered.map((entry, index) => {
const store = reader.read(resolveSessionStorePathCore(cfg.session?.store, { agentId: entry.id }), entry.id);
return {
agentId: entry.id,
name: entry.name,
isDefault: entry.id === defaultAgentId,
heartbeat: expectDefined(heartbeats[index], "heartbeat summary"),
sessions: projectHealthSessions(store.path, store)
};
});
}
function buildPluginHealthSummary(cfg) {
function projectError(plugin, error) {
return {
...error,
activationSource: plugin?.activationSource,
activationReason: plugin?.activationReason,
error: truncateUtf16Safe(redactToolPayloadTextWithConfig(error.error, cfg.logging), 1e3)
};
}
const registry = getActivePluginRegistry();
const degradedPlugins = listActiveDegradedPlugins();
const unavailable = degradedPlugins.map(({ pluginId, state, diagnostic }) => ({
id: pluginId,
state,
diagnostic: toPublicPluginVerificationDiagnostic(diagnostic)
})).toSorted((left, right) => left.id.localeCompare(right.id));
const loaded = (registry?.plugins ?? []).filter((plugin) => plugin.status === "loaded").map((plugin) => plugin.id).toSorted((left, right) => left.localeCompare(right));
const loadErrors = (registry?.plugins ?? []).filter((plugin) => plugin.status === "error" && !degradedPlugins.some((degraded) => plugin.id === degraded.pluginId && plugin.failurePhase === "validation" && plugin.activationReason === `configured-unavailable: ${degraded.diagnostic.reason}` && Boolean(plugin.rootDir) && degradedPluginMatchesRoot(degraded, plugin.rootDir ?? ""))).map((plugin) => projectError(plugin, {
id: plugin.id,
origin: plugin.origin,
activated: plugin.activated === true,
error: plugin.error ?? "unknown plugin load error",
...plugin.failurePhase ? { failurePhase: plugin.failurePhase } : {}
}));
const serviceErrors = registry ? listPluginServiceHealthFailures(registry).map((failure) => projectError(registry.plugins.find((entry) => entry.id === failure.pluginId), {
id: failure.pluginId,
origin: failure.origin,
activated: true,
failurePhase: "service",
error: `service ${failure.serviceId}: ${failure.error}`
})) : [];
const errors = [...loadErrors, ...serviceErrors].toSorted((left, right) => left.id.localeCompare(right.id) || left.error.localeCompare(right.error));
if (loaded.length === 0 && errors.length === 0 && unavailable.length === 0) return;
return {
loaded,
errors,
unavailable
};
}
let activeHealthOperations = 0;
const healthOperationWaiters = [];
function settleHealthOperationWaiter(waiter, release) {
if (waiter.settled) return;
waiter.settled = true;
clearTimeout(waiter.timer);
waiter.resolve(release);
}
function releaseNextHealthOperationWaiter() {
while (healthOperationWaiters.length > 0) {
const waiter = healthOperationWaiters.shift();
if (!waiter || waiter.settled) continue;
if (Date.now() >= waiter.deadlineAtMs) {
settleHealthOperationWaiter(waiter, null);
continue;
}
settleHealthOperationWaiter(waiter, createHealthOperationRelease());
return;
}
}
function createHealthOperationRelease() {
activeHealthOperations += 1;
let released = false;
return () => {
if (released) return;
released = true;
activeHealthOperations -= 1;
releaseNextHealthOperationWaiter();
};
}
async function acquireHealthOperationPermit(deadlineAtMs) {
if (Date.now() >= deadlineAtMs) return null;
if (activeHealthOperations < HEALTH_PROBE_CONCURRENCY) return createHealthOperationRelease();
return await new Promise((resolve) => {
const waiter = {
deadlineAtMs,
resolve,
timer: setTimeout(() => {
const index = healthOperationWaiters.indexOf(waiter);
if (index >= 0) healthOperationWaiters.splice(index, 1);
settleHealthOperationWaiter(waiter, null);
}, Math.max(1, deadlineAtMs - Date.now())),
settled: false
};
if (typeof waiter.timer === "object" && "unref" in waiter.timer) waiter.timer.unref();
healthOperationWaiters.push(waiter);
});
}
function buildHealthTimeoutRecord(accountId, timeoutMs) {
const error = `health collection timed out after ${timeoutMs}ms`;
return {
accountId,
lastError: error,
probe: {
ok: false,
timedOut: true,
error
}
};
}
function resolveHealthProbeTimeoutMs(deadlineAtMs) {
return Math.max(1, deadlineAtMs - Date.now());
}
async function buildHealthAccountRecord(params) {
const timedOut = () => buildHealthTimeoutRecord(params.accountId, params.timeoutMs);
const runtimeSnapshot = params.runtimeSnapshot?.channelAccounts[params.plugin.id]?.[params.accountId] ?? (params.accountId === params.defaultAccountId ? params.runtimeSnapshot?.channels[params.plugin.id] : void 0);
const unavailable = resolveUnavailableChannelAccountSnapshot(params.cfg, {
channelId: params.plugin.id,
accountId: params.accountId,
runtime: runtimeSnapshot
});
if (unavailable) return unavailable;
const { probeAccount, inspectedAccount, enabled, configured, diagnostics } = await resolveHealthAccountContext({
plugin: params.plugin,
cfg: params.cfg,
accountId: params.accountId
});
if (Date.now() >= params.deadlineAtMs) return timedOut();
if (diagnostics.length > 0) debugHealth(params.cfg, "account.diagnostics", {
channel: params.plugin.id,
accountId: params.accountId,
diagnostics
});
let probe;
let lastProbeAt = null;
if (probeAccount !== void 0 && enabled && configured === true && params.probe && params.plugin.status?.probeAccount) try {
probe = await params.plugin.status.probeAccount({
account: probeAccount,
timeoutMs: resolveHealthProbeTimeoutMs(params.deadlineAtMs),
cfg: params.cfg
});
lastProbeAt = Date.now();
} catch (error) {
probe = {
ok: false,
error: formatErrorMessage(error)
};
lastProbeAt = Date.now();
}
if (Date.now() >= params.deadlineAtMs) return timedOut();
const probeRecord = probe && typeof probe === "object" ? probe : null;
const bot = probeRecord && typeof probeRecord.bot === "object" ? probeRecord.bot : null;
if (bot?.username) debugHealth(params.cfg, "probe.bot", {
channel: params.plugin.id,
accountId: params.accountId,
username: bot.username
});
const nonSensitiveProbeFailure = buildNonSensitiveProbeFailure(params.plugin.id, probe);
const snapshotProbe = params.includeSensitive ? probe : nonSensitiveProbeFailure;
const snapshot = probeAccount === void 0 ? buildChannelAccountSnapshotFromInspection({
account: inspectedAccount,
accountId: params.accountId,
runtime: runtimeSnapshot,
probe: snapshotProbe
}) : await buildChannelAccountSnapshotFromAccount({
plugin: params.plugin,
cfg: params.cfg,
accountId: params.accountId,
account: probeAccount,
runtime: runtimeSnapshot,
probe: snapshotProbe,
enabledFallback: enabled,
configuredFallback: configured
});
if (Date.now() >= params.deadlineAtMs) return timedOut();
if (lastProbeAt) snapshot.lastProbeAt = lastProbeAt;
const healthState = resolveChannelHealthState(snapshot, {
channelId: params.plugin.id,
now: Date.now(),
staleEventThresholdMs: DEFAULT_CHANNEL_STALE_EVENT_THRESHOLD_MS,
channelConnectGraceMs: DEFAULT_CHANNEL_CONNECT_GRACE_MS
});
if (healthState !== void 0) snapshot.healthState = healthState;
const summary = probeAccount !== void 0 && params.plugin.status?.buildChannelSummary ? await params.plugin.status.buildChannelSummary({
account: probeAccount,
cfg: params.cfg,
defaultAccountId: params.accountId,
snapshot
}) : void 0;
if (Date.now() >= params.deadlineAtMs) return timedOut();
const record = redactChannelStatusSummaryBaseUrl(summary && typeof summary === "object" ? {
...snapshot,
...summary
} : {
...snapshot,
accountId: params.accountId
});
if (record.configured === void 0 && probeAccount !== void 0) record.configured = configured;
if (params.includeSensitive && record.probe === void 0 && probe !== void 0) record.probe = probe;
if (!params.includeSensitive) {
const safeProbeFailure = buildNonSensitiveProbeFailure(params.plugin.id, record.probe) ?? nonSensitiveProbeFailure;
if (safeProbeFailure) record.probe = safeProbeFailure;
else delete record.probe;
}
if (record.lastProbeAt === void 0 && lastProbeAt) record.lastProbeAt = lastProbeAt;
record.accountId = params.accountId;
return record;
}
async function runHealthAccountWithinDeadline(params) {
const operation = trackAsyncWork(async () => {
const release = await acquireHealthOperationPermit(params.deadlineAtMs);
if (!release) return buildHealthTimeoutRecord(params.accountId, params.timeoutMs);
try {
return await buildHealthAccountRecord(params);
} finally {
release();
}
});
const result = await awaitWithinDeadline(() => operation, params.deadlineAtMs);
return result === ABSOLUTE_DEADLINE_EXPIRED ? buildHealthTimeoutRecord(params.accountId, params.timeoutMs) : result;
}
/** Collects the gateway-owned health snapshot for an explicit trust audience. */
async function collectGatewayHealthSnapshot(params) {
const start = Date.now();
const timeoutMs = Math.min(resolveTimerTimeoutMs(params.timeoutMs, HEALTH_COLLECTION_TIMEOUT_MS, 50), HEALTH_COLLECTION_TIMEOUT_MS);
const deadlineAtMs = start + timeoutMs;
const cfg = await readRuntimeHealthConfig();
const { defaultAgentId, ordered } = resolveHealthAgentOrder(cfg);
const channelBindings = buildChannelAccountBindings(cfg);
const agents = await buildHealthAgentSummaries(cfg, {
defaultAgentId,
ordered
});
const summaryAgent = agents.find((agent) => agent.isDefault) ?? agents[0];
const configuredHeartbeatAgentId = normalizeOptionalString(cfg.agents?.defaults?.heartbeat?.agentId);
const heartbeatSummaryAgent = (configuredHeartbeatAgentId ? agents.find((agent) => agent.heartbeat.enabled && agent.agentId === normalizeAgentId(configuredHeartbeatAgentId)) : void 0) ?? agents.find((agent) => agent.heartbeat.enabled) ?? summaryAgent;
const heartbeatSeconds = heartbeatSummaryAgent?.heartbeat.everyMs ? Math.round(heartbeatSummaryAgent.heartbeat.everyMs / 1e3) : 0;
const sessions = summaryAgent?.sessions ?? await buildHealthSessionSummary(resolveSessionStorePathCore(cfg.session?.store, { agentId: summaryAgent?.agentId }), summaryAgent?.agentId);
const includeSensitive = params.audience === "admin";
const channels = {};
const plugins = listReadOnlyChannelPluginsForConfig(cfg, {
includeSetupFallbackPlugins: false,
includePersistedAuthState: false
});
const channelOrder = plugins.map((plugin) => plugin.id);
const channelLabels = {};
const channelPlans = plugins.map((plugin) => {
channelLabels[plugin.id] = plugin.meta.label ?? plugin.id;
const accountIds = plugin.config.listAccountIds(cfg);
const defaultAccountId = resolveChannelDefaultAccountId({
plugin,
cfg,
accountIds
});
const boundAccounts = defaultAgentId ? channelBindings.get(plugin.id)?.get(defaultAgentId) ?? [] : [];
const preferredAccountId = resolvePreferredAccountId({
accountIds,
defaultAccountId,
boundAccounts
});
const boundAccountIdsAll = Array.from(new Set(Array.from(channelBindings.get(plugin.id)?.values() ?? []).flat()));
const accountIdsToProbe = Array.from(new Set([
preferredAccountId,
defaultAccountId,
...accountIds,
...boundAccountIdsAll
].filter((value) => value && value.trim())));
debugHealth(cfg, "channel", {
id: plugin.id,
accountIds,
defaultAccountId,
boundAccounts,
preferredAccountId,
accountIdsToProbe
});
return {
plugin,
defaultAccountId,
preferredAccountId,
accountIds: accountIdsToProbe,
accountSummaries: {}
};
});
const accountTasks = channelPlans.flatMap((plan) => plan.accountIds.map((accountId) => ({
plan,
accountId
})));
const { results: accountResults } = await runTasksWithConcurrency({
tasks: accountTasks.map(({ plan, accountId }) => async () => ({
plan,
accountId,
record: await runHealthAccountWithinDeadline({
plugin: plan.plugin,
cfg,
accountId,
defaultAccountId: plan.defaultAccountId,
includeSensitive,
probe: params.probe,
deadlineAtMs,
timeoutMs,
runtimeSnapshot: params.runtimeSnapshot
})
})),
limit: params.probe ? HEALTH_PROBE_CONCURRENCY : 1,
throwOnError: true
});
for (const result of accountResults) if (result) result.plan.accountSummaries[result.accountId] = result.record;
for (const plan of channelPlans) {
const fallbackSummary = plan.accountSummaries[plan.preferredAccountId] ?? plan.accountSummaries[plan.defaultAccountId] ?? plan.accountSummaries[plan.accountIds[0] ?? plan.preferredAccountId] ?? plan.accountSummaries[expectDefined(Object.keys(plan.accountSummaries)[0], "object.keys(account summaries) entry at 0")];
if (fallbackSummary) channels[plan.plugin.id] = {
...fallbackSummary,
accounts: plan.accountSummaries
};
}
const pluginHealth = buildPluginHealthSummary(cfg);
const contextEngineHealth = buildContextEngineHealthSummary();
const deliveryQueueHealth = buildDeliveryQueueHealthSummary();
return {
ok: true,
ts: Date.now(),
durationMs: Date.now() - start,
...params.eventLoop ? { eventLoop: params.eventLoop } : {},
...pluginHealth ? { plugins: pluginHealth } : {},
...contextEngineHealth ? { contextEngines: contextEngineHealth } : {},
...deliveryQueueHealth ? { deliveryQueues: deliveryQueueHealth } : {},
...params.configReloadHotReloadStatus ? { configReload: { hotReloadStatus: params.configReloadHotReloadStatus } } : {},
channels,
channelOrder,
channelLabels,
heartbeatSeconds,
...defaultAgentId ? { defaultAgentId } : {},
agents,
sessions: {
path: sessions.path,
count: sessions.count,
recent: sessions.recent
}
};
}
async function readRuntimeHealthConfig() {
const { getRuntimeConfig } = await import("./config/config.js");
return getRuntimeConfig();
}
//#endregion
export { resolveHealthAccountContext as i, collectGatewayHealthSnapshot as n, resolveHealthAgentOrder as r, buildHealthAgentSummaries as t };