openclaw
Version:
Multi-channel AI gateway with extensible messaging integrations
514 lines (513 loc) • 22.7 kB
JavaScript
import { n as getRuntimeConfig } from "./io.runtime-B9iJRs3w.js";
import { i as allowsProcessHomeSessionScan } from "./paths-D2sRr1a_.js";
import { t as createSubsystemLogger } from "./subsystem-Dy2tqXOS.js";
import { r as getPluginRegistryState } from "./runtime-state-Ccxn1fO9.js";
import "./config-Cs0XXL3x.js";
import { b as runWithGatewayIndependentRootWorkAdmission, f as isGatewayWorkAdmissionClosed } from "./gateway-work-admission-R1IpuDim.js";
import "./session-accessor-YsytfDtG.js";
import { E as resolveSessionStorePathForScope, u as loadSessionEntryReadOnly } from "./session-accessor.sqlite-entry-CWk3jL7s.js";
import { f as recordSessionHumanDirectMessage, p as recordSessionStateEvent } from "./session-state-events-DNCKmH78.js";
import { i as updateSessionUpstreamLinkMarker, n as listWatchedSessionUpstreamLinks, r as readSessionUpstreamLink, t as deleteSessionUpstreamLink } from "./session-upstream-links-DJQuiStM.js";
import { u as isEmbeddedAgentRunActive } from "./runs-Cb42qain.js";
import { i as readRecentUserAssistantTextForSession } from "./transcript-Dv8-hmIo.js";
import { t as computeBackoffMs } from "./delivery-recovery.shared-C1ZTHSCT.js";
import { t as resolveSkillWorkshopConfig } from "./config-Cjp42tXL.js";
import { i as resolveHeartbeatIntervalMs, n as resolveHeartbeatAgents } from "./heartbeat-config-DYNUPVum.js";
import "./heartbeat-summary-DTnNVs3l.js";
import { c as startSessionDeliveryRuntime, o as schedulePendingSessionDeliveries } from "./subagent-completion-admission.store-ieYRlFt3.js";
import "./embedded-agent-D9hkv0XX.js";
import { n as runHeartbeatOnce, t as startHeartbeatRunner } from "./heartbeat-runner-BE2w39or.js";
import { n as runWithScheduledGatewayContext, t as fenceScheduledGatewayContextResolver } from "./scheduled-run-gateway-context-DXYV7Oco.js";
import { t as resolveGatewayPluginConfig } from "./runtime-plugin-config-CKZ9Aehs.js";
import { r as assertQueuedConversationDeliveryAttemptAuthorized } from "./conversation-route-ownership-CD7PPt5R.js";
import { t as createNoopHeartbeatRunner } from "./server-runtime-service-shared-iwns63ly.js";
import "./server-idle-task-sVCvNzia.js";
import "./server-runtime-startup-services-CrdkNCOa.js";
import { createHash } from "node:crypto";
//#region src/sessions/session-upstream-monitor.ts
/** Polls watched adopted sessions for direct upstream human activity. */
const SESSION_UPSTREAM_MONITOR_INTERVAL_MS = 6e4;
const SESSION_UPSTREAM_MONITOR_INITIAL_DELAY_MS = 15e3;
const SESSION_UPSTREAM_OWN_USER_TEXT_LIMIT = 10;
const SESSION_UPSTREAM_MISSING_THRESHOLD = 3;
const log = createSubsystemLogger("sessions/upstream-monitor");
function currentProviders() {
return (getPluginRegistryState()?.activeRegistry?.sessionCatalogs ?? []).map((registration) => registration.provider);
}
function databaseOptions(options) {
return {
...options.env ? { env: options.env } : {},
...options.path ? { path: options.path } : {}
};
}
function normalizeUserText(text) {
return text.trim().replace(/\s+/g, " ");
}
function upstreamSourceKey(probe) {
return createHash("sha256").update(`${probe.hostId}\u0000${probe.threadId}\u0000${JSON.stringify(probe.upstreamRef)}`).digest("hex").slice(0, 16);
}
function upstreamMonitorLinkKey(probe) {
return `${probe.sessionKey}\n${probe.agentId}\n${upstreamSourceKey(probe)}`;
}
function loadProbeSession(probe, options) {
if (options.signal?.aborted) return;
const entry = (options.loadEntry ?? loadSessionEntryReadOnly)({
sessionKey: probe.sessionKey,
agentId: probe.agentId,
clone: false,
...options.env ? { env: options.env } : {}
});
return entry?.sessionId ? entry : void 0;
}
function loadIdleProbeSession(probe, options, expectedSessionId) {
const entry = loadProbeSession(probe, options);
if (!entry || expectedSessionId !== void 0 && entry.sessionId !== expectedSessionId || (options.isRunActive ?? isEmbeddedAgentRunActive)(entry.sessionId)) return;
return entry;
}
function readMatchingProbeLink(probe, expectedUpdatedAt, options) {
const currentLink = readSessionUpstreamLink(probe.sessionKey, probe.agentId, options);
return currentLink && currentLink.updatedAt === expectedUpdatedAt && upstreamSourceKey(currentLink) === upstreamSourceKey(probe) ? currentLink : void 0;
}
async function loadOwnRecentUserTexts(probe, entry, options) {
if (options.loadOwnRecentUserTexts) return await options.loadOwnRecentUserTexts({
entry,
probe
});
const storePath = resolveSessionStorePathForScope({
agentId: probe.agentId,
sessionKey: probe.sessionKey,
...options.env ? { env: options.env } : {}
});
return (await readRecentUserAssistantTextForSession({
agentId: probe.agentId,
sessionKey: probe.sessionKey,
storePath,
limit: SESSION_UPSTREAM_OWN_USER_TEXT_LIMIT,
preferUpstreamUserText: true,
role: "user"
})).map((item) => normalizeUserText(item.text)).filter(Boolean);
}
async function probeProvenanceUnchanged(probe, expectedSessionId, options) {
const entry = loadIdleProbeSession(probe, options, expectedSessionId);
if (!entry) return false;
const current = await loadOwnRecentUserTexts(probe, entry, options);
if (!loadIdleProbeSession(probe, options, expectedSessionId)) return false;
return options.signal?.aborted !== true && current.length === probe.ownRecentUserTexts.length && current.every((text, index) => text === probe.ownRecentUserTexts[index]);
}
async function runSessionUpstreamMonitorTick(options = {}, missingCounts = /* @__PURE__ */ new Map()) {
if (options.signal?.aborted) return;
const dbOptions = databaseOptions(options);
const linksByCatalog = listWatchedSessionUpstreamLinks(dbOptions);
const watchedLinkKeys = new Set([...linksByCatalog.values()].flatMap((links) => links.map(upstreamMonitorLinkKey)));
for (const key of missingCounts.keys()) if (!watchedLinkKeys.has(key)) missingCounts.delete(key);
const providers = options.providers ?? currentProviders();
const providerById = new Map(providers.map((provider) => [provider.id, provider]));
for (const [catalogId, links] of linksByCatalog) {
const provider = providerById.get(catalogId);
if (!provider?.checkUpstreamActivity) continue;
const probes = [];
const sessionIdBySessionKey = /* @__PURE__ */ new Map();
for (const link of links) {
const probe = {
sessionKey: link.sessionKey,
agentId: link.agentId,
threadId: link.threadId,
hostId: link.hostId,
upstreamKind: link.upstreamKind,
upstreamRef: link.upstreamRef,
marker: link.marker
};
try {
const entry = loadIdleProbeSession(probe, options);
if (!entry) continue;
const ownRecentUserTexts = await loadOwnRecentUserTexts(probe, entry, options);
if (options.signal?.aborted) return;
probes.push({
...probe,
ownRecentUserTexts
});
sessionIdBySessionKey.set(probe.sessionKey, entry.sessionId);
} catch (error) {
log.warn(`upstream transcript provenance failed for ${probe.sessionKey}: ${String(error)}`);
}
}
if (probes.length === 0) continue;
const probeBySessionKey = new Map(probes.map((probe) => [probe.sessionKey, probe]));
const linkUpdatedAtBySessionKey = new Map(links.map((link) => [link.sessionKey, link.updatedAt]));
try {
const outcomes = await provider.checkUpstreamActivity(probes, { allowProcessHomeFallback: allowsProcessHomeSessionScan(options.env ?? process.env) });
if (options.signal?.aborted) return;
const missingSessionKeys = new Set(outcomes.filter((outcome) => outcome.kind === "missing").map((outcome) => outcome.sessionKey));
for (const probe of probes) if (!missingSessionKeys.has(probe.sessionKey)) missingCounts.delete(upstreamMonitorLinkKey(probe));
for (const outcome of outcomes) {
const probe = probeBySessionKey.get(outcome.sessionKey);
if (!probe) continue;
const missingCountKey = upstreamMonitorLinkKey(probe);
if (outcome.kind === "missing") {
const expectedUpdatedAt = linkUpdatedAtBySessionKey.get(outcome.sessionKey);
const expectedSessionId = sessionIdBySessionKey.get(outcome.sessionKey);
if (expectedUpdatedAt === void 0 || expectedSessionId === void 0) {
missingCounts.delete(missingCountKey);
continue;
}
if (options.signal?.aborted) return;
const currentLink = readMatchingProbeLink(probe, expectedUpdatedAt, dbOptions);
if (!currentLink) {
missingCounts.delete(missingCountKey);
continue;
}
const currentSession = loadProbeSession(probe, options);
if (!currentSession || currentSession.sessionId !== expectedSessionId) {
missingCounts.delete(missingCountKey);
continue;
}
if ((options.isRunActive ?? isEmbeddedAgentRunActive)(currentSession.sessionId)) continue;
const previous = missingCounts.get(missingCountKey);
const missingCount = Math.min(SESSION_UPSTREAM_MISSING_THRESHOLD, (previous?.linkUpdatedAt === expectedUpdatedAt ? previous.count : 0) + 1);
missingCounts.set(missingCountKey, {
count: missingCount,
linkUpdatedAt: expectedUpdatedAt
});
if (missingCount < SESSION_UPSTREAM_MISSING_THRESHOLD) continue;
const sourceKey = upstreamSourceKey(probe);
if (!recordSessionStateEvent({
sessionKey: probe.sessionKey,
agentId: probe.agentId,
kind: "upstream_missing",
actorType: "system",
dedupeKey: `upstream-missing:${probe.sessionKey}:${sourceKey}:${currentLink.updatedAt}`,
summary: `upstream missing via ${catalogId}`,
payload: { channel: catalogId }
}, {
...dbOptions,
now: (options.now ?? Date.now)()
})) {
missingCounts.set(missingCountKey, {
count: 2,
linkUpdatedAt: expectedUpdatedAt
});
continue;
}
deleteSessionUpstreamLink(probe.sessionKey, probe.agentId, dbOptions);
missingCounts.delete(missingCountKey);
continue;
}
missingCounts.delete(missingCountKey);
const activity = outcome;
if (!Number.isSafeInteger(activity.humanTurns) || activity.humanTurns < 0) continue;
try {
if (!await probeProvenanceUnchanged(probe, sessionIdBySessionKey.get(probe.sessionKey), options)) continue;
} catch (error) {
log.warn(`upstream transcript provenance failed for ${probe.sessionKey}: ${String(error)}`);
continue;
}
const expectedUpdatedAt = linkUpdatedAtBySessionKey.get(activity.sessionKey);
if (!readMatchingProbeLink(probe, expectedUpdatedAt, dbOptions)) continue;
if (activity.humanTurns === 0) {
updateSessionUpstreamLinkMarker(probe.sessionKey, probe.agentId, activity.nextMarker, {
...dbOptions,
now: (options.now ?? Date.now)(),
...expectedUpdatedAt === void 0 ? {} : { expectedUpdatedAt }
});
continue;
}
if (!Number.isFinite(activity.occurredAt) || !activity.dedupeId) continue;
if (!recordSessionHumanDirectMessage({
sessionKey: probe.sessionKey,
agentId: probe.agentId,
actor: { actorType: "human" },
channel: catalogId,
dedupeKey: `upstream:${probe.sessionKey}:${upstreamSourceKey(probe)}:${activity.dedupeId}`,
...activity.humanTurns > 1 ? { payload: { turns: activity.humanTurns } } : {},
occurredAt: activity.occurredAt
}, {
...dbOptions,
now: (options.now ?? Date.now)()
})) continue;
updateSessionUpstreamLinkMarker(probe.sessionKey, probe.agentId, activity.nextMarker, {
...dbOptions,
now: (options.now ?? Date.now)(),
...expectedUpdatedAt === void 0 ? {} : { expectedUpdatedAt }
});
}
} catch (error) {
log.warn(`upstream activity probe failed for ${catalogId}: ${String(error)}`);
}
}
}
function startSessionUpstreamMonitor(options = {}) {
let stopped = false;
let running = false;
const lifecycle = new AbortController();
const tickOptions = {
...options,
signal: lifecycle.signal
};
const missingCounts = /* @__PURE__ */ new Map();
const run = () => {
if (stopped || running) return;
running = true;
runSessionUpstreamMonitorTick(tickOptions, missingCounts).catch((error) => {
log.warn(`upstream monitor tick failed: ${String(error)}`);
}).finally(() => {
running = false;
});
};
const initialTimer = setTimeout(run, SESSION_UPSTREAM_MONITOR_INITIAL_DELAY_MS);
initialTimer.unref?.();
const interval = setInterval(run, SESSION_UPSTREAM_MONITOR_INTERVAL_MS);
interval.unref?.();
return { stop: () => {
if (stopped) return;
stopped = true;
lifecycle.abort();
clearTimeout(initialTimer);
clearInterval(interval);
} };
}
if (process.env.VITEST || false) globalThis[Symbol.for("openclaw.sessionUpstreamMonitorTestApi")] = { runSessionUpstreamMonitorTick };
//#endregion
//#region src/gateway/server-runtime-services.ts
/** Starts cron without making the surrounding startup or reload transaction wait. */
function startGatewayCronWithLogging(params) {
const reconciliation = params.cronReconciliation.arm({
reason: params.reason,
config: params.config,
cronState: params.cronState
});
runWithGatewayIndependentRootWorkAdmission(async () => {
try {
await params.cronState.cron.start();
await params.afterStart?.();
await reconciliation.complete();
} catch (err) {
params.logCron.error(`failed to start: ${String(err)}`);
params.onStartError?.(err);
}
}, "runtime:cron-start").catch((err) => params.logCron.error(`failed to enter start root: ${String(err)}`));
}
async function clearGatewayMaintenanceHandles(maintenance) {
if (!maintenance) return;
clearInterval(maintenance.tickInterval);
clearInterval(maintenance.healthInterval);
clearInterval(maintenance.dedupeCleanup);
await maintenance.stopMediaCleanup();
clearInterval(maintenance.worktreeCleanup);
maintenance.skillUsageCleanup();
}
/** Schedules post-ready maintenance and cancels/cleans handles if shutdown wins the race. */
function scheduleGatewayPostReadyMaintenance(params) {
const timer = setTimeout(() => {
params.onStarted?.();
if (params.isClosing()) return;
runWithGatewayIndependentRootWorkAdmission(async () => {
try {
if (!params.isClosing()) {
const maintenance = await params.startMaintenance();
if (params.isClosing()) await clearGatewayMaintenanceHandles(maintenance);
else if (maintenance) await params.applyMaintenance(maintenance);
}
} catch (err) {
params.log.warn(`gateway post-ready maintenance startup failed: ${String(err)}`);
}
if (!params.isClosing() && params.shouldStartCron()) {
params.markCronStartHandled();
startGatewayCronWithLogging({
cronState: params.cronState,
cronReconciliation: params.cronReconciliation,
reason: "startup",
config: params.cronConfig,
logCron: params.logCron
});
}
if (!params.isClosing()) params.recordPostReadyMemory();
}, "runtime:maintenance").catch((err) => params.log.warn(`gateway post-ready maintenance deferred task failed: ${String(err)}`));
}, params.delayMs);
timer.unref?.();
return timer;
}
const RECOVERY_SHUTDOWN_STILL_PENDING_WARN_MS = 5e3;
function startPendingOutboundDeliveryRecovery(params) {
let stopped = false;
let migrationPending = true;
let initialPass = true;
let inFlight = null;
let stopPromise = null;
let logRecovery;
const recover = () => {
if (stopped || inFlight || isGatewayWorkAdmissionClosed()) return;
const settled = runWithGatewayIndependentRootWorkAdmission(async () => {
if (stopped) return;
const { drainPendingDeliveriesCore, recoverPendingDeliveries } = await import("./delivery-queue-recovery-BTVAVgJ0.js");
const { deliverOutboundPayloadsInternal } = await import("./deliver-D701DyDF.js");
if (stopped) return;
const deliverWithCurrentConversationAuthority = async (deliveryParams) => {
const completion = deliveryParams.deliveryCompletion;
const attemptAuthority = completion?.kind === "conversation" ? completion : deliveryParams.conversationDeliveryAttemptAuthority;
if (!attemptAuthority) return await deliverOutboundPayloadsInternal(deliveryParams);
return await deliverOutboundPayloadsInternal({
...deliveryParams,
onDeliveryAttempt: async () => {
await deliveryParams.onDeliveryAttempt?.();
if (!attemptAuthority.routeFingerprint) return;
assertQueuedConversationDeliveryAttemptAuthorized({
config: resolveGatewayPluginConfig({ config: getRuntimeConfig() }),
agentId: attemptAuthority.agentId,
operationId: attemptAuthority.operationId,
...attemptAuthority.storePath ? { storePath: attemptAuthority.storePath } : {},
routeFingerprint: attemptAuthority.routeFingerprint
});
}
});
};
logRecovery ??= params.log.child("delivery-recovery");
if (migrationPending) {
const cfg = initialPass ? params.cfg : getRuntimeConfig();
initialPass = false;
const { migrateLegacyPendingOutboundDeliveries } = await import("./delivery-queue-migration-y5gSReTM.js");
const migration = await migrateLegacyPendingOutboundDeliveries({
cfg,
log: logRecovery
});
migrationPending = migration.skipped > 0 || migration.remaining > 0;
await recoverPendingDeliveries({
deliver: deliverWithCurrentConversationAuthority,
log: logRecovery,
cfg,
shouldContinue: () => !stopped
});
return;
}
await drainPendingDeliveriesCore({
drainKey: "gateway:outbound",
logLabel: "Outbound delivery retry",
cfg: getRuntimeConfig(),
log: logRecovery,
deliver: deliverWithCurrentConversationAuthority,
selectEntry: () => ({
match: true,
bypassBackoff: false
}),
shouldContinue: () => !stopped
});
}, "runtime:delivery-recovery").catch((err) => params.log.error(`Delivery recovery failed: ${String(err)}`)).finally(() => {
if (inFlight === settled) inFlight = null;
});
inFlight = settled;
};
const retryTimer = setInterval(recover, computeBackoffMs(1));
retryTimer.unref?.();
recover();
return () => {
stopped = true;
clearInterval(retryTimer);
if (stopPromise) return stopPromise;
const recovery = inFlight;
if (!recovery) {
stopPromise = Promise.resolve();
return stopPromise;
}
const stillPendingTimer = setTimeout(() => {
(logRecovery ??= params.log.child("delivery-recovery")).warn(`delivery recovery is still pending after ${RECOVERY_SHUTDOWN_STILL_PENDING_WARN_MS}ms; waiting before runtime teardown`);
}, RECOVERY_SHUTDOWN_STILL_PENDING_WARN_MS);
stillPendingTimer.unref?.();
stopPromise = recovery.finally(() => {
clearTimeout(stillPendingTimer);
});
return stopPromise;
};
}
function startPendingSessionDeliveryRuntime(params) {
let stopped = false;
let recovery;
let stopPromise;
let stopRuntime;
const timer = setTimeout(() => {
recovery = runWithGatewayIndependentRootWorkAdmission(async () => {
const { deliverQueuedSessionDelivery, recoverPendingRestartContinuationDeliveries, settleQueuedSessionDelivery } = await import("./server-restart-sentinel-a6E8pkSs.js");
if (stopped) return;
const logRecovery = params.log.child("session-delivery-recovery");
stopRuntime = startSessionDeliveryRuntime({
deliver: (entry, context = {}) => deliverQueuedSessionDelivery({
deps: params.deps,
entry,
...context.stateDir !== void 0 ? { stateDir: context.stateDir } : {},
...params.resolveGatewayContext ? { resolveGatewayContext: params.resolveGatewayContext } : {}
}),
log: logRecovery,
onSettled: settleQueuedSessionDelivery
});
try {
await recoverPendingRestartContinuationDeliveries({
deps: params.deps,
log: logRecovery,
maxEnqueuedAt: params.maxEnqueuedAt,
...params.resolveGatewayContext ? { resolveGatewayContext: params.resolveGatewayContext } : {}
});
} finally {
if (!stopped) await schedulePendingSessionDeliveries();
}
}, "runtime:session-delivery-recovery").catch((err) => params.log.error(`Session delivery recovery failed: ${String(err)}`));
}, 1250);
timer.unref?.();
return () => {
stopped = true;
clearTimeout(timer);
stopPromise ??= Promise.all([recovery, stopRuntime?.()]).then(() => {});
return stopPromise;
};
}
/** Activates background gateway services after core runtime startup is ready. */
function activateGatewayScheduledServices(params) {
if (params.minimalTestGateway) return {
heartbeatRunner: createNoopHeartbeatRunner(),
stopDeliveryRecovery: async () => {}
};
if (!params.cronState.cronEnabled && resolveHeartbeatAgents(params.cfgAtStart).some((agent) => Boolean(resolveHeartbeatIntervalMs(params.cfgAtStart, void 0, agent.heartbeat)))) params.log.child("heartbeat").warn("scheduled heartbeats are disabled because the cron scheduler is disabled; enable cron and restart the gateway");
if (!params.cronState.cronEnabled && resolveSkillWorkshopConfig(params.cfgAtStart).autonomous.mode === "auto") params.log.child("skill-workshop").warn("scheduled skill collection reviews are disabled because the cron scheduler is disabled; enable cron and restart the gateway");
const heartbeatGatewayContextResolver = fenceScheduledGatewayContextResolver(params.resolveGatewayContext);
const heartbeatRunner = startHeartbeatRunner({
cfg: params.cfgAtStart,
readCurrentConfig: getRuntimeConfig,
...heartbeatGatewayContextResolver ? { runOnce: async (opts) => await runWithScheduledGatewayContext({
resolveGatewayContext: heartbeatGatewayContextResolver,
run: async () => await runHeartbeatOnce(opts)
}) } : {}
});
const sessionUpstreamMonitor = startSessionUpstreamMonitor();
const stopSessionDeliveryRuntime = startPendingSessionDeliveryRuntime({
deps: params.deps,
log: params.log,
maxEnqueuedAt: params.sessionDeliveryRecoveryMaxEnqueuedAt,
...params.resolveGatewayContext ? { resolveGatewayContext: params.resolveGatewayContext } : {}
});
if (params.startCron !== false) startGatewayCronWithLogging({
cronState: params.cronState,
cronReconciliation: params.cronReconciliation,
reason: "startup",
config: params.cfgAtStart,
logCron: params.logCron
});
const stopOutboundDeliveryRecovery = startPendingOutboundDeliveryRecovery({
cfg: params.cfgAtStart,
log: params.log
});
let deliveryRecoveryStopPromise;
const stopDeliveryRecovery = () => {
deliveryRecoveryStopPromise ??= Promise.all([stopOutboundDeliveryRecovery(), stopSessionDeliveryRuntime()]).then(() => {});
return deliveryRecoveryStopPromise;
};
return {
heartbeatRunner: {
updateConfig: heartbeatRunner.updateConfig,
stop: () => {
stopDeliveryRecovery();
sessionUpstreamMonitor.stop();
heartbeatRunner.stop();
}
},
stopDeliveryRecovery
};
}
//#endregion
export { startGatewayCronWithLogging as i, clearGatewayMaintenanceHandles as n, scheduleGatewayPostReadyMaintenance as r, activateGatewayScheduledServices as t };