openclaw
Version:
Multi-channel AI gateway with extensible messaging integrations
758 lines (757 loc) • 33.5 kB
JavaScript
import { t as isTruthyEnvValue } from "./env-Di49gJwo.js";
import { i as formatErrorMessage } from "./errors-BXgSefBE.js";
import { n as loadInstalledPluginIndexInstallRecordsSync, t as loadInstalledPluginIndexInstallRecords } from "./installed-plugin-index-record-reader-CPHayOZI.js";
import { i as getRuntimeConfig } from "./io-Gi7-pyU-.js";
import "./installed-plugin-index-records-DXbAzXDd.js";
import { n as formatConfigIssueLines } from "./issue-format-CwCoGNbt.js";
import { h as resolveConfigWriteFollowUp } from "./runtime-snapshot-D93_HOsR.js";
import "./config-C9RxTsn1.js";
import { n as isRestartEnabled } from "./commands.flags-Bx5KKfOp.js";
import { f as resolveGatewayRestartDeferralTimeoutMs, m as setGatewaySigusr1RestartPolicy, o as deferGatewayRestartUntilIdle, s as emitGatewayRestart } from "./restart-CiO1ILZU.js";
import { a as resetModelCatalogCache } from "./model-catalog-0-HKr2yW.js";
import { r as clearCurrentProviderAuthState } from "./model-provider-auth-state-DW_JYm-o.js";
import { a as warmCurrentProviderAuthStateOffMainThread } from "./model-provider-auth-DawbDRrd.js";
import { d as listActiveEmbeddedRunSessionIds, f as listActiveEmbeddedRunSessionKeys, u as getActiveEmbeddedRunCount } from "./run-state-BYGFyjne.js";
import { u as getTotalQueueSize } from "./command-queue-DGvXlE4p.js";
import { n as getInspectableActiveTaskRestartBlockers } from "./task-registry.maintenance-BO6_16yZ.js";
import { r as disposeAllSessionMcpRuntimes } from "./agent-bundle-mcp-runtime-Bf36JLIh.js";
import "./agent-bundle-mcp-tools-tGjdbeEf.js";
import { n as clearSecretsRuntimeSnapshot, o as getActiveSecretsRuntimeSnapshot } from "./runtime-state-B3MH_XLp.js";
import { r as resetDirectoryCache } from "./target-resolver-CUW6ke8W.js";
import { t as bumpSkillsSnapshotVersion } from "./refresh-state-DzUL33ps.js";
import { t as getTotalPendingReplies } from "./dispatcher-registry-CaTZukRA.js";
import { v as resolveHooksConfig } from "./hooks-BuwUpRxl.js";
import { n as markGatewayModelCatalogStaleForReload } from "./server-model-catalog-CcvpyvTp.js";
import { a as diffConfigPaths, n as listPluginInstallTimestampMetadataPaths, r as listPluginInstallWholeRecordPaths, t as buildGatewayReloadPlan } from "./config-reload-plan-D2TNMpG_.js";
import { t as resolveGatewayReloadSettings } from "./config-reload-settings-BUK2a-7x.js";
import { a as setCurrentSharedGatewaySessionGeneration, n as disconnectStaleSharedGatewayAuthClients } from "./server-shared-auth-generation-kMfFmuwl.js";
import { t as buildGatewayCronService } from "./server-cron-BUhoP9it.js";
import { n as applyGatewayLaneConcurrency, t as resolveHookClientIpConfig } from "./hook-client-ip-config-CWB_cWpU.js";
import { t as startGatewayChannelHealthMonitor } from "./server-runtime-startup-services-B_m8NSaC.js";
import { i as startGatewayCronWithLogging } from "./server-runtime-services-Blpvr6WF.js";
import chokidar from "chokidar";
//#region src/gateway/config-reload.ts
const MISSING_CONFIG_RETRY_DELAY_MS = 150;
const MISSING_CONFIG_MAX_RETRIES = 2;
const WATCHER_RECREATE_MAX_RETRIES = 3;
const WATCHER_RECREATE_BACKOFF_MS = [
500,
2e3,
5e3
];
/**
* Paths under `skills.*` always change the snapshot that sessions cache in
* sessions.json. Any prefix match here (for example `skills.allowBundled`,
* `skills.entries.X.enabled`, `skills.profile`) forces sessions to rebuild
* their snapshot on the next turn rather than silently advertising stale
* tools to the model.
*/
const SKILLS_INVALIDATION_PREFIXES = ["skills"];
function matchesSkillsInvalidationPrefix(path) {
return SKILLS_INVALIDATION_PREFIXES.some((prefix) => path === prefix || path.startsWith(`${prefix}.`));
}
function firstSkillsChangedPath(changedPaths) {
return changedPaths.find(matchesSkillsInvalidationPrefix);
}
function isNoopReloadPlan(plan) {
return !plan.restartGateway && plan.hotReasons.length === 0 && !plan.reloadHooks && !plan.restartGmailWatcher && !plan.restartCron && !plan.restartHeartbeat && !plan.restartHealthMonitor && !plan.reloadPlugins && !plan.disposeMcpRuntimes && plan.restartChannels.size === 0;
}
function asPluginInstallConfig(records) {
return { plugins: { installs: records } };
}
function startGatewayConfigReloader(opts) {
let currentConfig = opts.initialConfig;
let currentCompareConfig = opts.initialCompareConfig ?? opts.initialConfig;
let settings = resolveGatewayReloadSettings(currentConfig);
let debounceTimer = null;
let pending = false;
let running = false;
let stopped = false;
let restartQueued = false;
let missingConfigRetries = 0;
let pendingInProcessConfig = null;
let lastAppliedWriteHash = opts.initialInternalWriteHash ?? null;
let currentPluginInstallRecords = opts.initialPluginInstallRecords ?? loadInstalledPluginIndexInstallRecordsSync();
const readPluginInstallRecords = opts.readPluginInstallRecords ?? loadInstalledPluginIndexInstallRecords;
const scheduleAfter = (wait) => {
if (stopped) return;
if (debounceTimer) clearTimeout(debounceTimer);
debounceTimer = setTimeout(() => {
runReload();
}, wait);
};
const schedule = () => {
scheduleAfter(settings.debounceMs);
};
const queueRestart = (plan, nextConfig) => {
if (restartQueued) return;
restartQueued = true;
(async () => {
try {
await opts.onRestart(plan, nextConfig);
} catch (err) {
restartQueued = false;
opts.log.error(`config restart failed: ${String(err)}`);
}
})();
};
const handleMissingSnapshot = (snapshot) => {
if (snapshot.exists) {
missingConfigRetries = 0;
return false;
}
if (missingConfigRetries < MISSING_CONFIG_MAX_RETRIES) {
missingConfigRetries += 1;
opts.log.info(`config reload retry (${missingConfigRetries}/${MISSING_CONFIG_MAX_RETRIES}): config file not found`);
scheduleAfter(MISSING_CONFIG_RETRY_DELAY_MS);
return true;
}
opts.log.warn("config reload skipped (config file not found)");
return true;
};
const handleInvalidSnapshot = (snapshot) => {
if (snapshot.valid) return false;
const issues = formatConfigIssueLines(snapshot.issues, "").join(", ");
opts.log.warn(`config reload skipped (invalid config): ${issues}`);
return true;
};
const applySnapshot = async (nextConfig, nextCompareConfig, afterWrite) => {
const configChangedPaths = diffConfigPaths(currentCompareConfig, nextCompareConfig);
const configPluginInstallTimestampNoopPaths = listPluginInstallTimestampMetadataPaths(currentCompareConfig, nextCompareConfig);
const configPluginInstallWholeRecordPaths = listPluginInstallWholeRecordPaths(currentCompareConfig, nextCompareConfig);
let nextPluginInstallRecords = currentPluginInstallRecords;
try {
nextPluginInstallRecords = await readPluginInstallRecords();
} catch (err) {
opts.log.warn(`config reload plugin install record check failed: ${String(err)}`);
}
const previousPluginInstallConfig = asPluginInstallConfig(currentPluginInstallRecords);
const nextPluginInstallConfig = asPluginInstallConfig(nextPluginInstallRecords);
const pluginInstallRecordChangedPaths = diffConfigPaths(previousPluginInstallConfig, nextPluginInstallConfig);
const pluginInstallRecordTimestampNoopPaths = listPluginInstallTimestampMetadataPaths(previousPluginInstallConfig, nextPluginInstallConfig);
const pluginInstallRecordWholeRecordPaths = listPluginInstallWholeRecordPaths(previousPluginInstallConfig, nextPluginInstallConfig);
const changedPaths = [...configChangedPaths, ...pluginInstallRecordChangedPaths];
const pluginInstallTimestampNoopPaths = [...configPluginInstallTimestampNoopPaths, ...pluginInstallRecordTimestampNoopPaths];
const pluginInstallWholeRecordPaths = [...configPluginInstallWholeRecordPaths, ...pluginInstallRecordWholeRecordPaths];
currentConfig = nextConfig;
currentCompareConfig = nextCompareConfig;
currentPluginInstallRecords = nextPluginInstallRecords;
settings = resolveGatewayReloadSettings(nextConfig);
if (changedPaths.length === 0) return;
const skillsChangedPath = firstSkillsChangedPath(changedPaths);
if (skillsChangedPath !== void 0) {
bumpSkillsSnapshotVersion({
reason: "config-change",
changedPath: skillsChangedPath
});
opts.log.info(`skills snapshot invalidated by config change (${skillsChangedPath})`);
}
const followUp = resolveConfigWriteFollowUp(afterWrite);
opts.log.info(`config change detected; evaluating reload (${changedPaths.join(", ")})`);
if (followUp.mode === "none") {
opts.log.info(`config reload skipped by writer intent (${followUp.reason})`);
return;
}
const plan = buildGatewayReloadPlan(changedPaths, {
noopPaths: pluginInstallTimestampNoopPaths,
forceChangedPaths: pluginInstallWholeRecordPaths
});
if (isNoopReloadPlan(plan) && !followUp.requiresRestart) return;
if (settings.mode === "off") {
opts.log.info("config reload disabled (gateway.reload.mode=off)");
return;
}
if (followUp.requiresRestart) {
queueRestart({
...plan,
restartGateway: true,
restartReasons: [...plan.restartReasons, followUp.reason]
}, nextConfig);
return;
}
if (settings.mode === "restart") {
queueRestart(plan, nextConfig);
return;
}
if (plan.restartGateway) {
if (settings.mode === "hot") {
opts.log.warn(`config reload requires gateway restart; hot mode ignoring (${plan.restartReasons.join(", ")})`);
return;
}
queueRestart(plan, nextConfig);
return;
}
await opts.onHotReload(plan, nextConfig);
};
const promoteAcceptedSnapshot = async (snapshot, reason) => {
if (!opts.promoteSnapshot || !snapshot.exists || !snapshot.valid) return;
try {
await opts.promoteSnapshot(snapshot, reason);
} catch (err) {
opts.log.warn(`config reload last-known-good promotion failed: ${String(err)}`);
}
};
const promoteAcceptedInProcessWrite = async (persistedHash) => {
if (!opts.promoteSnapshot) return;
try {
const snapshot = await opts.readSnapshot();
if (snapshot.hash !== persistedHash || !snapshot.valid) return;
await promoteAcceptedSnapshot(snapshot, "in-process-write");
} catch (err) {
opts.log.warn(`config reload in-process last-known-good promotion failed: ${String(err)}`);
}
};
const runReload = async () => {
if (stopped) return;
if (running) {
pending = true;
return;
}
running = true;
if (debounceTimer) {
clearTimeout(debounceTimer);
debounceTimer = null;
}
try {
if (pendingInProcessConfig) {
const pendingWrite = pendingInProcessConfig;
pendingInProcessConfig = null;
missingConfigRetries = 0;
await applySnapshot(pendingWrite.config, pendingWrite.compareConfig, pendingWrite.afterWrite);
await promoteAcceptedInProcessWrite(pendingWrite.persistedHash);
return;
}
const snapshot = await opts.readSnapshot();
if (lastAppliedWriteHash && typeof snapshot.hash === "string") {
if (snapshot.hash === lastAppliedWriteHash) return;
lastAppliedWriteHash = null;
}
if (handleMissingSnapshot(snapshot)) return;
if (!snapshot.valid) {
handleInvalidSnapshot(snapshot);
return;
}
await applySnapshot(snapshot.config, snapshot.sourceConfig);
await promoteAcceptedSnapshot(snapshot, "valid-config");
} catch (err) {
opts.log.error(`config reload failed: ${String(err)}`);
} finally {
running = false;
if (pending) {
pending = false;
schedule();
}
}
};
const scheduleFromWatcher = () => {
schedule();
};
const unsubscribeFromWrites = opts.subscribeToWrites?.((event) => {
if (event.configPath !== opts.watchPath) return;
pendingInProcessConfig = {
config: event.runtimeConfig,
compareConfig: event.sourceConfig,
persistedHash: event.persistedHash,
afterWrite: event.afterWrite
};
lastAppliedWriteHash = event.persistedHash;
scheduleAfter(0);
}) ?? (() => {});
let watcher = null;
let watcherRecreateRetries = 0;
let watcherRecreateTimer = null;
let hotReloadStatus = "active";
const createWatcher = () => {
if (stopped) return;
const next = chokidar.watch(opts.watchPath, {
ignoreInitial: true,
awaitWriteFinish: {
stabilityThreshold: 200,
pollInterval: 50
},
usePolling: Boolean(process.env.VITEST)
});
next.on("add", scheduleFromWatcher);
next.on("change", scheduleFromWatcher);
next.on("unlink", scheduleFromWatcher);
next.on("error", (err) => {
handleWatcherError(next, err);
});
watcher = next;
hotReloadStatus = "active";
};
const handleWatcherError = (source, err) => {
if (stopped || source !== watcher) return;
watcher = null;
source?.close().catch(() => {});
if (watcherRecreateRetries >= WATCHER_RECREATE_MAX_RETRIES) {
hotReloadStatus = "disabled";
opts.log.error(`config hot-reload disabled: watcher failed after ${WATCHER_RECREATE_MAX_RETRIES} re-create attempts: ${String(err)}`);
return;
}
const backoff = WATCHER_RECREATE_BACKOFF_MS[watcherRecreateRetries] ?? WATCHER_RECREATE_BACKOFF_MS[WATCHER_RECREATE_BACKOFF_MS.length - 1] ?? 0;
watcherRecreateRetries += 1;
opts.log.warn(`config watcher error; re-creating watcher (attempt ${watcherRecreateRetries}/${WATCHER_RECREATE_MAX_RETRIES} in ${backoff}ms): ${String(err)}`);
watcherRecreateTimer = setTimeout(() => {
watcherRecreateTimer = null;
createWatcher();
}, backoff);
};
createWatcher();
return {
stop: async () => {
stopped = true;
if (debounceTimer) clearTimeout(debounceTimer);
debounceTimer = null;
if (watcherRecreateTimer) {
clearTimeout(watcherRecreateTimer);
watcherRecreateTimer = null;
}
unsubscribeFromWrites();
const active = watcher;
watcher = null;
await active?.close().catch(() => {});
},
hotReloadStatus: () => hotReloadStatus
};
}
//#endregion
//#region src/gateway/server-reload-handlers.ts
async function activateSecretsRuntimeSnapshot(snapshot) {
(await import("./runtime-BzZlvYH7.js")).activateSecretsRuntimeSnapshot(snapshot);
}
const MCP_RUNTIME_RELOAD_DISPOSE_TIMEOUT_MS = 5e3;
const CHANNEL_RELOAD_DEFERRAL_POLL_MS = 500;
const CHANNEL_RELOAD_STILL_PENDING_WARN_MS = 3e4;
function resetPreparedModelRuntimeStateForHotReload() {
resetModelCatalogCache();
clearCurrentProviderAuthState();
markGatewayModelCatalogStaleForReload();
}
async function disposeMcpRuntimesWithTimeout(params) {
let timer;
const disposePromise = params.dispose().catch((error) => {
params.onWarn(`${params.label} failed: ${String(error)}`);
});
const timeoutPromise = new Promise((resolve) => {
timer = setTimeout(() => resolve("timeout"), params.timeoutMs);
timer.unref?.();
});
const result = await Promise.race([disposePromise.then(() => "done"), timeoutPromise]);
if (timer) clearTimeout(timer);
if (result === "timeout") params.onWarn(`${params.label} exceeded ${params.timeoutMs}ms; continuing`);
}
async function collectChannelOperationFailures(params) {
const failures = [];
for (const channel of params.channels) try {
await params.run(channel);
} catch (err) {
failures.push(channel);
params.onFailure(channel, err);
}
return failures;
}
function createGatewayReloadHandlers(params) {
const getActiveCounts = () => {
const queueSize = getTotalQueueSize();
const pendingReplies = getTotalPendingReplies();
const embeddedRuns = getActiveEmbeddedRunCount();
const activeTasks = getInspectableActiveTaskRestartBlockers().length;
return {
queueSize,
pendingReplies,
embeddedRuns,
activeTasks,
totalActive: queueSize + pendingReplies + embeddedRuns + activeTasks
};
};
const formatActiveDetails = (counts) => {
const details = [];
if (counts.queueSize > 0) details.push(`${counts.queueSize} operation(s)`);
if (counts.pendingReplies > 0) details.push(`${counts.pendingReplies} reply(ies)`);
if (counts.embeddedRuns > 0) details.push(`${counts.embeddedRuns} embedded run(s)`);
if (counts.activeTasks > 0) details.push(`${counts.activeTasks} background task run(s)`);
return details;
};
const formatTaskBlocker = (task) => {
return [
`taskId=${task.taskId}`,
task.runId ? `runId=${task.runId}` : null,
`status=${task.status}`,
`runtime=${task.runtime}`,
task.label ? `label=${task.label}` : null,
task.title ? `title=${task.title.slice(0, 80)}` : null
].filter((value) => Boolean(value)).join(" ");
};
const formatTaskBlockers = () => {
const blockers = getInspectableActiveTaskRestartBlockers();
if (blockers.length === 0) return null;
const shown = blockers.slice(0, 8).map(formatTaskBlocker);
const omitted = blockers.length - shown.length;
return omitted > 0 ? `${shown.join("; ")}; +${omitted} more` : shown.join("; ");
};
const collectActiveRestartSessionKeys = () => {
return new Set(listActiveEmbeddedRunSessionKeys());
};
const collectActiveRestartSessionIds = () => {
return new Set(listActiveEmbeddedRunSessionIds());
};
const markActiveMainSessionsForRestart = async (nextConfig, reason) => {
const sessionKeys = collectActiveRestartSessionKeys();
const sessionIds = collectActiveRestartSessionIds();
if (sessionKeys.size === 0 && sessionIds.size === 0) return;
const { markRestartAbortedMainSessions } = await import("./main-session-restart-recovery-FE3cfva8.js");
await markRestartAbortedMainSessions({
cfg: nextConfig,
additionalCfgs: [getRuntimeConfig()],
sessionKeys,
sessionIds,
reason
});
};
const waitForActiveWorkBeforeChannelReload = async (channels, nextConfig) => {
const initial = getActiveCounts();
if (initial.totalActive <= 0) return;
const channelNames = [...channels].join(", ");
const initialDetails = formatActiveDetails(initial);
params.logReload.warn(`config change requires channel reload (${channelNames}) — deferring until ${initialDetails.join(", ")} complete`);
const timeoutMs = resolveGatewayRestartDeferralTimeoutMs(nextConfig.gateway?.reload?.deferralTimeoutMs);
const startedAt = Date.now();
let nextStillPendingAt = startedAt + CHANNEL_RELOAD_STILL_PENDING_WARN_MS;
while (true) {
await new Promise((resolve) => {
setTimeout(resolve, CHANNEL_RELOAD_DEFERRAL_POLL_MS).unref?.();
});
const current = getActiveCounts();
if (current.totalActive <= 0) {
params.logReload.info("active operations and replies completed; reloading channels now");
return;
}
const elapsedMs = Date.now() - startedAt;
if (timeoutMs !== void 0 && elapsedMs >= timeoutMs) {
const remaining = formatActiveDetails(current);
params.logReload.warn(`channel reload timeout after ${elapsedMs}ms with ${remaining.join(", ")} still active; reloading channels anyway`);
return;
}
if (Date.now() >= nextStillPendingAt) {
const remaining = formatActiveDetails(current);
params.logReload.warn(`channel reload still deferred after ${elapsedMs}ms with ${remaining.join(", ")} active`);
nextStillPendingAt = Date.now() + CHANNEL_RELOAD_STILL_PENDING_WARN_MS;
}
}
};
const applyHotReload = async (plan, nextConfig) => {
setGatewaySigusr1RestartPolicy({ allowExternal: isRestartEnabled(nextConfig) });
const state = params.getState();
const nextState = { ...state };
resetPreparedModelRuntimeStateForHotReload();
if (plan.reloadHooks) try {
nextState.hooksConfig = resolveHooksConfig(nextConfig);
} catch (err) {
params.logHooks.warn(`hooks config reload failed: ${String(err)}`);
}
nextState.hookClientIpConfig = resolveHookClientIpConfig(nextConfig);
if (plan.restartHeartbeat) nextState.heartbeatRunner.updateConfig(nextConfig);
resetDirectoryCache();
const channelsToRestart = new Set(plan.restartChannels);
const channelsStoppedBeforePluginReload = /* @__PURE__ */ new Set();
let activePluginChannelsAfterReload = null;
const shouldSkipChannelRestart = () => isTruthyEnvValue(process.env.OPENCLAW_SKIP_CHANNELS) || isTruthyEnvValue(process.env.OPENCLAW_SKIP_PROVIDERS);
if (plan.reloadPlugins) {
const stopChannelsBeforePluginReplace = async (channels) => {
for (const channel of channels) channelsToRestart.add(channel);
if (channelsToRestart.size === 0 || shouldSkipChannelRestart()) return;
await waitForActiveWorkBeforeChannelReload(channelsToRestart, nextConfig);
const stoppedChannels = [];
const stopFailures = await collectChannelOperationFailures({
channels: channelsToRestart,
run: async (channel) => {
if (channelsStoppedBeforePluginReload.has(channel)) return;
params.logChannels.info(`stopping ${channel} channel before plugin reload`);
stoppedChannels.push(channel);
await params.stopChannel(channel, void 0, { manual: false });
channelsStoppedBeforePluginReload.add(channel);
},
onFailure: (channel, err) => {
params.logChannels.error(`failed to stop ${channel} channel before plugin reload: ${formatErrorMessage(err)}`);
}
});
if (stopFailures.length > 0) {
const rollbackFailures = await collectChannelOperationFailures({
channels: stoppedChannels,
run: async (channel) => {
params.logChannels.info(`restarting ${channel} channel after failed plugin reload pre-stop`);
await params.startChannel(channel);
channelsStoppedBeforePluginReload.delete(channel);
},
onFailure: (channel, err) => {
params.logChannels.error(`failed to restart ${channel} channel after failed plugin reload pre-stop: ${formatErrorMessage(err)}`);
}
});
const rollbackSuffix = rollbackFailures.length > 0 ? `; rollback restart failed for: ${rollbackFailures.join(", ")}` : "";
throw new Error(`failed to stop channels before plugin reload: ${stopFailures.join(", ")}${rollbackSuffix}`);
}
};
const pluginReloadResult = await params.reloadPlugins({
nextConfig,
changedPaths: plan.changedPaths,
beforeReplace: stopChannelsBeforePluginReplace
});
for (const channel of pluginReloadResult.restartChannels) channelsToRestart.add(channel);
activePluginChannelsAfterReload = pluginReloadResult.activeChannels;
resetPreparedModelRuntimeStateForHotReload();
}
if (plan.restartCron) {
params.onCronRestart?.();
state.cronState.cron.stop();
nextState.cronState = buildGatewayCronService({
cfg: nextConfig,
deps: params.deps,
broadcast: params.broadcast
});
startGatewayCronWithLogging({
cron: nextState.cronState.cron,
logCron: params.logCron
});
}
if (plan.restartHealthMonitor) {
state.channelHealthMonitor?.stop();
nextState.channelHealthMonitor = params.createHealthMonitor(nextConfig);
}
if (plan.disposeMcpRuntimes) await disposeMcpRuntimesWithTimeout({
dispose: disposeAllSessionMcpRuntimes,
timeoutMs: MCP_RUNTIME_RELOAD_DISPOSE_TIMEOUT_MS,
onWarn: params.logReload.warn,
label: "bundle-mcp runtime disposal during config reload"
});
if (plan.restartGmailWatcher) {
await params.stopPostReadySidecars?.();
const restartAbortController = params.createGmailRestartAbortController?.() ?? new AbortController();
try {
if (!restartAbortController.signal.aborted) {
const [{ stopGmailWatcher }, { startGmailWatcherWithLogs }] = await Promise.all([import("./gmail-watcher-B2RJfulT.js"), import("./gmail-watcher-lifecycle-Dkg6iU7k.js")]);
if (!restartAbortController.signal.aborted) await stopGmailWatcher().catch((err) => {
params.logHooks.warn(`gmail watcher stop failed during reload: ${String(err)}`);
});
if (!restartAbortController.signal.aborted) await startGmailWatcherWithLogs({
cfg: nextConfig,
log: params.logHooks,
isCancelled: () => restartAbortController.signal.aborted,
signal: restartAbortController.signal,
onSkipped: () => params.logHooks.info("skipping gmail watcher restart (OPENCLAW_SKIP_GMAIL_WATCHER=1)")
});
}
} finally {
params.clearGmailRestartAbortController?.(restartAbortController);
}
}
if (channelsToRestart.size > 0) if (shouldSkipChannelRestart()) params.logChannels.info("skipping channel reload (OPENCLAW_SKIP_CHANNELS=1 or OPENCLAW_SKIP_PROVIDERS=1)");
else {
if (!plan.reloadPlugins) await waitForActiveWorkBeforeChannelReload(channelsToRestart, nextConfig);
const restartChannel = async (name) => {
if (plan.reloadPlugins && activePluginChannelsAfterReload?.has(name) === false) return;
params.logChannels.info(`restarting ${name} channel`);
if (!channelsStoppedBeforePluginReload.has(name)) await params.stopChannel(name, void 0, { manual: false });
await params.startChannel(name);
};
const restartFailures = await collectChannelOperationFailures({
channels: channelsToRestart,
run: restartChannel,
onFailure: (channel, err) => {
params.logChannels.error(`failed to restart ${channel} channel during hot reload: ${formatErrorMessage(err)}`);
}
});
if (restartFailures.length > 0) throw new Error(`failed to restart channels during hot reload: ${restartFailures.join(", ")}`);
}
applyGatewayLaneConcurrency(nextConfig);
warmCurrentProviderAuthStateOffMainThread(nextConfig).catch((err) => {
params.logReload.warn(`provider auth state rewarm failed: ${String(err)}`);
});
if (plan.hotReasons.length > 0) params.logReload.info(`config hot reload applied (${plan.hotReasons.join(", ")})`);
else if (plan.noopPaths.length > 0) params.logReload.info(`config change applied (dynamic reads: ${plan.noopPaths.join(", ")})`);
params.setState(nextState);
};
let restartPending = false;
const requestGatewayRestart = (plan, nextConfig) => {
setGatewaySigusr1RestartPolicy({ allowExternal: isRestartEnabled(nextConfig) });
const reasons = plan.restartReasons.length ? plan.restartReasons.join(", ") : plan.changedPaths.join(", ");
if (process.listenerCount("SIGUSR1") === 0) {
params.logReload.warn("no SIGUSR1 listener found; restart skipped");
return false;
}
const active = getActiveCounts();
if (active.totalActive > 0) {
if (restartPending) {
params.logReload.info(`config change requires gateway restart (${reasons}) — already waiting for operations to complete`);
return true;
}
restartPending = true;
const initialDetails = formatActiveDetails(active);
params.logReload.warn(`config change requires gateway restart (${reasons}) — deferring until ${initialDetails.join(", ")} complete`);
const taskBlockers = formatTaskBlockers();
if (taskBlockers) params.logReload.warn(`restart blocked by active background task run(s): ${taskBlockers}`);
deferGatewayRestartUntilIdle({
getPendingCount: () => getActiveCounts().totalActive,
maxWaitMs: resolveGatewayRestartDeferralTimeoutMs(nextConfig.gateway?.reload?.deferralTimeoutMs),
timeoutIntent: {
force: true,
reason: "config reload forced restart"
},
emitHooks: { beforeEmit: () => markActiveMainSessionsForRestart(nextConfig, "config reload forced restart") },
hooks: {
onReady: () => {
restartPending = false;
params.logReload.info("all operations and replies completed; restarting gateway now");
},
onStillPending: (_pending, elapsedMs) => {
const remaining = formatActiveDetails(getActiveCounts());
const taskBlockersValue = formatTaskBlockers();
params.logReload.warn(`restart still deferred after ${elapsedMs}ms with ${remaining.join(", ")} active${taskBlockersValue ? ` (${taskBlockersValue})` : ""}`);
},
onTimeout: (_pending, elapsedMs) => {
const remaining = formatActiveDetails(getActiveCounts());
const taskBlockersLocal = formatTaskBlockers();
restartPending = false;
params.logReload.warn(`restart timeout after ${elapsedMs}ms with ${remaining.join(", ")} still active${taskBlockersLocal ? ` (${taskBlockersLocal})` : ""}; forcing restart`);
},
onCheckError: (err) => {
restartPending = false;
params.logReload.warn(`restart deferral check failed (${String(err)}); restarting gateway now`);
}
}
});
return true;
}
params.logReload.warn(`config change requires gateway restart (${reasons})`);
if (!emitGatewayRestart()) params.logReload.info("gateway restart already scheduled; skipping duplicate signal");
return true;
};
return {
applyHotReload,
requestGatewayRestart
};
}
function startManagedGatewayConfigReloader(params) {
if (params.minimalTestGateway) return { stop: async () => {} };
let stopped = false;
let activeGmailRestartAbortController = null;
const abortActiveGmailRestart = () => {
activeGmailRestartAbortController?.abort();
activeGmailRestartAbortController = null;
};
const createGmailRestartAbortController = () => {
abortActiveGmailRestart();
const abortController = new AbortController();
if (stopped) {
abortController.abort();
return abortController;
}
activeGmailRestartAbortController = abortController;
return abortController;
};
const { applyHotReload, requestGatewayRestart } = createGatewayReloadHandlers({
deps: params.deps,
broadcast: params.broadcast,
getState: params.getState,
setState: params.setState,
startChannel: params.startChannel,
stopChannel: params.stopChannel,
stopPostReadySidecars: params.stopPostReadySidecars,
reloadPlugins: params.reloadPlugins,
logHooks: params.logHooks,
logChannels: params.logChannels,
logCron: params.logCron,
logReload: params.logReload,
createGmailRestartAbortController,
clearGmailRestartAbortController: (abortController) => {
if (activeGmailRestartAbortController === abortController) activeGmailRestartAbortController = null;
},
...params.onCronRestart ? { onCronRestart: params.onCronRestart } : {},
createHealthMonitor: (config) => startGatewayChannelHealthMonitor({
cfg: config,
channelManager: params.channelManager
})
});
const configReloader = startGatewayConfigReloader({
initialConfig: params.initialConfig,
initialCompareConfig: params.initialCompareConfig,
initialInternalWriteHash: params.initialInternalWriteHash,
readSnapshot: params.readSnapshot,
promoteSnapshot: async (snapshot, _reason) => await params.promoteSnapshot(snapshot),
subscribeToWrites: params.subscribeToWrites,
onHotReload: async (plan, nextConfig) => {
const previousSharedGatewaySessionGeneration = params.sharedGatewaySessionGenerationState.current;
const previousSnapshot = getActiveSecretsRuntimeSnapshot();
const prepared = await params.activateRuntimeSecrets(nextConfig, {
reason: "reload",
activate: true
});
const nextSharedGatewaySessionGeneration = params.resolveSharedGatewaySessionGenerationForConfig(prepared.config);
params.sharedGatewaySessionGenerationState.current = nextSharedGatewaySessionGeneration;
const sharedGatewaySessionGenerationChanged = previousSharedGatewaySessionGeneration !== nextSharedGatewaySessionGeneration;
if (sharedGatewaySessionGenerationChanged) disconnectStaleSharedGatewayAuthClients({
clients: params.clients,
expectedGeneration: nextSharedGatewaySessionGeneration
});
try {
await applyHotReload(plan, prepared.config);
} catch (err) {
if (previousSnapshot) await activateSecretsRuntimeSnapshot(previousSnapshot);
else clearSecretsRuntimeSnapshot();
params.sharedGatewaySessionGenerationState.current = previousSharedGatewaySessionGeneration;
if (sharedGatewaySessionGenerationChanged) disconnectStaleSharedGatewayAuthClients({
clients: params.clients,
expectedGeneration: previousSharedGatewaySessionGeneration
});
throw err;
}
setCurrentSharedGatewaySessionGeneration(params.sharedGatewaySessionGenerationState, nextSharedGatewaySessionGeneration);
},
onRestart: async (plan, nextConfig) => {
const previousRequiredSharedGatewaySessionGeneration = params.sharedGatewaySessionGenerationState.required;
const previousSharedGatewaySessionGeneration = params.sharedGatewaySessionGenerationState.current;
try {
const prepared = await params.activateRuntimeSecrets(nextConfig, {
reason: "restart-check",
activate: false
});
const nextSharedGatewaySessionGeneration = params.resolveSharedGatewaySessionGenerationForConfig(prepared.config);
if (!requestGatewayRestart(plan, nextConfig)) {
if (previousSharedGatewaySessionGeneration !== nextSharedGatewaySessionGeneration) {
await activateSecretsRuntimeSnapshot(prepared);
setCurrentSharedGatewaySessionGeneration(params.sharedGatewaySessionGenerationState, nextSharedGatewaySessionGeneration);
params.sharedGatewaySessionGenerationState.required = null;
disconnectStaleSharedGatewayAuthClients({
clients: params.clients,
expectedGeneration: nextSharedGatewaySessionGeneration
});
} else params.sharedGatewaySessionGenerationState.required = null;
return;
}
if (previousSharedGatewaySessionGeneration !== nextSharedGatewaySessionGeneration) {
params.sharedGatewaySessionGenerationState.required = nextSharedGatewaySessionGeneration;
disconnectStaleSharedGatewayAuthClients({
clients: params.clients,
expectedGeneration: nextSharedGatewaySessionGeneration
});
} else params.sharedGatewaySessionGenerationState.required = null;
} catch (error) {
params.sharedGatewaySessionGenerationState.required = previousRequiredSharedGatewaySessionGeneration;
throw error;
}
},
log: {
info: (msg) => params.logReload.info(msg),
warn: (msg) => params.logReload.warn(msg),
error: (msg) => params.logReload.error(msg)
},
watchPath: params.watchPath
});
return { stop: async () => {
stopped = true;
abortActiveGmailRestart();
await configReloader.stop();
} };
}
//#endregion
export { startManagedGatewayConfigReloader };