openclaw
Version:
Multi-channel AI gateway with extensible messaging integrations
1,144 lines (1,143 loc) • 46.4 kB
JavaScript
import { s as sleepWithAbort, t as RetrySupervisor } from "./src-BQ327IOM.js";
import "./session-key-BnWWjqNc.js";
import { n as normalizeAccountId, r as normalizeOptionalAccountId } from "./account-id-CETVCrTz.js";
import { t as formatErrorMessage } from "./errors-Db3Ymjlb.js";
import { r as runtimeForLogger, t as createSubsystemLogger } from "./subsystem-Dy2tqXOS.js";
import { t as createDeferredCore } from "./deferred-D0La5CRk.js";
import { t as runTasksWithConcurrency } from "./run-with-concurrency-Dtu208ef.js";
import { a as listLoadedChannelPluginsForRegistry, n as getLoadedChannelPluginEntryById } from "./registry-loaded-Bh7xuMJh.js";
import { f as withPluginRuntimeRegistryScope } from "./gateway-request-scope-BCMYlsDI.js";
import { y as runOutsideGatewayRootWorkAdmission } from "./gateway-work-admission-R1IpuDim.js";
import { n as resolveNormalizedAccountEntry, t as resolveAccountEntry } from "./account-lookup-CvFhSs3G.js";
import "./backoff-BkMI1WEL.js";
import { g as setActiveCredentialDegradedOwner, o as clearActiveCredentialDegradedOwner, r as assertSecretOwnerAvailable } from "./runtime-degraded-state-D5EZZ925.js";
import { n as resolveChannelApprovalCapability } from "./plugins-Cozj1Enf.js";
import { r as resetDirectoryCache } from "./target-resolver-q_MBPTOw.js";
import { t as isAccountEnabled } from "./account-enabled-ClTLgAXM.js";
import { i as resolveChannelDefaultAccountId } from "./helpers-VcK_WJ24.js";
import { t as getCredentialUnavailableDiagnostics } from "./account-snapshot-fields-a22gRqLh.js";
import { a as resolveUnavailableChannelAccountSnapshot, i as resolveChannelAccountState, o as isChannelAccountExplicitlyDisabled, t as applyChannelAccountState } from "./account-state-CPvd6WGN.js";
import { t as buildChannelAccountSnapshotFromInspection } from "./account-summary-oKRoWiin.js";
import { n as withGatewayNativeApprovalRuntime } from "./approval-gateway-runtime-context-BSAo6TQe.js";
import { t as CHANNEL_APPROVAL_NATIVE_RUNTIME_CONTEXT_CAPABILITY } from "./approval-handler-adapter-runtime-Tv9LYgST.js";
import { a as isExecApprovalChannelRuntimeTerminalStartError } from "./approval-native-runtime-DVAlcchD.js";
import { n as createChannelApprovalHandlerFromCapability } from "./approval-handler-runtime-BZs1zst-.js";
import { i as watchChannelRuntimeContexts, n as getChannelRuntimeContext, r as registerChannelRuntimeContext, t as createTaskScopedChannelRuntime } from "./channel-runtime-context-gztTEkoq.js";
import { n as isChannelIngressUnavailableError } from "./ingress-unavailable-CiovoHDb.js";
import { i as withPluginHttpRouteRegistry, n as createPluginHttpRouteHandoff } from "./http-registry-BDq8iS_8.js";
import { i as formatGatewayCrashLoopManualChannelStartHint } from "./gateway-boot-lifecycle-JblQ4jbl.js";
import { t as createPluginRuntimeCapabilityLease } from "./capability-lease-Dybi4lS4.js";
//#region src/infra/approval-handler-bootstrap.ts
const APPROVAL_HANDLER_BOOTSTRAP_RETRY_MS = 1e3;
function isRetryableApprovalBootstrapStartError(error) {
const message = String(error);
return message.includes("gateway readiness unavailable before approval client start") || message.includes("gateway approval client start aborted before readiness") || message.includes("gateway readiness unavailable before exec approval runtime start") || message.includes("gateway approval runtime start aborted before readiness") || message.includes("gateway event loop readiness timeout") || message.includes("gateway starting") || message.includes("code=1013") || message.includes("close code 1013");
}
function formatRetryableApprovalBootstrapStartError(error) {
const message = String(error);
if (message.includes("gateway event loop readiness timeout")) return "gateway readiness unavailable before approval handler start";
return message;
}
/** Starts the native approval handler for a channel runtime context and returns its cleanup hook. */
async function startChannelApprovalHandlerBootstrap(params) {
const capability = resolveChannelApprovalCapability(params.plugin);
if (!capability?.nativeRuntime || !params.channelRuntime) return async () => {};
const channelLabel = params.plugin.meta.label || params.plugin.id;
const logger = params.logger ?? createSubsystemLogger(`${params.plugin.id}/approval-bootstrap`);
let activeGeneration = 0;
let activeHandler = null;
let retryTimer = null;
const invalidateActiveHandler = () => {
activeGeneration += 1;
};
const clearRetryTimer = () => {
if (!retryTimer) return;
clearTimeout(retryTimer);
retryTimer = null;
};
const stopHandler = async () => {
const handler = activeHandler;
activeHandler = null;
if (!handler) return;
await handler.stop();
};
const startHandlerForContext = async (context, generation) => {
if (generation !== activeGeneration) return;
await stopHandler();
if (generation !== activeGeneration) return;
const handler = await withGatewayNativeApprovalRuntime(params.gatewayRuntime, () => createChannelApprovalHandlerFromCapability({
capability,
label: `${params.plugin.id}/native-approvals`,
clientDisplayName: `${channelLabel} Native Approvals (${params.accountId})`,
channel: params.plugin.id,
channelLabel,
cfg: params.cfg,
accountId: params.accountId,
context
}));
if (!handler) return;
if (generation !== activeGeneration) {
await handler.stop().catch(() => {});
return;
}
activeHandler = handler;
try {
await withGatewayNativeApprovalRuntime(params.gatewayRuntime, () => handler.start());
} catch (error) {
if (activeHandler === handler) activeHandler = null;
await handler.stop().catch(() => {});
throw error;
}
};
const spawn = (label, promise) => {
promise.catch((error) => {
logger.error(`${label}: ${String(error)}`);
});
};
const scheduleRetryForContext = (context, generation) => {
if (generation !== activeGeneration) return;
clearRetryTimer();
retryTimer = setTimeout(() => {
retryTimer = null;
if (generation !== activeGeneration) return;
spawn("failed to retry native approval handler", startHandlerForRegisteredContext(context, generation));
}, APPROVAL_HANDLER_BOOTSTRAP_RETRY_MS);
retryTimer.unref?.();
};
const startHandlerForRegisteredContext = async (context, generation) => {
try {
await startHandlerForContext(context, generation);
} catch (error) {
if (generation === activeGeneration) {
if (isExecApprovalChannelRuntimeTerminalStartError(error)) {
logger.error(`native approval handler disabled: ${String(error)}`);
return;
}
if (isRetryableApprovalBootstrapStartError(error)) {
logger.warn(`native approval handler deferred until gateway readiness recovers: ${formatRetryableApprovalBootstrapStartError(error)}`);
scheduleRetryForContext(context, generation);
return;
}
logger.error(`failed to start native approval handler: ${String(error)}`);
scheduleRetryForContext(context, generation);
}
}
};
const unsubscribe = watchChannelRuntimeContexts({
channelRuntime: params.channelRuntime,
channelId: params.plugin.id,
accountId: params.accountId,
capability: "approval.native",
onEvent: (event) => {
if (event.type === "registered") {
clearRetryTimer();
invalidateActiveHandler();
const generation = activeGeneration;
spawn("failed to start native approval handler", startHandlerForRegisteredContext(event.context, generation));
return;
}
clearRetryTimer();
invalidateActiveHandler();
spawn("failed to stop native approval handler", stopHandler());
}
}) ?? (() => {});
const existingContext = getChannelRuntimeContext({
channelRuntime: params.channelRuntime,
channelId: params.plugin.id,
accountId: params.accountId,
capability: CHANNEL_APPROVAL_NATIVE_RUNTIME_CONTEXT_CAPABILITY
});
if (existingContext !== void 0) {
clearRetryTimer();
invalidateActiveHandler();
spawn("failed to start native approval handler", startHandlerForRegisteredContext(existingContext, activeGeneration));
}
return async () => {
unsubscribe();
clearRetryTimer();
invalidateActiveHandler();
await stopHandler();
};
}
//#endregion
//#region src/gateway/server-channels.ts
const RESTART_POLICY = {
initialMs: 5e3,
maxMs: 3e5,
factor: 2,
jitter: .1
};
const MAX_RESTARTS = 10;
const CHANNEL_STABLE_RUN_MS = RESTART_POLICY.maxMs;
const CHANNEL_STOP_ABORT_TIMEOUT_MS = 5e3;
const CHANNEL_STARTUP_CONCURRENCY = 4;
const CHANNEL_APPROVAL_GATEWAY_RUNTIME_CONTEXT_CAPABILITY = "approval.gateway";
function waitForChannelStartupHandoff() {
return new Promise((resolve) => {
setImmediate(resolve).unref?.();
});
}
function sanitizeAbortedTaskStatusPatch(patch, current) {
const next = { ...patch };
delete next.running;
delete next.restartPending;
delete next.reconnectAttempts;
delete next.lastStartAt;
delete next.lastStopAt;
delete next.lifecycle;
if (next.connected === true) {
delete next.connected;
delete next.lastConnectedAt;
delete next.lastEventAt;
delete next.lastTransportActivityAt;
}
if (next.lastError === null && current.lastError) delete next.lastError;
return next;
}
function createRuntimeStore() {
return {
lifetimes: /* @__PURE__ */ new Map(),
routeHandoffs: /* @__PURE__ */ new Map(),
starting: /* @__PURE__ */ new Map(),
stops: /* @__PURE__ */ new Map(),
tasks: /* @__PURE__ */ new Map(),
runtimes: /* @__PURE__ */ new Map()
};
}
async function waitForChannelStopGracefully(task, timeoutMs) {
if (!task) return true;
return await new Promise((resolve) => {
let settled = false;
const timer = setTimeout(() => {
if (!settled) {
settled = true;
resolve(false);
}
}, timeoutMs);
timer.unref?.();
const resolveSettled = () => {
if (settled) return;
settled = true;
clearTimeout(timer);
resolve(true);
};
task.then(resolveSettled, resolveSettled);
});
}
async function waitForDeferredAccountStart(deferred, abortSignal) {
if (abortSignal.aborted) return;
await Promise.race([deferred, new Promise((resolve) => {
abortSignal.addEventListener("abort", () => resolve(), { once: true });
})]);
}
function createChannelManager(opts) {
const { getRuntimeConfig, channelLogs, channelRuntimeEnvs, channelRuntime, resolveChannelRuntime, getPluginRegistry, startupTrace } = opts;
const withRegistry = (run) => {
const registry = getPluginRegistry();
return withPluginRuntimeRegistryScope(registry, () => run(registry));
};
const getChannelPlugin = (channelId) => getLoadedChannelPluginEntryById(channelId, getPluginRegistry())?.plugin;
const cloneDefaultRuntime = (channelId, accountId) => ({
...getChannelPlugin(channelId)?.status?.defaultRuntime,
accountId
});
const channelStores = /* @__PURE__ */ new Map();
let channelStartPause;
const restarts = /* @__PURE__ */ new Map();
const manuallyStopped = /* @__PURE__ */ new Set();
const recoveryStopTimedOut = /* @__PURE__ */ new Set();
const recoveryStartRequested = /* @__PURE__ */ new Set();
const pendingAutoRestarts = /* @__PURE__ */ new Set();
let autostartSuppression = null;
let ambientAutostartSuppressedChannelIds = new Set(opts.ambientAutostartSuppressedChannelIds ?? []);
const restartKey = (channelId, accountId) => `${channelId}:${accountId}`;
const releaseRouteHandoff = (store, accountId, expected = store.routeHandoffs.get(accountId)) => {
if (expected && store.routeHandoffs.get(accountId) === expected) {
expected.handoff.release();
store.routeHandoffs.delete(accountId);
}
};
const releaseChannelRouteHandoffs = (channelId, accountId) => {
const store = getStore(channelId);
for (const id of accountId ? [accountId] : store.routeHandoffs.keys()) {
const admittedSignal = store.routeHandoffs.get(id)?.admittedSignal;
if (!admittedSignal || admittedSignal.aborted) releaseRouteHandoff(store, id);
}
};
const ensureChannelLog = (channelId) => {
channelLogs[channelId] ??= createSubsystemLogger("channels").child(channelId);
return channelLogs[channelId];
};
const ensureChannelRuntime = (channelId) => {
channelRuntimeEnvs[channelId] ??= runtimeForLogger(ensureChannelLog(channelId));
return channelRuntimeEnvs[channelId];
};
const resolveAccountHealthMonitorOverride = (channelConfig, accountId) => {
if (!channelConfig?.accounts) return;
const direct = resolveAccountEntry(channelConfig.accounts, accountId);
if (typeof direct?.healthMonitor?.enabled === "boolean") return direct.healthMonitor.enabled;
const normalizedAccountId = normalizeOptionalAccountId(accountId);
if (!normalizedAccountId) return;
const match = resolveNormalizedAccountEntry(channelConfig.accounts, normalizedAccountId, normalizeAccountId);
if (typeof match?.healthMonitor?.enabled !== "boolean") return;
return match.healthMonitor.enabled;
};
const isHealthMonitorEnabled = (channelId, accountId) => {
const channelConfig = getRuntimeConfig().channels?.[channelId];
const accountOverride = resolveAccountHealthMonitorOverride(channelConfig, accountId);
const channelOverride = channelConfig?.healthMonitor?.enabled;
if (typeof accountOverride === "boolean") return accountOverride;
if (typeof channelOverride === "boolean") return channelOverride;
return true;
};
const getStore = (channelId) => {
const existing = channelStores.get(channelId);
if (existing) return existing;
const next = createRuntimeStore();
channelStores.set(channelId, next);
return next;
};
const getRuntime = (channelId, accountId) => {
return getStore(channelId).runtimes.get(accountId) ?? cloneDefaultRuntime(channelId, accountId);
};
const setRuntime = (channelId, accountId, patch) => {
const store = getStore(channelId);
const current = getRuntime(channelId, accountId);
const hasExplicitReadyRecovery = Object.hasOwn(patch, "lifecycle") && patch.lifecycle === "ready" && Object.hasOwn(patch, "terminalDisconnect") && patch.terminalDisconnect === void 0;
const lifecycle = current.lifecycle === "blocked" && current.terminalDisconnect === true && patch.lifecycle !== "starting" && !hasExplicitReadyRecovery ? "blocked" : patch.lifecycle ?? (patch.restartPending === true ? "recovering" : patch.connected === true ? "ready" : void 0);
const next = {
...current,
...patch,
...lifecycle ? { lifecycle } : {},
accountId
};
store.runtimes.set(accountId, next);
return next;
};
const setRuntimeFromTaskStatus = (channelId, accountId, patch, abortSignal) => {
const safePatch = abortSignal.aborted ? sanitizeAbortedTaskStatusPatch(patch, getRuntime(channelId, accountId)) : patch;
const next = setRuntime(channelId, accountId, safePatch);
if (!abortSignal.aborted && (next.lifecycle === "ready" || patch.terminalDisconnect === true)) releaseRouteHandoff(getStore(channelId), accountId);
return next;
};
const setStoppedRuntime = (channelId, accountId, patch = {}) => {
const current = getRuntime(channelId, accountId);
return setRuntime(channelId, accountId, {
accountId,
running: false,
lifecycle: patch.restartPending === true ? "recovering" : "stopped",
...typeof current.connected === "boolean" ? { connected: false } : {},
...patch
});
};
const getChannelRuntime = async () => {
if (channelRuntime) return channelRuntime;
return await resolveChannelRuntime?.();
};
const createAccountContext = (channelId, accountId, cfg, account, abortSignal) => ({
cfg,
accountId,
account,
abortSignal,
runtime: ensureChannelRuntime(channelId),
log: ensureChannelLog(channelId),
getStatus: () => getRuntime(channelId, accountId)
});
const measureStartup = async (name, run) => {
return startupTrace ? startupTrace.measure(name, run) : await run();
};
const evictStaleChannelAccountState = (channelId, store, accountIds) => {
const activeAccountIds = new Set(accountIds);
for (const id of store.routeHandoffs.keys()) if (!activeAccountIds.has(id)) releaseRouteHandoff(store, id);
for (const id of store.runtimes.keys()) {
if (activeAccountIds.has(id) || store.lifetimes.has(id) || store.starting.has(id) || store.stops.has(id) || store.tasks.has(id)) continue;
store.runtimes.delete(id);
clearActiveCredentialDegradedOwner("account", restartKey(channelId, normalizeAccountId(id)));
restarts.delete(restartKey(channelId, id));
manuallyStopped.delete(restartKey(channelId, id));
recoveryStartRequested.delete(restartKey(channelId, id));
}
};
const pruneInactiveChannelAccountState = (activeChannelIds) => {
for (const [channelId, store] of channelStores) if (!activeChannelIds.has(channelId)) evictStaleChannelAccountState(channelId, store, []);
};
const startChannelProcessOwned = async (registry, channelId, accountId, optsValue = {}) => {
const assertStartCurrent = () => {
if (channelStartPause || registry !== getPluginRegistry()) throw new Error("Channel plugins are reloading; retry the start after reload completes.");
};
assertStartCurrent();
const registration = getLoadedChannelPluginEntryById(channelId, registry);
const plugin = registration?.plugin;
const startAccount = plugin?.gateway?.startAccount;
if (!startAccount) {
const store = getStore(channelId);
for (const id of accountId ? [accountId] : store.routeHandoffs.keys()) releaseRouteHandoff(store, id);
return accountId ? /* @__PURE__ */ new Map([[accountId, {
status: "skipped",
reason: "unsupported"
}]]) : /* @__PURE__ */ new Map();
}
const { preserveRestartAttempts = false, preserveManualStop = false } = optsValue;
const cfg = getRuntimeConfig();
resetDirectoryCache({
cfg,
channel: channelId,
accountId
});
const store = getStore(channelId);
const accountIds = accountId ? [accountId] : await measureStartup(`channels.${channelId}.list-accounts`, () => plugin.config.listAccountIds(cfg));
assertStartCurrent();
if (!accountId) evictStaleChannelAccountState(channelId, store, accountIds);
if (accountIds.length === 0) return /* @__PURE__ */ new Map();
if (autostartSuppression && optsValue.manual !== true) {
const suffix = accountId ? ` account ${accountId}` : "";
ensureChannelLog(channelId).warn?.(`channel autostart suppressed by crash-loop breaker; refusing automatic start for ${channelId}${suffix}. ${formatGatewayCrashLoopManualChannelStartHint({
channelId,
...accountId ? { accountId } : {}
})}`);
for (const id of accountIds) {
releaseRouteHandoff(store, id);
setStoppedRuntime(channelId, id, {
restartPending: false,
lastError: autostartSuppression.message
});
}
return new Map(accountIds.map((id) => [id, {
status: "skipped",
reason: "autostart-suppressed"
}]));
}
if (ambientAutostartSuppressedChannelIds.has(channelId) && optsValue.manual !== true) {
for (const id of accountIds) {
releaseRouteHandoff(store, id);
setStoppedRuntime(channelId, id, {
restartPending: false,
lastError: "ambient channel credentials suppressed; configure the channel or start the gateway with --ambient-channels"
});
}
return new Map(accountIds.map((id) => [id, {
status: "skipped",
reason: "ambient-suppressed"
}]));
}
const startOutcomes = /* @__PURE__ */ new Map();
const startup = await runTasksWithConcurrency({
limit: CHANNEL_STARTUP_CONCURRENCY,
tasks: accountIds.map((id) => async () => {
assertStartCurrent();
const rKey = restartKey(channelId, id);
const explicitlyDisabled = isChannelAccountExplicitlyDisabled({
cfg,
channel: channelId,
accountId: id
});
if (explicitlyDisabled) releaseRouteHandoff(store, id);
if (!preserveManualStop && !store.stops.has(id)) manuallyStopped.delete(rKey);
for (;;) {
if (store.stops.has(id)) {
startOutcomes.set(id, {
status: "retry",
reason: "stop-in-flight"
});
return;
}
if (store.tasks.has(id)) {
let clearedTimedOutRecoveryTask = false;
if (recoveryStopTimedOut.has(rKey)) {
if (manuallyStopped.has(rKey)) {
startOutcomes.set(id, {
status: "skipped",
reason: "manual-stop"
});
return;
}
if (recoveryStartRequested.has(rKey)) {
recoveryStopTimedOut.delete(rKey);
recoveryStartRequested.delete(rKey);
restarts.delete(rKey);
store.lifetimes.get(id)?.capabilityLease.revoke();
store.lifetimes.delete(id);
store.tasks.delete(id);
clearedTimedOutRecoveryTask = true;
setRuntime(channelId, id, {
accountId: id,
restartPending: false,
reconnectAttempts: 0
});
} else {
recoveryStartRequested.add(rKey);
setRuntime(channelId, id, {
accountId: id,
restartPending: true
});
startOutcomes.set(id, {
status: "retry",
reason: "task-owned"
});
return;
}
}
if (!clearedTimedOutRecoveryTask) {
startOutcomes.set(id, {
status: "retry",
reason: "task-owned"
});
return;
}
}
const existingStart = store.starting.get(id);
if (!existingStart) break;
await existingStart;
assertStartCurrent();
}
const startGate = createDeferredCore();
store.starting.set(id, startGate.promise);
const routeHandoff = store.routeHandoffs.get(id);
const abort = new AbortController();
const capabilityLease = createPluginRuntimeCapabilityLease("channel account");
const lifetime = {
abort,
capabilityLease
};
store.lifetimes.set(id, lifetime);
let handedOffTask = false;
const log = ensureChannelLog(channelId);
let scopedChannelRuntime = null;
let channelRuntimeForTask;
let stopApprovalBootstrap = async () => {};
const stopTaskScopedApprovalRuntime = async () => {
const scopedRuntime = scopedChannelRuntime;
scopedChannelRuntime = null;
const stopBootstrap = stopApprovalBootstrap;
stopApprovalBootstrap = async () => {};
scopedRuntime?.dispose();
await stopBootstrap();
};
const cleanupTaskScopedApprovalRuntime = async (label) => {
try {
await stopTaskScopedApprovalRuntime();
} catch (error) {
log.error?.(`[${id}] ${label}: ${formatErrorMessage(error)}`);
}
};
const skipDisabledAccount = () => {
setRuntime(channelId, id, {
accountId: id,
enabled: false,
running: false,
restartPending: false
});
startOutcomes.set(id, {
status: "skipped",
reason: "disabled"
});
};
try {
const secretOwnerId = `${channelId}:${normalizeAccountId(id)}`;
clearActiveCredentialDegradedOwner("account", secretOwnerId);
if (explicitlyDisabled && plugin.config.listAccountIds(cfg).some((listed) => normalizeAccountId(listed) === normalizeAccountId(id))) {
skipDisabledAccount();
return;
}
try {
assertSecretOwnerAvailable("account", secretOwnerId);
} catch (error) {
if (!optsValue.skipUnavailableAccounts) throw error;
setStoppedRuntime(channelId, id, {
restartPending: false,
lastError: formatErrorMessage(error)
});
startOutcomes.set(id, {
status: "skipped",
reason: "secret-unavailable"
});
return;
}
const account = plugin.config.resolveAccount(cfg, id);
const accountContext = createAccountContext(channelId, id, cfg, account, abort.signal);
if (plugin.gateway?.stopAccount) lifetime.teardown = {
context: accountContext,
run: plugin.gateway.stopAccount.bind(plugin.gateway)
};
const described = plugin.config.describeAccount?.(account, cfg);
if (!(plugin.config.isEnabled ? plugin.config.isEnabled(account, cfg) : isAccountEnabled(account))) {
skipDisabledAccount();
return;
}
const credentialDiagnostics = getCredentialUnavailableDiagnostics(account);
if (credentialDiagnostics.length > 0) {
setActiveCredentialDegradedOwner({
ownerKind: "account",
ownerId: secretOwnerId,
state: "unavailable",
paths: credentialDiagnostics.map((diagnostic) => diagnostic.path),
refKeys: [],
reason: "credential file is unavailable"
});
assertSecretOwnerAvailable("account", secretOwnerId);
}
let configured = true;
if (plugin.config.isConfigured) configured = await measureStartup(`channels.${channelId}.is-configured`, () => plugin.config.isConfigured(account, cfg));
if (!configured) {
setRuntime(channelId, id, {
accountId: id,
enabled: true,
configured: false,
linked: void 0,
running: false,
restartPending: false
});
startOutcomes.set(id, {
status: "skipped",
reason: "unconfigured"
});
return;
}
setRuntime(channelId, id, {
accountId: id,
enabled: true,
configured: true,
...plugin.config.isLinked ? { linked: void 0 } : {}
});
const fallbackLinked = described?.linked ?? getRuntime(channelId, id).linked;
const linkState = plugin.config.isLinked ? await measureStartup(`channels.${channelId}.is-linked`, () => plugin.config.isLinked(account, cfg)) : fallbackLinked === true ? "linked" : fallbackLinked === false ? "not-linked" : void 0;
if (linkState === "not-linked" || linkState === "unknown") {
setRuntime(channelId, id, {
accountId: id,
enabled: true,
linked: linkState === "not-linked" ? false : void 0,
running: false,
restartPending: false
});
startOutcomes.set(id, {
status: "skipped",
reason: "unlinked"
});
return;
}
if (abort.signal.aborted || manuallyStopped.has(rKey)) {
setStoppedRuntime(channelId, id, {
restartPending: false,
lastStopAt: Date.now()
});
startOutcomes.set(id, {
status: "skipped",
reason: "manual-stop"
});
return;
}
scopedChannelRuntime = await measureStartup(`channels.${channelId}.runtime`, async () => createTaskScopedChannelRuntime({ channelRuntime: registration?.resolveChannelRuntime?.() ?? await getChannelRuntime() }));
channelRuntimeForTask = scopedChannelRuntime.channelRuntime;
if (!preserveRestartAttempts) restarts.delete(rKey);
try {
stopApprovalBootstrap = await measureStartup(`channels.${channelId}.approval-bootstrap`, () => startChannelApprovalHandlerBootstrap({
plugin,
cfg,
accountId: id,
channelRuntime: channelRuntimeForTask,
gatewayRuntime: opts.getNativeApprovalRuntime?.(),
logger: log
}));
} catch (error) {
log.error?.(`[${id}] native approval bootstrap failed: ${formatErrorMessage(error)}`);
}
assertStartCurrent();
if (abort.signal.aborted || manuallyStopped.has(rKey) || opts.isClosing?.()) {
startOutcomes.set(id, {
status: "skipped",
reason: "manual-stop"
});
return;
}
let channelRunDurationMs;
setRuntime(channelId, id, {
accountId: id,
enabled: true,
...linkState === "linked" ? { linked: true } : {},
running: true,
lifecycle: "starting",
restartPending: false,
lastStartAt: Date.now(),
lastError: null,
ingressUnavailable: void 0,
terminalDisconnect: void 0,
reconnectAttempts: preserveRestartAttempts ? restarts.get(rKey)?.attempts ?? 0 : 0
});
const trackedPromise = Promise.resolve().then(async () => {
if (optsValue.deferAccountStartUntil) await waitForDeferredAccountStart(optsValue.deferAccountStartUntil, abort.signal);
else if (startupTrace) await waitForChannelStartupHandoff();
if (abort.signal.aborted || manuallyStopped.has(rKey) || opts.isClosing?.()) return;
const gatewayApprovalRuntime = opts.getNativeApprovalRuntime?.();
if (channelRuntimeForTask && gatewayApprovalRuntime) registerChannelRuntimeContext({
channelRuntime: channelRuntimeForTask,
channelId,
accountId: id,
capability: CHANNEL_APPROVAL_GATEWAY_RUNTIME_CONTEXT_CAPABILITY,
context: { request: async (method, requestParams, requestOptions) => {
if (method !== "approval.resolve") throw new Error(`channel approval runtime cannot dispatch ${method}`);
return await gatewayApprovalRuntime.request("approval.resolve", requestParams, requestOptions);
} },
abortSignal: abort.signal
});
let startAccountTask;
await measureStartup(`channels.${channelId}.start-account-handoff`, () => {
if (abort.signal.aborted || manuallyStopped.has(rKey) || opts.isClosing?.()) return;
const runStartAccount = () => {
const startedAt = Date.now();
const recordDuration = () => {
channelRunDurationMs = Date.now() - startedAt;
};
try {
return withGatewayNativeApprovalRuntime(opts.getNativeApprovalRuntime?.(), () => startAccount({
...accountContext,
setStatus: (next) => isCurrentTask() ? setRuntimeFromTaskStatus(channelId, id, next, abort.signal) : getRuntime(channelId, id),
invalidateDirectoryCache: () => resetDirectoryCache({
cfg,
channel: channelId,
accountId: id
}),
...channelRuntimeForTask ? { channelRuntime: channelRuntimeForTask } : {}
})).finally(recordDuration);
} catch (error) {
recordDuration();
throw error;
}
};
startAccountTask = withPluginHttpRouteRegistry(registry, runStartAccount, capabilityLease);
});
if (!startAccountTask) return;
await startAccountTask;
}).finally(() => capabilityLease.revoke()).then(() => {
if (abort.signal.aborted || manuallyStopped.has(rKey) || opts.isClosing?.() || !isCurrentTask()) return;
if (getRuntime(channelId, id).terminalDisconnect) return;
const message = "channel exited without an error";
setRuntime(channelId, id, {
accountId: id,
lastError: message
});
log.error?.(`[${id}] ${message}`);
}).catch((err) => {
if (!isCurrentTask() || store.stops.has(id) || opts.isClosing?.()) return;
const message = formatErrorMessage(err);
setRuntime(channelId, id, {
accountId: id,
lastError: message,
...isChannelIngressUnavailableError(err) ? { ingressUnavailable: true } : {}
});
log.error?.(`[${id}] channel exited: ${message}`);
}).then(async () => {
await cleanupTaskScopedApprovalRuntime("channel cleanup failed");
if (!isCurrentTask() || store.stops.has(id) || opts.isClosing?.()) return;
setStoppedRuntime(channelId, id, { lastStopAt: Date.now() });
}).then(async () => {
if (!isCurrentTask() || store.stops.has(id) || opts.isClosing?.()) return;
if (manuallyStopped.has(rKey)) {
recoveryStopTimedOut.delete(rKey);
recoveryStartRequested.delete(rKey);
return;
}
if (getRuntime(channelId, id).terminalDisconnect) {
recoveryStopTimedOut.delete(rKey);
recoveryStartRequested.delete(rKey);
restarts.delete(rKey);
setRuntime(channelId, id, {
accountId: id,
restartPending: false,
reconnectAttempts: 0
});
log.info?.(`[${id}] auto-restart skipped, terminal disconnect`);
return;
}
if (recoveryStopTimedOut.has(rKey)) {
recoveryStopTimedOut.delete(rKey);
if (!recoveryStartRequested.delete(rKey)) {
setRuntime(channelId, id, {
accountId: id,
restartPending: false,
reconnectAttempts: 0
});
releaseTask();
return;
}
restarts.delete(rKey);
log.info?.(`[${id}] restarting after timed-out channel stop completed`);
setRuntime(channelId, id, {
accountId: id,
restartPending: true,
reconnectAttempts: 0
});
releaseTask();
try {
await startChannelInternal(channelId, id, { preserveManualStop: true });
} catch {}
return;
}
if (channelRunDurationMs !== void 0 && channelRunDurationMs >= CHANNEL_STABLE_RUN_MS) restarts.delete(rKey);
const restart = restarts.get(rKey) ?? new RetrySupervisor(RESTART_POLICY, MAX_RESTARTS);
restarts.set(rKey, restart);
const retry = restart.next(abort.signal);
if (!retry) {
setRuntime(channelId, id, {
accountId: id,
restartPending: false,
reconnectAttempts: restart.attempts
});
log.error?.(`[${id}] giving up after ${MAX_RESTARTS} restart attempts`);
return;
}
log.info?.(`[${id}] auto-restart attempt ${restart.attempts}/${MAX_RESTARTS} in ${Math.round(retry.delayMs / 1e3)}s`);
setRuntime(channelId, id, {
accountId: id,
restartPending: true,
reconnectAttempts: restart.attempts
});
pendingAutoRestarts.add(rKey);
try {
await sleepWithAbort(retry.delayMs, retry.signal);
if (manuallyStopped.has(rKey) || opts.isClosing?.()) return;
releaseTask();
await startChannelInternal(channelId, id, {
preserveRestartAttempts: true,
preserveManualStop: true
});
} catch {} finally {
pendingAutoRestarts.delete(rKey);
}
}).finally(() => {
releaseTask();
if (routeHandoff?.admittedSignal === abort.signal) releaseRouteHandoff(store, id, routeHandoff);
});
function releaseTask() {
if (store.tasks.get(id) === trackedPromise) store.tasks.delete(id);
if (store.lifetimes.get(id) === lifetime && !store.stops.has(id)) store.lifetimes.delete(id);
abort.abort();
}
function isCurrentTask() {
return store.tasks.get(id) === trackedPromise;
}
handedOffTask = true;
store.tasks.set(id, trackedPromise);
if (routeHandoff) routeHandoff.admittedSignal = abort.signal;
startOutcomes.set(id, { status: "handed-off" });
} catch (error) {
if (!handedOffTask) setStoppedRuntime(channelId, id, {
restartPending: false,
lastError: formatErrorMessage(error)
});
throw error;
} finally {
if (!handedOffTask) {
capabilityLease.revoke();
if (routeHandoff) releaseRouteHandoff(store, id, routeHandoff);
await cleanupTaskScopedApprovalRuntime("channel startup cleanup failed");
}
if (!handedOffTask && store.lifetimes.get(id) === lifetime && !store.stops.has(id)) store.lifetimes.delete(id);
if (store.starting.get(id) === startGate.promise) store.starting.delete(id);
startGate.resolve();
}
})
});
if (startup.hasError) throw startup.firstError;
return startOutcomes;
};
const startChannelInternal = (...args) => withRegistry((registry) => runOutsideGatewayRootWorkAdmission(() => startChannelProcessOwned(registry, ...args)));
const stopChannelInRegistry = async (registry, channelId, accountId, optsLocal = {}) => {
const manual = optsLocal.manual ?? true;
const plugin = getLoadedChannelPluginEntryById(channelId, registry)?.plugin;
const store = getStore(channelId);
if (manual || !optsLocal.routeHandoff) releaseChannelRouteHandoffs(channelId, accountId);
const lifecycleIds = /* @__PURE__ */ new Set([
...store.lifetimes.keys(),
...store.starting.keys(),
...store.stops.keys(),
...store.tasks.keys()
]);
if (!accountId && lifecycleIds.size === 0) return;
const cfg = getRuntimeConfig();
const configuredAccountIds = !accountId || optsLocal.routeHandoff ? plugin?.config.listAccountIds(cfg) ?? [] : [];
const knownIds = new Set(accountId ? [accountId] : [...lifecycleIds, ...configuredAccountIds]);
const failedStop = (await Promise.all(Array.from(knownIds.values()).map(async (id) => {
const rKey = restartKey(channelId, id);
if (manual) manuallyStopped.add(rKey);
const runStopAttempt = async (previousOutcome) => {
const lifetime = store.lifetimes.get(id);
const abort = lifetime?.abort;
const canHandoff = optsLocal.routeHandoff && configuredAccountIds.includes(id) && !isChannelAccountExplicitlyDisabled({
cfg,
channel: channelId,
accountId: id
}) && !manuallyStopped.has(rKey);
if (!canHandoff) releaseRouteHandoff(store, id);
const task = store.tasks.get(id);
if (!abort && !task && !plugin?.gateway?.stopAccount) return previousOutcome;
const lease = lifetime?.capabilityLease;
if (canHandoff && abort && lease && store.routeHandoffs.get(id)?.parkedBy !== abort) {
const handoff = store.routeHandoffs.get(id)?.handoff ?? createPluginHttpRouteHandoff();
handoff.park(lease);
store.routeHandoffs.set(id, {
handoff,
parkedBy: abort
});
}
if (optsLocal.routeHandoff) lease?.revoke();
abort?.abort();
const log = ensureChannelLog(channelId);
let outcome = { status: "fulfilled" };
let capabilityLease;
try {
let teardown = lifetime?.teardown;
if (!lifetime && plugin?.gateway?.stopAccount) teardown = {
context: createAccountContext(channelId, id, cfg, plugin.config.resolveAccount(cfg, id), new AbortController().signal),
run: plugin.gateway.stopAccount.bind(plugin.gateway)
};
if (teardown) {
const { context, run } = teardown;
const stopLease = createPluginRuntimeCapabilityLease("channel account stop");
capabilityLease = stopLease;
let stopAttemptAbandoned = false;
const runStopAccount = () => run({
...context,
setStatus: (next) => stopLease.isActive() ? setRuntime(channelId, id, next) : getRuntime(channelId, id)
});
if (!await waitForChannelStopGracefully(withPluginHttpRouteRegistry(registry, runStopAccount, stopLease).catch((error) => {
if (stopAttemptAbandoned) {
log.warn?.(`[${id}] abandoned stopAccount failed late: ${formatErrorMessage(error)}`);
return;
}
outcome = {
status: "rejected",
error
};
log.warn?.(`[${id}] stopAccount failed: ${formatErrorMessage(error)}`);
}), CHANNEL_STOP_ABORT_TIMEOUT_MS)) {
stopAttemptAbandoned = true;
log.warn?.(`[${id}] stopAccount exceeded ${CHANNEL_STOP_ABORT_TIMEOUT_MS}ms; continuing stop`);
}
}
} catch (error) {
outcome = {
status: "rejected",
error
};
log.warn?.(`[${id}] stopAccount failed: ${formatErrorMessage(error)}`);
} finally {
capabilityLease?.revoke();
}
const stoppedCleanly = await waitForChannelStopGracefully(task, CHANNEL_STOP_ABORT_TIMEOUT_MS);
if (!stoppedCleanly) log.warn?.(`[${id}] channel stop exceeded ${CHANNEL_STOP_ABORT_TIMEOUT_MS}ms after abort; continuing shutdown`);
if (outcome.status === "rejected") {
recoveryStopTimedOut.delete(rKey);
recoveryStartRequested.delete(rKey);
if (stoppedCleanly && store.tasks.get(id) === task) store.tasks.delete(id);
setRuntime(channelId, id, {
accountId: id,
running: true,
restartPending: false,
lastError: formatErrorMessage(outcome.error)
});
return outcome;
}
if (!stoppedCleanly) {
const stoppedPatch = {
restartPending: !manual,
lastError: `channel stop timed out after ${CHANNEL_STOP_ABORT_TIMEOUT_MS}ms`
};
if (manual) setRuntime(channelId, id, {
accountId: id,
running: true,
...stoppedPatch
});
else {
setStoppedRuntime(channelId, id, stoppedPatch);
recoveryStopTimedOut.add(rKey);
}
return outcome;
}
recoveryStopTimedOut.delete(rKey);
recoveryStartRequested.delete(rKey);
if (store.tasks.get(id) === task) store.tasks.delete(id);
setStoppedRuntime(channelId, id, {
restartPending: false,
lastStopAt: Date.now()
});
return outcome;
};
const currentStop = store.stops.get(id);
const stopAttempt = (currentStop?.status === "stopping" ? currentStop.attempt : Promise.resolve(currentStop ?? { status: "fulfilled" })).then(runStopAttempt);
store.stops.set(id, {
status: "stopping",
attempt: stopAttempt
});
const outcome = await stopAttempt;
const latestStop = store.stops.get(id);
if (latestStop?.status === "stopping" && latestStop.attempt === stopAttempt) {
if (outcome.status === "rejected") store.stops.set(id, outcome);
else {
store.stops.delete(id);
if (!store.tasks.has(id) && !store.starting.has(id)) store.lifetimes.delete(id);
}
}
return outcome;
}))).find((outcome) => outcome.status === "rejected");
if (failedStop?.status === "rejected") throw failedStop.error;
};
const stopChannel = (...args) => withRegistry((registry) => stopChannelInRegistry(registry, ...args));
const startChannelsWithOptions = async (startOptions = {}) => {
let releaseAccountStarts;
const deferAccountStartUntil = opts.deferStartupAccountStartsUntil ?? (startupTrace ? new Promise((resolve) => {
releaseAccountStarts = () => {
setImmediate(resolve).unref?.();
};
}) : void 0);
try {
await runTasksWithConcurrency({
limit: CHANNEL_STARTUP_CONCURRENCY,
tasks: listLoadedChannelPluginsForRegistry(getPluginRegistry()).map((plugin) => async () => {
try {
await measureStartup(`channels.${plugin.id}.start`, () => startChannelInternal(plugin.id, void 0, {
...startOptions,
...deferAccountStartUntil ? { deferAccountStartUntil } : {}
}));
} catch (err) {
ensureChannelLog(plugin.id).error?.(`[${plugin.id}] channel startup failed: ${formatErrorMessage(err)}`);
}
})
});
} finally {
releaseAccountStarts?.();
}
};
const startChannels = async () => await startChannelsWithOptions();
const recoverAutostartSuppression = async () => {
if (!autostartSuppression || opts.isClosing?.() || !opts.tryRecoverAutostartSuppression?.() || opts.isClosing?.()) return false;
autostartSuppression = null;
await startChannelsWithOptions({ preserveManualStop: true });
return true;
};
const markChannelLoggedOut = (channelId, cleared, accountId) => {
const plugin = getChannelPlugin(channelId);
if (!plugin) return;
const cfg = getRuntimeConfig();
const resolvedId = accountId ?? resolveChannelDefaultAccountId({
plugin,
cfg
});
const current = getRuntime(channelId, resolvedId);
setStoppedRuntime(channelId, resolvedId, {
...cleared ? { linked: false } : {},
restartPending: false,
lastError: cleared ? "logged out" : current.lastError
});
};
const getRuntimeSnapshot = () => {
const registry = getPluginRegistry();
const cfg = getRuntimeConfig();
const channels = {};
const channelAccounts = {};
for (const plugin of listLoadedChannelPluginsForRegistry(registry)) {
const store = getStore(plugin.id);
const accountIds = plugin.config.listAccountIds(cfg);
const defaultAccountId = resolveChannelDefaultAccountId({
plugin,
cfg,
accountIds
});
const accounts = {};
for (const id of accountIds) {
const current = store.runtimes.get(id) ?? cloneDefaultRuntime(plugin.id, id);
const unavailable = resolveUnavailableChannelAccountSnapshot(cfg, {
registry,
channelId: plugin.id,
accountId: id,
runtime: current
});
if (unavailable) {
accounts[id] = unavailable;
continue;
}
const inspected = plugin.config.inspectAccount?.(cfg, id);
if (inspected) {
accounts[id] = buildChannelAccountSnapshotFromInspection({
account: inspected,
accountId: id,
runtime: current
});
continue;
}
const account = plugin.config.resolveAccount(cfg, id);
const enabled = plugin.config.isEnabled ? plugin.config.isEnabled(account, cfg) : isAccountEnabled(account);
const described = plugin.config.describeAccount?.(account, cfg);
const configured = described?.configured ?? current.configured ?? true;
const state = resolveChannelAccountState({
enabled,
configured,
linked: plugin.config.isLinked ? current.linked : typeof current.linked === "boolean" ? current.linked : described?.linked,
runtime: current,
disabledReason: plugin.config.disabledReason?.(account, cfg),
unconfiguredReason: plugin.config.unconfiguredReason?.(account, cfg),
unlinkedReason: plugin.config.unlinkedReason?.(account, cfg)
});
const next = {
...current,
accountId: id,
enabled
};
applyChannelAccountState(next, state);
if (described?.mode !== void 0) next.mode = described.mode;
accounts[id] = next;
}
const defaultAccount = accounts[defaultAccountId] ?? cloneDefaultRuntime(plugin.id, defaultAccountId);
channels[plugin.id] = defaultAccount;
channelAccounts[plugin.id] = accounts;
}
return {
channels,
channelAccounts
};
};
const isManuallyStoppedFlag = (channelId, accountId) => {
return manuallyStopped.has(restartKey(channelId, accountId));
};
const isAutoRestartScheduled = (channelId, accountId) => {
return pendingAutoRestarts.has(restartKey(channelId, accountId));
};
const resetRestartAttempts = (channelId, accountId) => {
restarts.delete(restartKey(channelId, accountId));
};
return {
getRuntimeSnapshot,
pauseChannelStarts: () => {
const previous = channelStartPause;
const release = (outcome) => {
if (channelStartPause === release) channelStartPause = outcome === "published" ? void 0 : previous;
};
channelStartPause = release;
return release;
},
startChannels,
startChannel: startChannelInternal,
stopChannel,
releaseChannelRouteHandoffs,
pruneInactiveChannelAccountState,
setAutostartSuppression: (suppression) => {
autostartSuppression = suppression;
},
getAutostartSuppression: () => autostartSuppression,
recoverAutostartSuppression,
setAmbientAutostartSuppressedChannelIds: (channelIds) => {
ambientAutostartSuppressedChannelIds = new Set(channelIds);
},
isAmbientAutostartSuppressed: (channelId) => ambientAutostartSuppressedChannelIds.has(channelId),
markChannelLoggedOut,
isManuallyStopped: isManuallyStoppedFlag,
resolveRuntimeAccountId: (channelId, accountId) => {
const matches = [...channelStores.get(channelId)?.runtimes.keys() ?? []].filter((id) => normalizeAccountId(id) === accountId);
return matches.length === 1 ? matches[0] : void 0;
},
isAutoRestartScheduled,
resetRestartAttempts,
isHealthMonitorEnabled
};
}
//#endregion
export { createChannelManager };