openclaw
Version:
Multi-channel AI gateway with extensible messaging integrations
661 lines (660 loc) • 28 kB
JavaScript
import { l as normalizeOptionalString, o as normalizeLowercaseStringOrEmpty } from "./string-coerce-CIXf7egm.js";
import { r as truncateUtf16Safe } from "./utf16-slice-D_ngcYKd.js";
import { S as createConfigIO } from "./io.runtime-B9iJRs3w.js";
import "./utils-P__uGsPB.js";
import { t as sleep } from "./sleep-D7nua6TP.js";
import { o as redactSensitiveUrlLikeString } from "./redact-sensitive-url-BN1NZvXG.js";
import { t as formatErrorMessage } from "./errors-Db3Ymjlb.js";
import { n as sanitizeTerminalText } from "./safe-text-BGBqp1a4.js";
import { a as READ_SCOPE } from "./operator-scopes-Dw7Gu2cA.js";
import { i as GATEWAY_CLIENT_NAMES, r as GATEWAY_CLIENT_MODES } from "./client-info-B1bPgeKr.js";
import "./io-bdCzpGWJ.js";
import { i as readActiveGatewayLockIdentity, r as isSameGatewayLockIdentity } from "./gateway-lock-B0QIxQaj.js";
import { t as LOOPBACK_PORT_PROBE_HOSTS } from "./ports-probe-funEe1wt.js";
import { t as resolveGatewayServiceProbeHosts } from "./gateway-service-probe-hosts-CH6J1p0Q.js";
import { n as classifyPortListener, r as formatPortDiagnostics } from "./ports-format-D-ReVpaa.js";
import { n as inspectPortUsage } from "./ports-inspect-CHTlDwBh.js";
import "./restart-stale-pids-CDR-7Ato.js";
import { o as classifyGatewayConnectFailure } from "./connect-error-details-DaOfGNN8.js";
import "./client-I-RoP1Al.js";
import { r as isGatewayProtocolResponseError } from "./protocol-request-BMUN1re6.js";
import { o as callGateway } from "./call-BWR5pPgw.js";
import "./method-scopes-K6J_UQGL.js";
import { t as createConfiguredGatewayLocalProbe } from "./local-http-probe-OUAnqErL.js";
import { i as hasActiveStartupMigrationLease, t as STARTUP_MIGRATION_LEASE_TTL_MS } from "./startup-migration-checkpoint-C-ZEoQYl.js";
import { r as resolveGatewayProbeAuthSafeWithSecretInputs } from "./probe-auth-DiKkHhJQ.js";
import "./ports-CU06KPCY.js";
//#region src/cli/daemon-cli/restart-port-ownership.ts
function hasListenerAttributionGap(portUsage) {
if (portUsage.status !== "busy" || portUsage.listeners.length > 0) return false;
if (portUsage.errors?.length) return true;
return portUsage.hints.some((hint) => hint.includes("process details are unavailable"));
}
function listenerOwnedByRuntimePid(params) {
return params.listener.pid === params.runtimePid || params.listener.ppid === params.runtimePid;
}
function allListenersOwnedByRuntimePid(listeners, runtimePid) {
return listeners.length > 0 && listeners.every((listener) => listenerOwnedByRuntimePid({
listener,
runtimePid
}));
}
//#endregion
//#region src/cli/daemon-cli/restart-health-probe.ts
/** Waits for the unauthenticated HTTP(S) readiness contracts reported by service start. */
async function waitForGatewayHttpReadiness(params) {
const probe = createConfiguredGatewayLocalProbe(params.config ?? {});
let latest = {
healthz: null,
readyz: null
};
for (let attempt = 0; attempt < params.attempts; attempt += 1) {
const remainingMs = params.deadlineAt - Date.now();
if (remainingMs <= 0) return latest;
const [healthz, readyz] = await Promise.all([probe.requestHttp({
host: "127.0.0.1",
pathname: "/healthz",
port: params.port,
timeoutMs: Math.min(remainingMs, 3e3)
}).then((result) => result?.statusCode ?? null), probe.requestHttp({
host: "127.0.0.1",
pathname: "/readyz",
port: params.port,
timeoutMs: Math.min(remainingMs, 3e3)
}).then((result) => result?.statusCode ?? null)]);
latest = {
healthz,
readyz
};
if (healthz === 200 && readyz === 200) return latest;
if (attempt + 1 < params.attempts) {
const remainingDelayMs = params.deadlineAt - Date.now();
if (remainingDelayMs <= 0) return latest;
await sleep(Math.min(params.delayMs, remainingDelayMs));
}
}
return latest;
}
function formatGatewayRestartProbeError(error) {
return truncateUtf16Safe(sanitizeTerminalText(redactSensitiveUrlLikeString(formatErrorMessage(error))), 1024);
}
function isGatewayAuthRejection(reason) {
const normalized = normalizeLowercaseStringOrEmpty(reason);
if (classifyGatewayConnectFailure({ reason: normalized }).kind === "pairing-required" && (normalized === "pairing required" || normalized.startsWith("pairing required:"))) return true;
return normalized === "auth required" || normalized === "owner auth required" || normalized === "connect failed" || normalized === "device required" || normalized.startsWith("unauthorized: gateway token missing") || normalized.startsWith("unauthorized: gateway token mismatch") || normalized.startsWith("unauthorized: gateway token not configured") || normalized.startsWith("unauthorized: gateway password missing") || normalized.startsWith("unauthorized: gateway password mismatch") || normalized.startsWith("unauthorized: gateway password not configured") || normalized.startsWith("unauthorized: bootstrap token invalid or expired") || normalized.startsWith("unauthorized: tailscale identity missing") || normalized.startsWith("unauthorized: tailscale proxy headers missing") || normalized.startsWith("unauthorized: tailscale identity check failed") || normalized.startsWith("unauthorized: tailscale identity mismatch") || normalized.startsWith("unauthorized: too many failed authentication attempts") || normalized.startsWith("unauthorized: device token mismatch") || normalized.startsWith("unauthorized: device token rejected");
}
function readActivatedPluginErrors(health) {
if (!health || typeof health !== "object") return [];
const plugins = health.plugins;
if (!plugins || typeof plugins !== "object") return [];
const errors = plugins.errors;
if (!Array.isArray(errors)) return [];
return errors.filter((entry) => {
if (!entry || typeof entry !== "object") return false;
const candidate = entry;
return candidate.activated === true && typeof candidate.id === "string" && typeof candidate.error === "string";
}).map((entry) => {
const error = {
id: entry.id,
origin: typeof entry.origin === "string" ? entry.origin : "unknown",
activated: true,
error: entry.error
};
if (typeof entry.activationSource === "string") error.activationSource = entry.activationSource;
if (typeof entry.activationReason === "string") error.activationReason = entry.activationReason;
if (typeof entry.failurePhase === "string") error.failurePhase = entry.failurePhase;
return error;
});
}
function readChannelProbeErrors(health) {
if (!health || typeof health !== "object") return [];
const channels = health.channels;
if (!channels || typeof channels !== "object" || Array.isArray(channels)) return [];
const errors = [];
for (const [id, summary] of Object.entries(channels)) {
if (!summary || typeof summary !== "object") continue;
const probe = summary.probe;
if (!probe || typeof probe !== "object") continue;
if (probe.ok !== false) continue;
const error = probe.error;
errors.push({
id,
error: typeof error === "string" && error.trim() ? error : "probe failed"
});
}
return errors;
}
async function confirmGatewayReachable(params) {
const result = {
reachable: false,
gatewayVersion: null,
gatewayBuildId: void 0,
activatedPluginErrors: [],
channelProbeErrors: []
};
try {
const context = params.config ? {
config: params.config,
auth: params.auth
} : await resolveGatewayRestartProbeContext(params.env);
const auth = params.auth ?? context.auth;
const target = await (params.configuredProbe ?? createConfiguredGatewayLocalProbe(context.config)).resolveWebSocketTarget(params.port);
if (!target) return {
...result,
gatewayBuildId: null,
probeError: "gateway TLS certificate unavailable"
};
const authNone = context.config.gateway?.auth?.mode === "none";
const health = await callGateway({
config: context.config,
localPortOverride: params.port,
token: auth?.token,
password: auth?.password,
skipImplicitAuth: true,
tlsFingerprint: target.tlsFingerprint,
method: "health",
scopes: [READ_SCOPE],
clientName: authNone ? GATEWAY_CLIENT_NAMES.GATEWAY_CLIENT : GATEWAY_CLIENT_NAMES.CLI,
mode: authNone ? GATEWAY_CLIENT_MODES.BACKEND : GATEWAY_CLIENT_MODES.CLI,
requireLocalBackendSharedAuth: authNone,
deviceIdentity: null,
sharedStateMode: "read-only",
timeoutMs: 3e3,
onHelloOk: (hello) => {
result.gatewayVersion = hello.server.version;
result.gatewayBuildId = hello.server.buildId ?? null;
}
});
result.reachable = true;
result.activatedPluginErrors = readActivatedPluginErrors(health);
result.channelProbeErrors = readChannelProbeErrors(health);
} catch (error) {
result.reachable = result.gatewayVersion === null && isGatewayProtocolResponseError(error) && (isGatewayAuthRejection(error.message) || params.allowDeviceIdentityRequired === true && error.message === "device identity required");
if (result.reachable) result.gatewayBuildId ??= null;
else result.probeError = formatGatewayRestartProbeError(error);
}
return result;
}
async function resolveGatewayRestartProbeContext(env) {
const mergedEnv = {
...process.env,
...env ?? void 0
};
const cfg = await createConfigIO({
env: mergedEnv,
observe: false,
pluginValidation: "skip",
suppressFutureVersionWarning: true
}).readBestEffortConfig().catch(() => ({}));
return {
auth: (await resolveGatewayProbeAuthSafeWithSecretInputs({
cfg,
mode: "local",
env: mergedEnv
})).auth,
config: cfg
};
}
async function inspectGatewayPortHealth(params) {
let portUsage;
try {
portUsage = await inspectPortUsage(params.port, { probeHosts: LOOPBACK_PORT_PROBE_HOSTS });
} catch (err) {
portUsage = {
port: params.port,
status: "unknown",
listeners: [],
hints: [],
errors: [String(err)]
};
}
if (portUsage.status !== "busy") return {
portUsage,
healthy: false
};
const expectedListenerPid = params.expectedListenerPid;
const listenerOwnershipVerified = expectedListenerPid !== void 0 && allListenersOwnedByRuntimePid(portUsage.listeners, expectedListenerPid);
const { reachable, probeError } = await confirmGatewayReachable({
port: params.port,
auth: params.auth,
...params.config ? { config: params.config } : {},
...params.configuredProbe ? { configuredProbe: params.configuredProbe } : {},
env: process.env,
allowDeviceIdentityRequired: listenerOwnershipVerified
});
return {
portUsage,
healthy: reachable,
...probeError ? { probeError } : {}
};
}
//#endregion
//#region src/cli/daemon-cli/restart-health.constants.ts
const DEFAULT_RESTART_HEALTH_TIMEOUT_MS = 6e4;
const DEFAULT_RESTART_HEALTH_DELAY_MS = 500;
const DEFAULT_RESTART_HEALTH_ATTEMPTS = Math.ceil(DEFAULT_RESTART_HEALTH_TIMEOUT_MS / 500);
//#endregion
//#region src/cli/daemon-cli/restart-health-diagnostics.ts
function renderPortUsageDiagnostics(snapshot) {
const lines = [];
if (snapshot.portUsage.status === "busy") lines.push(...formatPortDiagnostics(snapshot.portUsage));
else lines.push(`Gateway port ${snapshot.portUsage.port} status: ${snapshot.portUsage.status}.`);
if (snapshot.portUsage.errors?.length) lines.push(`Port diagnostics errors: ${snapshot.portUsage.errors.join("; ")}`);
if (snapshot.probeError) lines.push(`Gateway probe failed: ${snapshot.probeError}`);
return lines;
}
function renderRestartDiagnostics(snapshot) {
const lines = [];
if (snapshot.versionMismatch) {
const actual = snapshot.versionMismatch.actual ?? "unavailable";
lines.push(`Gateway version mismatch: expected ${snapshot.versionMismatch.expected}, running gateway reported ${actual}.`);
}
if (snapshot.buildIdMismatch) {
const actual = snapshot.buildIdMismatch.actual ?? "unavailable";
lines.push(`Gateway build mismatch: expected ${snapshot.buildIdMismatch.expected}, running gateway reported ${actual}.`);
}
if (snapshot.activatedPluginErrors?.length) {
lines.push("Activated plugin load errors:");
for (const plugin of snapshot.activatedPluginErrors) lines.push(`- ${plugin.id}: ${plugin.error}`);
}
if (snapshot.channelProbeErrors?.length) {
lines.push("Channel health probe errors:");
for (const channel of snapshot.channelProbeErrors) lines.push(`- ${channel.id}: ${channel.error}`);
}
const runtimeSummary = [
snapshot.runtime.status ? `status=${snapshot.runtime.status}` : null,
snapshot.runtime.state ? `state=${snapshot.runtime.state}` : null,
snapshot.runtime.pid != null ? `pid=${snapshot.runtime.pid}` : null,
snapshot.runtime.lastExitStatus != null ? `lastExit=${snapshot.runtime.lastExitStatus}` : null
].filter(Boolean).join(", ");
if (runtimeSummary) lines.push(`Service runtime: ${runtimeSummary}`);
lines.push(...renderPortUsageDiagnostics(snapshot));
return lines;
}
function formatGatewayRestartFailure(params) {
if (params.health.waitOutcome === "stopped-free") {
const elapsedSeconds = Math.max(1, Math.round((params.health.elapsedMs ?? 0) / 1e3));
return {
statusLine: `Gateway restart failed after ${elapsedSeconds}s: service stayed stopped and port ${params.port} stayed free.`,
failMessage: `Gateway restart failed after ${elapsedSeconds}s: service stayed stopped and health checks never came up.`
};
}
const timeoutSeconds = Math.max(1, Math.round(params.health.elapsedMs === void 0 ? params.defaultTimeoutSeconds : params.health.elapsedMs / 1e3));
return {
statusLine: `Timed out after ${timeoutSeconds}s waiting for gateway port ${params.port} to become healthy.`,
failMessage: `Gateway restart timed out after ${timeoutSeconds}s waiting for health checks.`
};
}
function renderGatewayPortHealthDiagnostics(snapshot) {
return renderPortUsageDiagnostics(snapshot);
}
//#endregion
//#region src/cli/daemon-cli/restart-lock-replacement.ts
async function waitForGatewayLockReplacement(params) {
let attemptsUsed = 0;
let previousOwnerReleased = false;
for (;;) {
let currentLockIdentity;
try {
currentLockIdentity = await readActiveGatewayLockIdentity();
} catch {
if (params.waitIndefinitelyForPreviousOwner && !previousOwnerReleased) {
await sleep(params.delayMs);
continue;
}
if (attemptsUsed >= params.attempts) return { status: "timeout" };
attemptsUsed += 1;
await sleep(params.delayMs);
continue;
}
if (!previousOwnerReleased) {
if (currentLockIdentity && isSameGatewayLockIdentity(params.previousLockIdentity, currentLockIdentity)) {
if (params.waitIndefinitelyForPreviousOwner) {
await sleep(params.delayMs);
continue;
}
} else {
previousOwnerReleased = true;
if (params.waitIndefinitelyForPreviousOwner) attemptsUsed = 0;
}
}
if (previousOwnerReleased && currentLockIdentity && !isSameGatewayLockIdentity(params.previousLockIdentity, currentLockIdentity)) return {
status: "replacement",
attemptsUsed,
lockIdentity: currentLockIdentity
};
if (attemptsUsed >= params.attempts) return { status: "timeout" };
attemptsUsed += 1;
await sleep(params.delayMs);
}
}
//#endregion
//#region src/cli/daemon-cli/restart-health-external.ts
async function waitForGatewayHealthyListener(params) {
const attempts = params.attempts ?? DEFAULT_RESTART_HEALTH_ATTEMPTS;
const delayMs = params.delayMs ?? 500;
const previousLockIdentity = params.previousLockIdentity;
const probeContext = await resolveGatewayRestartProbeContext(void 0).catch(() => ({
auth: void 0,
config: {}
}));
const configuredProbe = createConfiguredGatewayLocalProbe(probeContext.config);
let snapshot = previousLockIdentity ? {
portUsage: {
port: params.port,
status: "unknown",
listeners: [],
hints: [],
errors: [`Previous gateway lock owner ${previousLockIdentity.ownerId ?? previousLockIdentity.pid} is still active.`]
},
healthy: false
} : await inspectGatewayPortHealth({
port: params.port,
auth: probeContext.auth,
configuredProbe
});
let attempt = 0;
let expectedListenerPid;
if (previousLockIdentity) {
const replacement = await waitForGatewayLockReplacement({
previousLockIdentity,
attempts,
delayMs,
waitIndefinitelyForPreviousOwner: params.waitIndefinitelyForPreviousOwner === true
});
if (replacement.status === "timeout") return snapshot;
attempt = replacement.attemptsUsed;
expectedListenerPid = replacement.lockIdentity.pid;
snapshot = await inspectGatewayPortHealth({
port: params.port,
auth: probeContext.auth,
configuredProbe,
expectedListenerPid
});
}
if (snapshot.healthy) return snapshot;
while (attempt < attempts) {
attempt += 1;
await sleep(delayMs);
snapshot = await inspectGatewayPortHealth({
port: params.port,
auth: probeContext.auth,
configuredProbe,
expectedListenerPid
});
if (snapshot.healthy) return snapshot;
}
return snapshot;
}
//#endregion
//#region src/cli/daemon-cli/restart-health.ts
const STARTUP_MIGRATION_ACTIVITY_POLL_MS = 5e3;
const STOPPED_FREE_EARLY_EXIT_GRACE_MS = 1e4;
const WINDOWS_STOPPED_FREE_EARLY_EXIT_GRACE_MS = 9e4;
function applyExpectedVersion(snapshot, expectedVersion) {
if (!expectedVersion) return snapshot;
if (snapshot.gatewayVersion === expectedVersion) return {
...snapshot,
expectedVersion
};
if (snapshot.gatewayVersion == null) return {
...snapshot,
healthy: false,
expectedVersion
};
return {
...snapshot,
healthy: false,
expectedVersion,
versionMismatch: {
expected: expectedVersion,
actual: snapshot.gatewayVersion ?? null
}
};
}
function applyExpectedBuildId(snapshot, expectedBuildId) {
if (!expectedBuildId) return snapshot;
if (snapshot.gatewayBuildId === expectedBuildId) return {
...snapshot,
expectedBuildId
};
if (snapshot.gatewayBuildId === void 0) return {
...snapshot,
healthy: false,
expectedBuildId
};
return {
...snapshot,
healthy: false,
expectedBuildId,
buildIdMismatch: {
expected: expectedBuildId,
actual: snapshot.gatewayBuildId ?? null
}
};
}
function applyExpectedGatewayIdentity(snapshot, expectedVersion, expectedBuildId) {
return applyExpectedBuildId(applyExpectedVersion(snapshot, expectedVersion), expectedBuildId);
}
function applyActivatedPluginErrors(snapshot) {
if (!snapshot.activatedPluginErrors?.length) return snapshot;
return {
...snapshot,
healthy: false
};
}
function applyChannelProbeErrors(snapshot) {
if (!snapshot.channelProbeErrors?.length) return snapshot;
return {
...snapshot,
healthy: false
};
}
async function inspectGatewayRestart(params) {
const env = params.env ?? process.env;
const probeHosts = params.probeHosts ?? await resolveGatewayServiceProbeHosts({
env,
command: await params.service.readCommand?.(env).catch(() => null) ?? null
});
const expectedVersion = normalizeOptionalString(params.expectedVersion);
const expectedBuildId = normalizeOptionalString(params.expectedBuildId);
const requiresGatewayProbe = Boolean(expectedVersion || expectedBuildId);
let reachability = null;
let probeError;
let activatedPluginErrors = [];
let channelProbeErrors = [];
const loadReachability = async () => {
if (!reachability) {
reachability = await confirmGatewayReachable({
port: params.port,
...params.probeContext,
...params.configuredProbe ? { configuredProbe: params.configuredProbe } : {},
env
});
probeError = reachability.probeError;
activatedPluginErrors = reachability.activatedPluginErrors;
channelProbeErrors = reachability.channelProbeErrors;
}
return reachability;
};
let runtime = { status: "unknown" };
try {
runtime = await params.service.readRuntime(env);
} catch (err) {
runtime = {
status: "unknown",
detail: String(err)
};
}
let portUsage;
try {
portUsage = await inspectPortUsage(params.port, { probeHosts });
} catch (err) {
portUsage = {
port: params.port,
status: "unknown",
listeners: [],
hints: [],
errors: [String(err)]
};
}
if (portUsage.status === "busy" && runtime.status !== "running") {
const reachable = await loadReachability();
if (reachable.reachable) return applyChannelProbeErrors(applyActivatedPluginErrors(applyExpectedGatewayIdentity({
runtime,
portUsage,
healthy: true,
staleGatewayPids: [],
gatewayVersion: reachable.gatewayVersion,
gatewayBuildId: reachable.gatewayBuildId,
...reachable.activatedPluginErrors.length > 0 ? { activatedPluginErrors: reachable.activatedPluginErrors } : {},
...reachable.channelProbeErrors.length > 0 ? { channelProbeErrors: reachable.channelProbeErrors } : {}
}, expectedVersion, expectedBuildId)));
}
const gatewayListeners = portUsage.status === "busy" ? portUsage.listeners.filter((listener) => classifyPortListener(listener, params.port) === "gateway") : [];
const fallbackListenerPids = params.includeUnknownListenersAsStale && process.platform === "win32" && runtime.status !== "running" && portUsage.status === "busy" ? portUsage.listeners.filter((listener) => classifyPortListener(listener, params.port) === "unknown").map((listener) => listener.pid).filter((pid) => Number.isFinite(pid)) : [];
const running = runtime.status === "running";
const runtimePid = runtime.pid;
const listenerAttributionGap = hasListenerAttributionGap(portUsage);
const ownsPort = runtimePid != null ? portUsage.listeners.some((listener) => listenerOwnedByRuntimePid({
listener,
runtimePid
})) || listenerAttributionGap : gatewayListeners.length > 0 || listenerAttributionGap;
let healthy = running && ownsPort;
let gatewayVersion;
let gatewayBuildId;
if (requiresGatewayProbe && healthy && portUsage.status === "busy") {
const reachable = await loadReachability();
healthy = reachable.reachable;
gatewayVersion = reachable.gatewayVersion;
gatewayBuildId = reachable.gatewayBuildId;
if (reachable.activatedPluginErrors.length > 0) healthy = false;
if (reachable.channelProbeErrors.length > 0) healthy = false;
}
if (!healthy && running && portUsage.status === "busy" && !requiresGatewayProbe) {
const reachable = await loadReachability();
healthy = reachable.reachable;
gatewayVersion = reachable.gatewayVersion;
gatewayBuildId = reachable.gatewayBuildId;
}
const staleGatewayPids = Array.from(/* @__PURE__ */ new Set([...gatewayListeners.filter((listener) => Number.isFinite(listener.pid)).filter((listener) => {
if (!running) return true;
if (runtimePid == null) return false;
return !listenerOwnedByRuntimePid({
listener,
runtimePid
});
}).map((listener) => listener.pid), ...fallbackListenerPids.filter((pid) => runtime.pid == null || pid !== runtime.pid || !running)]));
return applyChannelProbeErrors(applyActivatedPluginErrors(applyExpectedGatewayIdentity({
runtime,
portUsage,
healthy,
staleGatewayPids,
...gatewayVersion !== void 0 ? { gatewayVersion } : {},
...gatewayBuildId !== void 0 ? { gatewayBuildId } : {},
...probeError ? { probeError } : {},
...activatedPluginErrors.length ? { activatedPluginErrors } : {},
...channelProbeErrors.length ? { channelProbeErrors } : {}
}, expectedVersion, expectedBuildId)));
}
function shouldEarlyExitStoppedFree(snapshot, attempt, minAttempt) {
return attempt >= minAttempt && snapshot.runtime.status === "stopped" && snapshot.portUsage.status === "free";
}
function stoppedFreeEarlyExitGraceMs() {
return process.platform === "win32" ? WINDOWS_STOPPED_FREE_EARLY_EXIT_GRACE_MS : STOPPED_FREE_EARLY_EXIT_GRACE_MS;
}
function withWaitContext(snapshot, waitOutcome, elapsedMs) {
return {
...snapshot,
waitOutcome,
elapsedMs
};
}
async function waitForGatewayHealthyRestart(params) {
const startedAtMs = performance.now();
const attempts = params.attempts ?? DEFAULT_RESTART_HEALTH_ATTEMPTS;
const delayMs = params.delayMs ?? 500;
const settleProbes = Math.max(1, params.settle?.probes ?? 1);
const settleDurationMs = (settleProbes - 1) * delayMs;
const standardDeadlineMs = attempts * delayMs;
const probeContext = await resolveGatewayRestartProbeContext(params.env).catch(() => ({
auth: void 0,
config: {}
}));
const configuredProbe = createConfiguredGatewayLocalProbe(probeContext.config);
const probeHosts = params.probeHosts ?? await resolveGatewayServiceProbeHosts({
env: params.env,
command: await params.service.readCommand(params.env ?? process.env).catch(() => null)
});
let snapshot = await inspectGatewayRestart({
service: params.service,
port: params.port,
env: params.env,
expectedVersion: params.expectedVersion,
expectedBuildId: params.expectedBuildId,
includeUnknownListenersAsStale: params.includeUnknownListenersAsStale,
probeContext,
configuredProbe,
probeHosts
});
let consecutiveStoppedFreeCount = 0;
const STOPPED_FREE_THRESHOLD = 6;
const minAttemptForEarlyExit = Math.min(Math.ceil(stoppedFreeEarlyExitGraceMs() / delayMs), Math.floor(attempts / 2));
let migrationDeadlineMs;
let postMigrationDeadlineMs;
let migrationActive = false;
let nextMigrationActivityPollMs = 0;
let healthyStreak;
for (let attempt = 0;; attempt += 1) {
const elapsedMs = Math.max(0, performance.now() - startedAtMs);
if (snapshot.healthy && (!params.requireRunningService || snapshot.runtime.status === "running" && (process.platform === "win32" || typeof snapshot.runtime.pid === "number"))) {
if (healthyStreak && healthyStreak.pid === snapshot.runtime.pid) healthyStreak.probes += 1;
else healthyStreak = {
pid: snapshot.runtime.pid,
probes: 1
};
if (healthyStreak.probes >= settleProbes) return withWaitContext(snapshot, "healthy", elapsedMs);
} else healthyStreak = void 0;
if (settleProbes > 1 && snapshot.healthy) snapshot.healthy = false;
if (snapshot.activatedPluginErrors?.length) return withWaitContext(snapshot, "plugin-errors", elapsedMs);
if (snapshot.channelProbeErrors?.length) return withWaitContext(snapshot, "channel-errors", elapsedMs);
if (snapshot.versionMismatch) return withWaitContext(snapshot, "version-mismatch", elapsedMs);
if (snapshot.buildIdMismatch) return withWaitContext(snapshot, "build-id-mismatch", elapsedMs);
if (snapshot.staleGatewayPids.length > 0 && snapshot.runtime.status !== "running") return withWaitContext(snapshot, "stale-pids", elapsedMs);
if (!params.supervisorKeepsAlive && shouldEarlyExitStoppedFree(snapshot, attempt, minAttemptForEarlyExit)) {
consecutiveStoppedFreeCount += 1;
if (consecutiveStoppedFreeCount >= STOPPED_FREE_THRESHOLD) return withWaitContext(snapshot, "stopped-free", elapsedMs);
} else if (snapshot.runtime.status !== "stopped" || snapshot.portUsage.status !== "free") consecutiveStoppedFreeCount = 0;
if (snapshot.runtime.status !== "running") migrationActive = false;
else if (elapsedMs >= nextMigrationActivityPollMs) {
migrationActive = (() => {
try {
return (params.isStartupMigrationActive ?? hasActiveStartupMigrationLease)({ env: params.env });
} catch {
return false;
}
})();
nextMigrationActivityPollMs = elapsedMs + STARTUP_MIGRATION_ACTIVITY_POLL_MS;
if (migrationActive && migrationDeadlineMs === void 0) migrationDeadlineMs = elapsedMs + STARTUP_MIGRATION_LEASE_TTL_MS;
else if (!migrationActive && migrationDeadlineMs !== void 0) postMigrationDeadlineMs ??= elapsedMs + standardDeadlineMs;
}
if (elapsedMs >= standardDeadlineMs || migrationDeadlineMs !== void 0) {
const deadlineMs = migrationActive ? migrationDeadlineMs : (postMigrationDeadlineMs ?? standardDeadlineMs) + settleDurationMs;
if (deadlineMs === void 0 || elapsedMs >= deadlineMs) return withWaitContext(snapshot, "timeout", elapsedMs);
}
await sleep(delayMs);
snapshot = await inspectGatewayRestart({
service: params.service,
port: params.port,
env: params.env,
expectedVersion: params.expectedVersion,
expectedBuildId: params.expectedBuildId,
includeUnknownListenersAsStale: params.includeUnknownListenersAsStale,
probeContext,
configuredProbe,
probeHosts
});
}
}
//#endregion
export { renderGatewayPortHealthDiagnostics as a, DEFAULT_RESTART_HEALTH_DELAY_MS as c, formatGatewayRestartFailure as i, waitForGatewayHttpReadiness as l, waitForGatewayHealthyRestart as n, renderRestartDiagnostics as o, waitForGatewayHealthyListener as r, DEFAULT_RESTART_HEALTH_ATTEMPTS as s, inspectGatewayRestart as t };