openclaw
Version:
Multi-channel AI gateway with extensible messaging integrations
570 lines (569 loc) • 23.2 kB
JavaScript
import { P as resolvePositiveTimerTimeoutMs, p as clampPositiveTimerTimeoutMs } from "./number-coercion-CuVhdCBs.mjs";
import { u as toErrorObject } from "./error-coercion-CvJK4lmH.mjs";
import { t as mergeProcessEnv } from "./process-env-DlZFJzq6.mjs";
import { _ as redactSensitiveText, a as isSensitiveFieldKey } from "./redact-DJ8j6ADa.mjs";
import { d as sleepWithAbort } from "./src-D1BHGNTj.mjs";
import "./errors-O33D01XW.mjs";
import { r as setManagedProviderLocalServicesStop } from "./provider-runtime-lifecycle-fpszViWt.mjs";
import { t as createSubsystemLogger } from "./subsystem-I-uqdx4f.mjs";
import { n as truncateUtf8Suffix } from "./utf8-truncate-_hf7tp13.mjs";
import { i as signalChildProcessTree, n as isChildProcessTreeAlive, r as shouldDetachChildForProcessTree } from "./child-process-tree-Dg8n1gpZ.mjs";
import { f as isLoopbackIpAddress, i as isCanonicalDottedDecimalIPv4 } from "./ip-jvtZlu_y.mjs";
import { n as prepareOomScoreAdjustedSpawnPreservingExecEnv } from "./linux-oom-score-Ca8n5Xcm.mjs";
import { n as getModelProviderLocalServiceReconciler } from "./provider-local-service-reconcile-HqZ_bnuN.mjs";
import { n as unwrapHeadersInitSentinelsForProviderEgress } from "./provider-secret-egress-LltAZ95s.mjs";
import path from "node:path";
import { spawn } from "node:child_process";
import { createHash } from "node:crypto";
//#region src/agents/provider-local-service-diagnostics.ts
const LOCAL_SERVICE_OUTPUT_TAIL_MAX_BYTES = 8192;
function appendLocalServiceOutputTail(current, chunk, serviceEnv, inheritedEnv, serviceArgs, healthHeaders) {
let redacted = redactSensitiveText(`${current}${chunk.toString()}`, { mode: "tools" });
const secretValues = [
...Object.values(serviceEnv ?? {}),
...Object.entries(inheritedEnv).flatMap(([key, value]) => value && isSensitiveFieldKey(key) ? [value] : []),
...serviceArgs ?? [],
...new Headers(healthHeaders).values()
];
for (const value of secretValues) if (value) redacted = redacted.replaceAll(value, "[redacted]");
return truncateUtf8Suffix(redacted, LOCAL_SERVICE_OUTPUT_TAIL_MAX_BYTES);
}
function formatLocalServiceDiagnosticTail(diagnostics) {
return diagnostics.stderrTail ? `; stderr: ${diagnostics.stderrTail}` : "";
}
function formatLocalServiceExit(exit) {
return exit.signal ? `signal ${exit.signal}` : `code ${exit.code ?? 0}`;
}
//#endregion
//#region src/agents/provider-local-service-process.ts
const DEFAULT_PROBE_TIMEOUT_MS$1 = 2e3;
const PROCESS_TREE_EXIT_POLL_MS = 25;
function trackLocalServiceProcess(child) {
const owned = {
child,
closed: false,
pendingSignals: /* @__PURE__ */ new Set()
};
child.once("close", () => {
owned.closed = true;
});
return owned;
}
function unrefLocalServiceOutput(stream) {
stream?.unref?.();
}
function drainLocalServiceOutput(child) {
child.stdout?.removeAllListeners("data");
child.stderr?.removeAllListeners("data");
child.stdout?.resume();
child.stderr?.resume();
unrefLocalServiceOutput(child.stdout);
unrefLocalServiceOutput(child.stderr);
}
function isLocalServiceProcessTreeAlive(owned) {
const windows = process.platform === "win32";
if (windows && (owned.windowsRootGone || hasLocalServiceProcessExited(owned.child))) {
owned.windowsRootGone = true;
return false;
}
const alive = isChildProcessTreeAlive(owned.child);
if (windows && !alive) owned.windowsRootGone = true;
return alive;
}
function isLocalServiceProcessSettled(owned) {
return owned.closed && owned.pendingSignals.size === 0 && !isLocalServiceProcessTreeAlive(owned);
}
function signalLocalServiceProcess(owned, signal) {
if (owned.pendingSignals.has(signal) || !isLocalServiceProcessTreeAlive(owned)) return;
owned.pendingSignals.add(signal);
try {
signalChildProcessTree(owned.child, signal, () => owned.pendingSignals.delete(signal));
} catch (error) {
owned.pendingSignals.delete(signal);
throw error;
}
}
function forceStopLocalServiceProcess(owned) {
if (!isLocalServiceProcessTreeAlive(owned)) return;
drainLocalServiceOutput(owned.child);
signalLocalServiceProcess(owned, "SIGKILL");
}
async function stopLocalServiceProcess(owned) {
if (isLocalServiceProcessSettled(owned)) return;
drainLocalServiceOutput(owned.child);
const gracefulDeadline = Date.now() + DEFAULT_PROBE_TIMEOUT_MS$1;
if (owned.pendingSignals.size === 0) signalLocalServiceProcess(owned, "SIGTERM");
await waitForChildProcessTreeExit(owned, gracefulDeadline);
if (!isLocalServiceProcessSettled(owned)) {
const forceDeadline = Date.now() + DEFAULT_PROBE_TIMEOUT_MS$1;
signalLocalServiceProcess(owned, "SIGKILL");
await waitForChildProcessTreeExit(owned, forceDeadline);
}
if (!isLocalServiceProcessSettled(owned)) throw new Error(`Local model service process tree ${owned.child.pid} did not stop`);
}
async function waitForChildProcessTreeExit(owned, deadline) {
while (!isLocalServiceProcessSettled(owned)) {
const remainingMs = deadline - Date.now();
if (remainingMs <= 0) return;
await sleepWithAbort(Math.min(PROCESS_TREE_EXIT_POLL_MS, remainingMs));
}
}
/** Return whether a child process has already reported an exit code or signal. */
function hasLocalServiceProcessExited(child) {
return child.exitCode !== null || child.signalCode !== null;
}
//#endregion
//#region src/agents/provider-local-service-target.ts
function resolveConfiguredProviderLocalServiceTarget(config, target) {
const provider = config.models?.providers?.[target.providerId];
if (!provider?.localService) return;
if (!isConfiguredProviderBaseUrl(target.baseUrl, readConfiguredProviderBaseUrl(provider))) throw new Error(`Local endpoint must match models.providers.${target.providerId}.baseUrl`);
return {
...target,
service: provider.localService
};
}
function readConfiguredProviderBaseUrl(provider) {
const baseUrl = provider?.baseUrl?.trim();
const baseURL = typeof provider?.baseURL === "string" ? provider.baseURL.trim() : "";
return baseUrl || baseURL || void 0;
}
function normalizeProviderBaseUrl(value) {
const trimmed = value.trim();
const candidate = /^[a-z][a-z\d+.-]*:\/\//iu.test(trimmed) ? trimmed : `http://${trimmed}`;
try {
const url = new URL(candidate);
if (url.protocol !== "http:" && url.protocol !== "https:") return;
url.search = "";
url.hash = "";
url.pathname = url.pathname.replace(/\/+$/u, "") || "/";
return url.toString().replace(/\/$/u, "");
} catch {
return;
}
}
function configuredProviderBaseUrlVariants(value) {
const normalized = normalizeProviderBaseUrl(value);
const root = normalized?.replace(/\/v1$/iu, "");
return normalized && root ? /* @__PURE__ */ new Set([
normalized,
root,
`${root}/v1`
]) : /* @__PURE__ */ new Set();
}
function isLoopbackProviderBaseUrl(value) {
const normalized = normalizeProviderBaseUrl(value);
if (!normalized) return false;
const hostname = new URL(normalized).hostname.toLowerCase();
if (hostname === "localhost" || hostname === "[::1]") return true;
return isCanonicalDottedDecimalIPv4(hostname) && isLoopbackIpAddress(hostname);
}
function isConfiguredProviderBaseUrl(targetBaseUrl, configuredBaseUrl) {
const target = normalizeProviderBaseUrl(targetBaseUrl);
if (!target) return false;
const configured = configuredBaseUrl?.trim();
if (!configured) return isLoopbackProviderBaseUrl(target);
return configuredProviderBaseUrlVariants(configured).has(target);
}
//#endregion
//#region src/agents/provider-local-service.ts
/** Manages optional local provider processes; request leases keep shared services alive. */
const log = createSubsystemLogger("provider-local-service");
const DEFAULT_READY_TIMEOUT_MS = 12e4;
const DEFAULT_PROBE_TIMEOUT_MS = 2e3;
const PROBE_INTERVAL_MS = 250;
const MODEL_PROVIDER_LOCAL_SERVICE_SYMBOL = Symbol.for("openclaw.modelProviderLocalService");
const services = /* @__PURE__ */ new Map();
let exitHandlerInstalled = false;
/** Bind local-service acquisition to a host-owned config snapshot. */
function createConfiguredProviderLocalServiceAcquirer(getConfig) {
return async (target, signal) => {
const resolved = resolveConfiguredProviderLocalServiceTarget(getConfig(), target);
return resolved ? await ensureProviderLocalService(resolved, signal) : void 0;
};
}
/** Attach local-service startup metadata to a model without mutating the original object. */
function attachModelProviderLocalService(model, service) {
if (!service) return model;
return {
...model,
[MODEL_PROVIDER_LOCAL_SERVICE_SYMBOL]: service
};
}
/** Read local-service startup metadata attached to a model. */
function getModelProviderLocalService(model) {
return model[MODEL_PROVIDER_LOCAL_SERVICE_SYMBOL];
}
/** Ensure a model's local provider service is healthy and return a lease. */
async function ensureModelProviderLocalService(model, probeHeaders, signal) {
return await ensureProviderLocalService({
providerId: model.provider,
baseUrl: model.baseUrl,
headers: buildHealthProbeHeaders(model.headers, probeHeaders),
service: getModelProviderLocalService(model),
reconcile: getModelProviderLocalServiceReconciler(model)
}, signal);
}
/** Ensure a provider endpoint's local service is healthy and return a request lease. */
async function ensureProviderLocalService(target, signal) {
const lease = await acquireProviderLocalService(target, signal);
if (!lease || !target.reconcile) return lease;
try {
await target.reconcile({
baseUrl: target.baseUrl,
signal: signal ?? void 0
});
throwIfAborted(signal);
} catch (error) {
lease.release();
throw error;
}
return lease;
}
async function acquireProviderLocalService(target, signal) {
const service = target.service;
if (!service) return;
throwIfAborted(signal);
validateLocalServiceConfig(service, target.providerId);
const healthUrl = resolveHealthUrl(service, target.baseUrl);
const healthHeaders = buildHealthProbeHeaders(target.headers);
const key = localServiceKey(target.providerId, service, healthUrl);
installExitHandler();
let current = services.get(key);
while (current?.stopping) {
await waitForAbort(stopManagedService(key, current, "reacquire"), signal);
throwIfAborted(signal);
current = services.get(key);
}
const managed = current ?? { active: 0 };
services.set(key, managed);
setManagedProviderLocalServicesStop(stopManagedProviderLocalServices);
clearIdleTimer(managed);
managed.active += 1;
let released = false;
const release = () => {
if (released) return;
released = true;
managed.active = Math.max(0, managed.active - 1);
scheduleIdleStop(key, managed, service);
};
try {
target.onReadinessWait?.(true);
try {
const currentProcess = managed.process;
const healthy = currentProcess && !managed.processStop && !hasLocalServiceProcessExited(currentProcess.child) ? await probeHealth(healthUrl, healthHeaders, signal) : false;
assertCurrentServiceAcquisition(key, managed, signal);
if (healthy && currentProcess && managed.process === currentProcess && !managed.processStop && !hasLocalServiceProcessExited(currentProcess.child)) return { release };
if (!managed.starting) {
const startupAbort = new AbortController();
managed.startupAbort = startupAbort;
managed.starting = startAndWaitForLocalService({
key,
provider: target.providerId,
service,
healthUrl,
healthHeaders,
managed,
signal: startupAbort.signal
}).finally(() => {
managed.starting = void 0;
if (managed.startupAbort === startupAbort) managed.startupAbort = void 0;
});
}
await waitForAbort(managed.starting, signal);
assertCurrentServiceAcquisition(key, managed, signal);
const ready = !managed.processStop && (managed.process && !hasLocalServiceProcessExited(managed.process.child) || await probeHealth(healthUrl, healthHeaders, signal));
assertCurrentServiceAcquisition(key, managed, signal);
if (ready && !managed.processStop) return { release };
release();
return;
} finally {
target.onReadinessWait?.(false);
}
} catch (error) {
const abortingStartup = isAbortForSignal(error, signal) && Boolean(managed.starting);
release();
if (isAbortForSignal(error, signal)) {
if (abortingStartup && managed.active === 0) {
managed.startupAbort?.abort(toAbortError(signal));
await stopManagedService(key, managed, "startup-aborted");
}
} else await stopManagedService(key, managed, "startup-failed");
throw error;
}
}
function assertCurrentServiceAcquisition(key, managed, signal) {
throwIfAborted(signal);
if (managed.stopping || services.get(key) !== managed) throw new Error("Local model service stopped during acquisition");
}
/** Stop all managed local services owned by this process. */
async function stopManagedProviderLocalServices() {
await Promise.all([...services].map(([key, managed]) => stopManagedService(key, managed, "host-shutdown")));
}
/** Return bounded local-service state for focused lifecycle tests. */
function getManagedProviderLocalServiceDiagnosticsForTest() {
return structuredClone([...services.values()].map((managed) => managed.diagnostics).filter((value) => value !== void 0));
}
function validateLocalServiceConfig(service, provider) {
if (!path.isAbsolute(service.command)) throw new Error(`models.providers.${provider}.localService.command must be an absolute path`);
}
function resolveHealthUrl(service, baseUrl) {
return service.healthUrl?.trim() || `${baseUrl.replace(/\/+$/, "")}/models`;
}
function localServiceKey(provider, service, healthUrl) {
return JSON.stringify({
provider,
command: service.command,
args: service.args ?? [],
cwd: service.cwd ?? "",
envHash: hashStringRecord(service.env),
healthUrl
});
}
function hashStringRecord(record) {
const sorted = Object.entries(record ?? {}).toSorted(([left], [right]) => left.localeCompare(right));
return createHash("sha256").update(JSON.stringify(sorted)).digest("hex");
}
function buildHealthProbeHeaders(...inputs) {
const headers = new Headers();
for (const input of inputs) {
if (!input) continue;
for (const [key, value] of new Headers(input)) if (value.trim().length > 0 && value.trim().toLowerCase() !== "null") headers.set(key, value);
}
return [...headers].length > 0 ? headers : void 0;
}
async function probeHealth(url, headers, signal) {
throwIfAborted(signal);
const egressHeaders = unwrapHeadersInitSentinelsForProviderEgress(headers, "to probe local model provider health");
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), DEFAULT_PROBE_TIMEOUT_MS);
timeout.unref?.();
const onAbort = () => controller.abort(toAbortError(signal));
signal?.addEventListener("abort", onAbort, { once: true });
let response;
try {
response = await fetch(url, {
headers: egressHeaders,
signal: controller.signal
});
return response.ok;
} catch {
if (signal?.aborted) throw toAbortError(signal);
return false;
} finally {
clearTimeout(timeout);
signal?.removeEventListener("abort", onAbort);
await response?.body?.cancel?.().catch(() => void 0);
}
}
async function startAndWaitForLocalService(params) {
const { key, provider, service, healthUrl, healthHeaders, managed, signal } = params;
const healthy = await probeHealth(healthUrl, healthHeaders, signal);
assertCurrentServiceAcquisition(key, managed, signal);
if (healthy) return;
if (managed.process) {
log.info(`restarting unhealthy ${provider} local service`);
await stopManagedProcess(managed, signal);
}
const startedAt = Date.now();
const diagnostics = {
providerId: provider,
healthUrl,
startedAt,
stdoutTail: "",
stderrTail: ""
};
managed.diagnostics = diagnostics;
assertCurrentServiceAcquisition(key, managed, signal);
log.info(`starting ${provider} local service: ${service.command}`);
const serviceEnv = service.env ? mergeProcessEnv([process.env, service.env]) : process.env;
const preparedSpawn = prepareOomScoreAdjustedSpawnPreservingExecEnv(service.command, service.args ?? [], { env: serviceEnv });
const child = spawn(preparedSpawn.command, preparedSpawn.args, {
cwd: service.cwd,
env: preparedSpawn.env,
stdio: [
"ignore",
"pipe",
"pipe"
],
detached: shouldDetachChildForProcessTree()
});
const owned = trackLocalServiceProcess(child);
managed.process = owned;
diagnostics.pid = child.pid;
managed.lastExit = void 0;
for (const [stream, tail] of [[child.stdout, "stdoutTail"], [child.stderr, "stderrTail"]]) {
stream?.setEncoding("utf8");
stream?.on("data", (chunk) => {
diagnostics[tail] = appendLocalServiceOutputTail(diagnostics[tail], chunk, service.env, process.env, service.args, healthHeaders);
});
}
child.unref();
child.once("exit", (code, signalLocal) => {
const exit = {
code,
signal: signalLocal
};
diagnostics.lastExit = exit;
log.info(`${provider} local service exited: ${signalLocal ? `signal=${signalLocal}` : `code=${code ?? 0}`}`);
if (managed.process === owned) managed.lastExit = exit;
});
const spawnError = await waitForSpawnResult(child, signal);
if (spawnError) throw new Error(`${provider} local service failed to start: ${spawnError.message}${formatLocalServiceDiagnosticTail(diagnostics)}`);
diagnostics.spawnedAt = Date.now();
const readyTimeoutMs = resolvePositiveTimerTimeoutMs(service.readyTimeoutMs, DEFAULT_READY_TIMEOUT_MS);
const deadline = Date.now() + readyTimeoutMs;
for (;;) {
if (await probeHealth(healthUrl, healthHeaders, signal)) {
diagnostics.readyAt = Date.now();
diagnostics.lastHealthyAt = diagnostics.readyAt;
diagnostics.stdoutTail = "";
diagnostics.stderrTail = "";
drainLocalServiceOutput(child);
log.info(`${provider} local service ready: pid=${diagnostics.pid ?? "unknown"} spawnMs=${diagnostics.spawnedAt - startedAt} readyMs=${diagnostics.readyAt - startedAt}`);
return;
}
if (managed.lastExit) throw new Error(`${provider} local service exited before readiness with ${formatLocalServiceExit(managed.lastExit)}${formatLocalServiceDiagnosticTail(diagnostics)}`);
if (Date.now() >= deadline) throw new Error(`${provider} local service did not become ready at ${healthUrl}`);
await sleepWithAbort(PROBE_INTERVAL_MS, signal, { ref: false });
}
}
function scheduleIdleStop(key, managed, service) {
if (managed.stopping || services.get(key) !== managed) return;
const idleStopMs = clampPositiveTimerTimeoutMs(service.idleStopMs);
if (managed.active > 0) return;
if (!managed.process) {
if (!managed.starting) {
services.delete(key);
setManagedProviderLocalServicesStop(services.size > 0 ? stopManagedProviderLocalServices : void 0);
}
return;
}
if (idleStopMs === void 0) return;
managed.idleTimer = setTimeout(() => {
if (managed.active === 0) stopManagedService(key, managed, "idle").catch((error) => {
log.warn("idle local model service shutdown failed", { error: toErrorObject(error, "Local model service shutdown failed").message });
});
}, idleStopMs);
managed.idleTimer.unref?.();
}
function clearIdleTimer(managed) {
if (managed.idleTimer) {
clearTimeout(managed.idleTimer);
managed.idleTimer = void 0;
}
}
function stopManagedService(key, managed, reason) {
const stopping = managed.stopping ??= {};
if (stopping.promise) return stopping.promise;
clearIdleTimer(managed);
const starting = managed.starting;
const startupAbort = managed.startupAbort;
managed.startupAbort = void 0;
const pending = Promise.resolve().then(async () => {
startupAbort?.abort(/* @__PURE__ */ new Error(`local service stopped: ${reason}`));
if (managed.process && !hasLocalServiceProcessExited(managed.process.child)) log.info(`stopping local model service: reason=${reason}`);
await starting?.catch(() => {});
await stopManagedProcess(managed, new AbortController().signal);
if (services.get(key) === managed) services.delete(key);
setManagedProviderLocalServicesStop(services.size > 0 ? stopManagedProviderLocalServices : void 0);
}).catch((error) => {
stopping.promise = void 0;
throw error;
});
stopping.promise = pending;
return pending;
}
async function stopManagedProcess(managed, signal) {
throwIfAborted(signal);
if (!managed.processStop) {
const owned = managed.process;
managed.lastExit = void 0;
if (!owned) return;
managed.processStop = { process: owned };
}
const stopping = managed.processStop;
let pending = stopping.promise;
if (!pending) {
pending = Promise.resolve().then(() => stopLocalServiceProcess(stopping.process)).then(() => {
if (managed.process === stopping.process) managed.process = void 0;
if (managed.processStop === stopping) managed.processStop = void 0;
}).catch((error) => {
stopping.promise = void 0;
throw error;
});
stopping.promise = pending;
}
await waitForAbort(pending, signal);
}
function forceStopManagedService(key, managed) {
clearIdleTimer(managed);
const owned = managed.processStop?.process ?? managed.process;
managed.process = void 0;
if (services.get(key) === managed) services.delete(key);
if (owned) forceStopLocalServiceProcess(owned);
}
function installExitHandler() {
if (exitHandlerInstalled) return;
exitHandlerInstalled = true;
process.once("exit", () => {
for (const [key, managed] of services) forceStopManagedService(key, managed);
setManagedProviderLocalServicesStop(void 0);
});
}
function toAbortError(signal) {
if (signal?.reason instanceof Error) return signal.reason;
const error = /* @__PURE__ */ new Error("The operation was aborted.");
error.name = "AbortError";
return error;
}
function throwIfAborted(signal) {
if (signal?.aborted) throw toAbortError(signal);
}
function isAbortForSignal(error, signal) {
return Boolean(signal?.aborted) && (error === signal?.reason || error instanceof Error && error.name === "AbortError");
}
function waitForAbort(promise, signal) {
throwIfAborted(signal);
if (!signal) return promise;
return new Promise((resolve, reject) => {
const onAbort = () => {
cleanup();
reject(toAbortError(signal));
};
const cleanup = () => signal.removeEventListener("abort", onAbort);
signal.addEventListener("abort", onAbort, { once: true });
promise.then((value) => {
cleanup();
resolve(value);
}, (error) => {
cleanup();
reject(toErrorObject(error, "Non-Error rejection"));
});
});
}
function waitForSpawnResult(child, signal) {
throwIfAborted(signal);
return new Promise((resolve) => {
let settled = false;
const finish = (error) => {
if (settled) return;
settled = true;
child.off("error", onError);
child.off("spawn", onSpawn);
signal?.removeEventListener("abort", onAbort);
resolve(error);
};
const onError = (error) => finish(error);
const onSpawn = () => finish();
const onAbort = () => finish(toAbortError(signal));
child.once("error", onError);
child.once("spawn", onSpawn);
signal?.addEventListener("abort", onAbort, { once: true });
setImmediate(() => {
if (child.pid) finish();
});
});
}
//#endregion
export { getManagedProviderLocalServiceDiagnosticsForTest as a, ensureProviderLocalService as i, createConfiguredProviderLocalServiceAcquirer as n, getModelProviderLocalService as o, ensureModelProviderLocalService as r, stopManagedProviderLocalServices as s, attachModelProviderLocalService as t };