UNPKG

openclaw

Version:

Multi-channel AI gateway with extensible messaging integrations

679 lines (678 loc) 23.7 kB
import { w as resolveStateDir } from "./paths-D2sRr1a_.js"; import { t as createSubsystemLogger } from "./subsystem-Dy2tqXOS.js"; import { t as normalizeExactAllowedHost } from "./exact-hostname-B5MIU7_E.js"; import { a as containsSecretSentinel, c as resolveSecretSentinel, i as SECRET_SENTINEL_SUFFIX, n as SECRET_SENTINEL_PATTERN, o as looksLikeSecretSentinel, r as SECRET_SENTINEL_PREFIX } from "./sentinel-ZgmfESkJ.js"; import { r as publishSecretEgressProxy, t as clearSecretEgressProxy } from "./registry-AG1Awcc4.js"; import { n as ensureSecretEgressProxyCa, r as generateLocalProxyLeaf } from "./ca-ZY7QWghJ.js"; import fs from "node:fs"; import { URL } from "node:url"; import path from "node:path"; import net from "node:net"; import { randomBytes, timingSafeEqual } from "node:crypto"; import { Transform } from "node:stream"; import { rootCertificates } from "node:tls"; import { createServer as createServer$1 } from "node:http"; import { Agent, createServer as createServer$2, request as request$1 } from "node:https"; //#region src/secrets/egress-proxy/stream-substitution.ts const SENTINEL_PREFIX_BYTES = Buffer.from(SECRET_SENTINEL_PREFIX); const SENTINEL_SUFFIX_BYTES = Buffer.from(SECRET_SENTINEL_SUFFIX); var SecretEgressSubstitutionError = class extends Error { constructor(reason, details) { super(details ? `Secret "${details.secretName}" is not allowed for host "${details.host}". Run: openclaw secrets store set ${details.secretName} --allow-host ${details.host}` : "Secret egress proxy refused an unresolved secret sentinel"); this.reason = reason; this.details = details; this.name = "SecretEgressSubstitutionError"; } }; function processPendingBuffer(params) { let pending = params.buffer; for (;;) { const prefixIndex = pending.indexOf(SENTINEL_PREFIX_BYTES); if (prefixIndex === -1) { const carryBytes = params.flush ? 0 : Math.min(pending.length, SENTINEL_PREFIX_BYTES.length - 1); const emitBytes = pending.length - carryBytes; if (emitBytes > 0) params.push(pending.subarray(0, emitBytes)); return carryBytes > 0 ? pending.subarray(emitBytes) : Buffer.alloc(0); } if (prefixIndex > 0) { params.push(pending.subarray(0, prefixIndex)); pending = pending.subarray(prefixIndex); } const suffixIndex = pending.indexOf(SENTINEL_SUFFIX_BYTES, SENTINEL_PREFIX_BYTES.length); if (suffixIndex === -1) { if (params.flush || pending.length > 87445) throw new SecretEgressSubstitutionError("unresolved-sentinel"); return pending; } const sentinelEnd = suffixIndex + SENTINEL_SUFFIX_BYTES.length; if (sentinelEnd > 87445) throw new SecretEgressSubstitutionError("unresolved-sentinel"); const sentinel = pending.subarray(0, sentinelEnd).toString("ascii"); const resolved = looksLikeSecretSentinel(sentinel) ? params.resolveSentinel(sentinel) : void 0; if (resolved === void 0) throw new SecretEgressSubstitutionError("unresolved-sentinel"); params.push(Buffer.from(resolved, "utf8")); params.onSubstitution(); pending = pending.subarray(sentinelEnd); } } /** Rewrites process-local sentinels across arbitrary request-body chunk boundaries. */ function createSecretEgressBodyTransform(params) { let pending = Buffer.alloc(0); return new Transform({ transform(chunk, _encoding, callback) { try { const input = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); pending = processPendingBuffer({ buffer: pending.length > 0 ? Buffer.concat([pending, input]) : input, flush: false, onSubstitution: params.onSubstitution, resolveSentinel: params.resolveSentinel, push: (output) => this.push(output) }); callback(); } catch (error) { callback(error); } }, flush(callback) { try { pending = processPendingBuffer({ buffer: pending, flush: true, onSubstitution: params.onSubstitution, resolveSentinel: params.resolveSentinel, push: (output) => this.push(output) }); callback(); } catch (error) { callback(error); } } }); } //#endregion //#region src/secrets/egress-proxy/proxy-server.ts const PROXY_AUTH_USERNAME = "openclaw"; const PROXY_AUTH_REALM = "OpenClaw secret egress"; const REFUSAL_BODY = "Secret egress proxy refused the request.\n"; const UPSTREAM_ERROR_BODY = "Secret egress proxy could not reach the upstream host.\n"; function parseConnectTarget(rawTarget) { const raw = rawTarget?.trim(); if (!raw || /[\r\n]/u.test(raw)) throw new Error("Invalid CONNECT target"); const target = new URL(`https://${raw}`); if (target.pathname !== "/" || target.search || target.hash || target.username || target.password) throw new Error("Invalid CONNECT target"); const port = target.port ? Number(target.port) : 443; if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error("Invalid CONNECT target port"); return { hostname: normalizeExactAllowedHost(target.hostname), port }; } function runKey(run) { return `${run.runId}\0${run.instanceId}`; } function parseProxyToken(token) { if (!/^[A-Za-z0-9_-]{43}$/u.test(token)) return; const bytes = Buffer.from(token, "base64url"); return bytes.length === 32 && bytes.toString("base64url") === token ? bytes : void 0; } function parseBasicProxyPassword(header) { if (typeof header !== "string") return; const match = /^Basic\s+([A-Za-z0-9+/]+={0,2})$/iu.exec(header.trim()); if (!match?.[1]) return; let decoded; try { decoded = Buffer.from(match[1], "base64").toString("utf8"); } catch { return; } const colon = decoded.indexOf(":"); if (colon === -1 || decoded.slice(0, colon) !== PROXY_AUTH_USERNAME) return; return decoded.slice(colon + 1); } function sendProxyAuthRequired(socket) { socket.end(`HTTP/1.1 407 Proxy Authentication Required\r\nProxy-Authenticate: Basic realm="${PROXY_AUTH_REALM}"\r\nConnection: close\r\nContent-Length: ${Buffer.byteLength(REFUSAL_BODY)}\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n${REFUSAL_BODY}`); } function sendHttpRefusal(res, status = 502, body = REFUSAL_BODY) { if (res.destroyed || res.writableEnded) return; if (res.headersSent) { res.destroy(); return; } res.writeHead(status, { Connection: "close", "Content-Length": Buffer.byteLength(body), "Content-Type": "text/plain; charset=utf-8" }); res.end(body); } function resolveRegisteredSentinel(params) { if (!params.registered.isActive()) return; const binding = params.registered.sentinelBindings.get(params.sentinel); if (!binding) return; if (!binding.allowedHosts.has(params.host)) throw new SecretEgressSubstitutionError("destination-not-allowed", { host: params.host, secretName: binding.name }); return resolveSecretSentinel(params.sentinel); } function swapRequestText(params) { if (!containsSecretSentinel(params.value)) return { value: params.value, substituted: false }; let substituted = false; const swapped = params.value.replace(new RegExp(SECRET_SENTINEL_PATTERN.source, "g"), (sentinel) => { const resolved = resolveRegisteredSentinel({ sentinel, host: params.host, registered: params.registered }); if (resolved === void 0) return sentinel; substituted = true; return params.urlMode ? encodeURIComponent(resolved) : resolved; }); if (containsSecretSentinel(swapped)) throw new SecretEgressSubstitutionError("unresolved-sentinel"); return { value: swapped, substituted }; } function swapRequestHeaders(params) { const output = {}; let substituted = false; for (const [name, rawValue] of Object.entries(params.headers)) { const lowerName = name.toLowerCase(); if (lowerName === "proxy-authorization" || lowerName === "proxy-connection") continue; if (Array.isArray(rawValue)) { output[name] = rawValue.map((value) => { const swapped = swapRequestText({ value, urlMode: false, host: params.host, registered: params.registered }); substituted ||= swapped.substituted; return swapped.value; }); continue; } if (rawValue !== void 0) { const swapped = swapRequestText({ value: rawValue, urlMode: false, host: params.host, registered: params.registered }); substituted ||= swapped.substituted; output[name] = swapped.value; } } delete output["content-length"]; delete output["transfer-encoding"]; return { headers: output, substituted }; } /** Starts one authenticated, loopback-only substitution proxy. */ async function startSecretEgressProxyServer(params) { const ca = await ensureSecretEgressProxyCa(params.caDir); const caPem = fs.readFileSync(ca.certPath, "utf8"); const trustBundlePath = path.join(params.caDir, "trust-bundle.pem"); fs.writeFileSync(trustBundlePath, `${rootCertificates.join("\n")}\n${caPem}`, { mode: 420 }); const upstreamTlsAgent = new Agent({ ca: [...rootCertificates, caPem] }); const bypassHosts = new Set((params.bypassHosts ?? []).map(normalizeExactAllowedHost)); const allowedHosts = params.allowedHosts === void 0 ? void 0 : new Set(params.allowedHosts.map(normalizeExactAllowedHost)); const registrations = /* @__PURE__ */ new Map(); const sockets = /* @__PURE__ */ new Set(); const preparations = /* @__PURE__ */ new Set(); let stopped = false; let stopPromise; const ownResource = (registered, resource) => { if (!registered.resources.has(resource)) { registered.resources.add(resource); resource.once("close", () => registered.resources.delete(resource)); resource.on("error", () => resource.destroy()); } if (!registered.isActive()) resource.destroy(); return resource; }; const revokeRegistration = (registered) => { registrations.delete(registered.key); registered.sentinelBindings.clear(); for (const resource of registered.resources) resource.destroy(); registered.resources.clear(); for (const server of registered.tlsServers.values()) server.then((ready) => ready?.close(), () => {}); registered.tlsServers.clear(); }; const audit = (event) => params.onAudit(event); const hostAllowed = (host, registered) => { if (allowedHosts === void 0 || allowedHosts.has(host) || bypassHosts.has(host)) return true; for (const binding of registered.sentinelBindings.values()) if (binding.allowedHosts.has(host)) return true; return false; }; const hostNotAllowedBody = (host) => `Host "${host}" is not in the secret egress proxy traffic allowlist. Add it to secrets.egressProxy.allowedHosts or bind a store secret to it with: openclaw secrets store set <NAME> --allow-host ${host}, then restart the Gateway.\n`; const authorize = (headers) => { const rawHeader = headers["proxy-authorization"]; if (rawHeader === void 0) return "missing-proxy-auth"; const password = parseBasicProxyPassword(rawHeader); if (!password) return "invalid-proxy-auth"; const candidate = parseProxyToken(password); if (!candidate) return "invalid-proxy-auth"; for (const registered of registrations.values()) if (timingSafeEqual(candidate, registered.token)) return registered; return "invalid-proxy-auth"; }; const parseRequestTarget = (request, response, base) => { try { const target = new URL(request.url ?? "/", base); return { target, host: normalizeExactAllowedHost(target.hostname) }; } catch { audit({ kind: "refused", host: "unknown", substituted: false, reason: "upstream-error" }); sendHttpRefusal(response, 400); request.resume(); return; } }; const forwardRequest = (forward) => { ownResource(forward.registered, forward.request); ownResource(forward.registered, forward.response); if (!forward.registered.isActive()) return; const { host } = forward; if (forward.target.protocol !== "https:") { audit({ kind: "refused", host, substituted: false, reason: "non-https-request" }); sendHttpRefusal(forward.response); forward.request.resume(); return; } if (!hostAllowed(host, forward.registered)) { audit({ kind: "refused", host, substituted: false, reason: "host-not-allowed" }); sendHttpRefusal(forward.response, 403, hostNotAllowedBody(host)); forward.request.resume(); return; } let substituted = false; let target; let headers; try { const swappedUrl = swapRequestText({ value: forward.target.toString(), urlMode: true, host, registered: forward.registered }); target = new URL(swappedUrl.value); const swappedHeaders = swapRequestHeaders({ headers: forward.request.headers, host, registered: forward.registered }); headers = swappedHeaders.headers; headers.host = target.host; substituted = swappedUrl.substituted || swappedHeaders.substituted; } catch (error) { const reason = error instanceof SecretEgressSubstitutionError ? error.reason : "unresolved-sentinel"; audit({ kind: "refused", host, substituted, reason }); sendHttpRefusal(forward.response, 502, error instanceof SecretEgressSubstitutionError ? `${error.message}\n` : REFUSAL_BODY); forward.request.resume(); return; } const bodyTransform = ownResource(forward.registered, createSecretEgressBodyTransform({ onSubstitution: () => { substituted = true; }, resolveSentinel: (sentinel) => resolveRegisteredSentinel({ sentinel, host, registered: forward.registered }) })); let refused = false; const upstream = ownResource(forward.registered, request$1({ hostname: target.hostname, port: target.port || 443, path: `${target.pathname}${target.search}`, method: forward.request.method, headers, agent: upstreamTlsAgent }, (upstreamResponse) => { ownResource(forward.registered, upstreamResponse); if (refused || !forward.registered.isActive()) { upstreamResponse.destroy(); return; } upstreamResponse.once("error", () => forward.response.destroy()); forward.response.writeHead(upstreamResponse.statusCode ?? 502, upstreamResponse.headers); upstreamResponse.pipe(forward.response); })); forward.request.once("error", () => forward.response.destroy()); forward.response.once("close", () => { refused = true; forward.request.unpipe(bodyTransform); bodyTransform.destroy(); upstream.destroy(); }); bodyTransform.once("finish", () => { if (!refused && forward.registered.isActive()) audit({ kind: "forwarded", host, substituted }); }); bodyTransform.once("error", (error) => { if (refused || !forward.registered.isActive()) return; refused = true; forward.request.unpipe(bodyTransform); forward.request.resume(); upstream.destroy(); const reason = error instanceof SecretEgressSubstitutionError ? error.reason : "unresolved-sentinel"; audit({ kind: "refused", host, substituted, reason }); sendHttpRefusal(forward.response, 502, error instanceof SecretEgressSubstitutionError ? `${error.message}\n` : REFUSAL_BODY); }); upstream.once("error", () => { if (refused || !forward.registered.isActive()) return; refused = true; audit({ kind: "refused", host, substituted, reason: "upstream-error" }); sendHttpRefusal(forward.response, 502, UPSTREAM_ERROR_BODY); }); forward.request.pipe(bodyTransform).pipe(upstream); }; const tlsServerFor = (target, registered) => { const key = `${target.hostname}:${target.port}`; let server = registered.tlsServers.get(key); if (!server) { server = generateLocalProxyLeaf({ certDir: params.caDir, ca, hostname: target.hostname }).then((leaf) => { if (!registered.isActive()) return; return createServer$2(leaf, (request, response) => { const parsed = parseRequestTarget(request, response, `https://${target.hostname}${target.port === 443 ? "" : `:${target.port}`}`); if (parsed) forwardRequest({ request, response, ...parsed, registered }); }).on("secureConnection", (socket) => ownResource(registered, socket)); }); registered.tlsServers.set(key, server); preparations.add(server); const prepared = server; prepared.then(() => preparations.delete(prepared), () => preparations.delete(prepared)); } return server; }; const proxy = createServer$1((request, response) => { const parsed = parseRequestTarget(request, response); if (!parsed) return; const { host } = parsed; const authorization = authorize(request.headers); if (typeof authorization === "string") { audit({ kind: "refused", host, substituted: false, reason: authorization }); response.writeHead(407, { "Proxy-Authenticate": `Basic realm="${PROXY_AUTH_REALM}"`, Connection: "close", "Content-Length": Buffer.byteLength(REFUSAL_BODY), "Content-Type": "text/plain; charset=utf-8" }); response.end(REFUSAL_BODY); request.resume(); return; } forwardRequest({ request, response, ...parsed, registered: authorization }); }); proxy.on("connection", (socket) => { sockets.add(socket); socket.once("close", () => sockets.delete(socket)); socket.on("error", () => { socket.destroy(); }); }); proxy.on("connect", (request, clientSocket, head) => { (async () => { let target; try { target = parseConnectTarget(request.url); } catch { audit({ kind: "refused", host: "unknown", substituted: false, reason: "upstream-error" }); clientSocket.end("HTTP/1.1 400 Bad Request\r\nConnection: close\r\n\r\n"); return; } const authorization = authorize(request.headers); if (typeof authorization === "string") { audit({ kind: "refused", host: target.hostname, substituted: false, reason: authorization }); sendProxyAuthRequired(clientSocket); return; } ownResource(authorization, clientSocket); if (bypassHosts.has(target.hostname)) { const upstream = ownResource(authorization, net.connect(target.port, target.hostname, () => { if (!authorization.isActive() || clientSocket.destroyed) { upstream.destroy(); return; } clientSocket.write("HTTP/1.1 200 Connection Established\r\n\r\n"); if (head.length > 0) upstream.write(head); clientSocket.pipe(upstream).pipe(clientSocket); audit({ kind: "forwarded", host: target.hostname, substituted: false, reason: "bypass" }); })); clientSocket.once("close", () => upstream.destroy()); upstream.once("close", () => clientSocket.destroy()); upstream.once("error", () => clientSocket.destroy()); return; } if (!hostAllowed(target.hostname, authorization)) { const body = hostNotAllowedBody(target.hostname); audit({ kind: "refused", host: target.hostname, substituted: false, reason: "host-not-allowed" }); clientSocket.end(`HTTP/1.1 403 Forbidden\r\nConnection: close\r\nContent-Length: ${Buffer.byteLength(body)}\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n${body}`); return; } try { const tlsServer = await tlsServerFor(target, authorization); if (!tlsServer || !authorization.isActive() || clientSocket.destroyed) { clientSocket.destroy(); return; } clientSocket.write("HTTP/1.1 200 Connection Established\r\n\r\n"); if (head.length > 0) clientSocket.unshift(head); tlsServer.emit("connection", clientSocket); } catch { if (!authorization.isActive() || clientSocket.destroyed) return; audit({ kind: "refused", host: target.hostname, substituted: false, reason: "upstream-error" }); clientSocket.end("HTTP/1.1 502 Bad Gateway\r\nConnection: close\r\n\r\n"); } })(); }); await new Promise((resolve, reject) => { proxy.once("error", reject); proxy.listen(0, "127.0.0.1", () => { proxy.off("error", reject); resolve(); }); }); const address = proxy.address(); if (!address || typeof address === "string") throw new Error("Secret egress proxy failed to bind loopback"); const proxyOrigin = `http://127.0.0.1:${address.port}`; return { caCertPath: ca.certPath, proxyOrigin, registerRun: (run, bindings = []) => { if (stopped) throw new Error("Secret egress proxy has stopped"); const key = runKey(run); let registered = registrations.get(key); if (!registered) { registered = { key, sentinelBindings: /* @__PURE__ */ new Map(), token: randomBytes(32), isActive: () => !stopped && registrations.get(key) === registered, resources: /* @__PURE__ */ new Set(), tlsServers: /* @__PURE__ */ new Map() }; registrations.set(key, registered); } registered.sentinelBindings = new Map(bindings.map((binding) => [binding.sentinel, { allowedHosts: new Set(binding.allowedHosts.map(normalizeExactAllowedHost)), name: binding.name }])); const token = registered.token.toString("base64url"); const proxyUrl = `http://${PROXY_AUTH_USERNAME}:${token}@127.0.0.1:${address.port}`; return { HTTPS_PROXY: proxyUrl, HTTP_PROXY: proxyUrl, NODE_USE_ENV_PROXY: "1", NODE_EXTRA_CA_CERTS: trustBundlePath, SSL_CERT_FILE: trustBundlePath, CURL_CA_BUNDLE: trustBundlePath, REQUESTS_CA_BUNDLE: trustBundlePath }; }, revokeRun: (run) => { const registered = registrations.get(runKey(run)); if (registered) revokeRegistration(registered); }, stop: () => { if (stopPromise) return stopPromise; stopped = true; for (const registered of registrations.values()) revokeRegistration(registered); upstreamTlsAgent.destroy(); for (const socket of sockets) socket.destroy(); sockets.clear(); stopPromise = Promise.all([new Promise((resolve) => { proxy.close(() => resolve()); }), Promise.allSettled(preparations)]).then(() => {}); return stopPromise; } }; } //#endregion //#region src/secrets/egress-proxy/runtime.ts const log = createSubsystemLogger("secrets/egress-proxy"); const SECRET_EGRESS_PROXY_DIR_MODE = 448; function removeProxyDirBestEffort(proxyDir) { try { fs.rmSync(proxyDir, { recursive: true, force: true }); fs.rmdirSync(path.dirname(proxyDir)); } catch {} } function removeStaleProxyDirs(parentDir) { for (const entry of fs.readdirSync(parentDir, { withFileTypes: true })) if (entry.isDirectory() && entry.name.startsWith("gateway-")) fs.rmSync(path.join(parentDir, entry.name), { recursive: true, force: true }); } /** Starts the process-local proxy and registers it as the current Gateway owner. */ async function startGatewaySecretEgressProxy(params) { const parentDir = path.join(resolveStateDir(), "secret-egress-proxy"); fs.mkdirSync(parentDir, { recursive: true, mode: SECRET_EGRESS_PROXY_DIR_MODE }); fs.chmodSync(parentDir, SECRET_EGRESS_PROXY_DIR_MODE); removeStaleProxyDirs(parentDir); const proxyDir = fs.mkdtempSync(path.join(parentDir, "gateway-")); fs.chmodSync(proxyDir, SECRET_EGRESS_PROXY_DIR_MODE); let proxy; try { proxy = await startSecretEgressProxyServer({ caDir: proxyDir, ...params.allowedHosts !== void 0 ? { allowedHosts: params.allowedHosts } : {}, ...params.bypassHosts ? { bypassHosts: params.bypassHosts } : {}, onAudit: (event) => log.info("secret egress request", event) }); const ownedProxy = proxy; const cleanupOnProcessExit = () => removeProxyDirBestEffort(proxyDir); process.once("exit", cleanupOnProcessExit); const handle = { ...ownedProxy, stop: async () => { clearSecretEgressProxy(handle); process.off("exit", cleanupOnProcessExit); try { await ownedProxy.stop(); } finally { removeProxyDirBestEffort(proxyDir); } } }; publishSecretEgressProxy(handle); return handle; } catch (error) { await proxy?.stop().catch(() => void 0); removeProxyDirBestEffort(proxyDir); throw error; } } //#endregion export { startGatewaySecretEgressProxy };