UNPKG

openclaw

Version:

Multi-channel AI gateway with extensible messaging integrations

565 lines (564 loc) 25.5 kB
import { c as isPathInside } from "./fs-safe-B6pvPGnf.js"; import { w as resolveStateDir } from "./paths-D2sRr1a_.js"; import { i as registerSecretValueForRedaction } from "./secret-redaction-registry-BOLC6DkF.js"; import { i as generateSecureToken } from "./secure-random-Ds4AFLgz.js"; import { t as KeyedAsyncQueue } from "./keyed-async-queue-CTreGrmR.js"; import { r as runCommandWithTimeout } from "./exec-BIE-3oLG.js"; import { C as parseWorkerWorkspaceManifest, I as MAX_WORKSPACE_INVENTORY_TOTAL_BYTES, L as MAX_WORKSPACE_MANIFEST_BYTES, T as serializeWorkerWorkspaceManifest, r as readWorkspaceFileSnapshotWithLimit, x as MAX_RECONCILIATION_TOTAL_BYTES, y as MAX_RECONCILIATION_ENTRIES } from "./workspace-actual-manifest-BGixPzCq.js"; import { d as readActualWorkspaceManifest, o as assertWorkspaceMatchesManifest } from "./workspace-reconcile-U3HlbfL4.js"; import { f as workerWorkspaceTransferPaths } from "./workspace-result-staging-CwYxWnxD.js"; import { o as probeWorkspaceGitMode } from "./workspace-sync-helpers-DLkfl3bZ.js"; import { i as readWorkspaceTransferPaths, t as createWorkspaceGitTransferList } from "./workspace-sync-inventory-YnS0EYGS.js"; import { n as prepareWorkerWorkspaceGitPack } from "./workspace-git-base-qAMhycVO.js"; import path from "node:path"; import fs from "node:fs/promises"; import { createHash } from "node:crypto"; //#region src/gateway/worker-environments/node-workspace-transfer-snapshot.ts const TRANSFER_TIMEOUT_MS$1 = 6e5; async function prepareNodeWorkspaceTransferSnapshot(params) { const root = await fs.realpath(params.localPath); const git = await probeWorkspaceGitMode({ localPath: root, commandOptions: { timeoutMs: TRANSFER_TIMEOUT_MS$1, maxOutputBytes: 262144, maxCombinedOutputBytes: 524288, baseEnv: { ...process.env, GIT_TERMINAL_PROMPT: "0", GIT_ASKPASS: "" }, signal: params.signal }, runTask: runCommandWithTimeout }); let baseCommit = null; let includePaths; if (git.mode === "git") { if (await fs.realpath(git.gitRoot) !== root) throw new Error("Worker git workspace sync requires the managed worktree root"); baseCommit = git.baseCommit; if (!/^[a-f0-9]{40}(?:[a-f0-9]{24})?$/u.test(baseCommit)) throw new Error("Worker workspace Git base is not a commit id"); const transferList = await createWorkspaceGitTransferList({ gitRoot: root, temporaryDirectory: path.join(params.temporaryRoot, "inventory"), signal: params.signal ?? AbortSignal.timeout(TRANSFER_TIMEOUT_MS$1), timeoutMs: TRANSFER_TIMEOUT_MS$1 }); const transferable = await readWorkspaceTransferPaths(transferList); const manifestPaths = new Set(transferable); for (const relative of transferable) { const segments = relative.split("/"); for (let index = 1; index < segments.length; index += 1) manifestPaths.add(segments.slice(0, index).join("/")); } includePaths = manifestPaths; } const actual = await readActualWorkspaceManifest({ root, baseCommit, includePaths }); return { ...actual, rawManifest: serializeWorkerWorkspaceManifest(actual.manifest), root }; } //#endregion //#region src/gateway/worker-environments/node-workspace-transfer-token.ts const NODE_WORKSPACE_TRANSFER_TOKEN_BYTES = 32; const TOKEN_PATTERN = /^[A-Za-z0-9_-]{43}$/u; /** Mint one process-local bearer. Its owner stores every authority binding separately. */ function mintNodeWorkspaceTransferToken(generateToken = generateSecureToken) { const token = generateToken(NODE_WORKSPACE_TRANSFER_TOKEN_BYTES); if (!TOKEN_PATTERN.test(token)) throw new Error("Workspace transfer token generator returned an invalid bearer"); registerSecretValueForRedaction(token); return token; } //#endregion //#region src/gateway/worker-environments/node-workspace-upload-reader.ts const MAX_UPLOAD_BYTES = MAX_WORKSPACE_MANIFEST_BYTES * 2 + MAX_RECONCILIATION_TOTAL_BYTES + MAX_RECONCILIATION_ENTRIES * 8 + 8; var NodeWorkspaceTransferLimitError = class extends Error { constructor(..._args) { super(..._args); this.code = "workspace-transfer-limit"; } }; function isNodeWorkspaceTransferLimitError(error) { return error instanceof NodeWorkspaceTransferLimitError; } var NodeWorkspaceTransferInvalidError = class extends Error { constructor(reason, message, options) { super(message, options); this.reason = reason; this.code = "workspace-transfer-invalid"; } }; function nodeWorkspaceTransferInvalidReason(error) { return error instanceof NodeWorkspaceTransferInvalidError ? error.reason : void 0; } var RequestByteReader = class { #iterator; #signal; #assertCurrent; #pending; #done; constructor(request, signal, assertCurrent) { this.#pending = Buffer.alloc(0); this.#done = false; this.bytesRead = 0; this.#iterator = request[Symbol.asyncIterator](); this.#signal = signal; this.#assertCurrent = assertCurrent; } async take(maxBytes) { this.#signal.throwIfAborted(); if (this.#pending.length === 0 && !this.#done) { const next = await this.#iterator.next(); this.#assertCurrent(); this.#signal.throwIfAborted(); this.#done = Boolean(next.done); if (!next.done) { if (!Buffer.isBuffer(next.value)) throw new NodeWorkspaceTransferInvalidError("payload", "Workspace transfer upload must contain binary data"); this.#pending = next.value; } } if (this.#pending.length === 0) return Buffer.alloc(0); const count = Math.min(maxBytes, this.#pending.length); const value = this.#pending.subarray(0, count); this.#pending = count === this.#pending.length ? Buffer.alloc(0) : this.#pending.subarray(count); this.bytesRead += value.byteLength; if (this.bytesRead > MAX_UPLOAD_BYTES) throw new NodeWorkspaceTransferLimitError("Workspace transfer upload exceeds its byte limit"); return value; } async readExactly(bytes) { const chunks = []; let remaining = bytes; while (remaining > 0) { const chunk = await this.take(remaining); if (chunk.length === 0) throw new NodeWorkspaceTransferInvalidError("premature_eof", "Workspace transfer upload ended before its declared payload"); chunks.push(chunk); remaining -= chunk.length; } return Buffer.concat(chunks, bytes); } async assertEnd() { if ((await this.take(1)).length !== 0) throw new NodeWorkspaceTransferInvalidError("trailing_bytes", "Workspace transfer upload contains trailing bytes"); } }; async function streamUploadFile(params) { if ((await params.reader.readExactly(8)).readBigUInt64BE() !== BigInt(params.entry.size)) throw new NodeWorkspaceTransferInvalidError("file_size", "Workspace transfer file size differs from its manifest"); const hash = createHash("sha256"); let offset = 0; while (offset < params.entry.size) { const chunk = await params.reader.take(Math.min(65536, params.entry.size - offset)); if (chunk.length === 0) throw new NodeWorkspaceTransferInvalidError("premature_eof", "Workspace transfer upload ended mid-file"); hash.update(chunk); let chunkOffset = 0; while (chunkOffset < chunk.length) { const { bytesWritten } = await params.handle.write(chunk, chunkOffset, chunk.length - chunkOffset, offset + chunkOffset); params.assertCurrent(); if (bytesWritten === 0) throw new Error("Workspace transfer upload write made no progress"); chunkOffset += bytesWritten; } offset += chunk.length; } if (hash.digest("hex") !== params.entry.sha256) throw new NodeWorkspaceTransferInvalidError("file_digest", "Workspace transfer file digest differs from its manifest"); } //#endregion //#region src/gateway/worker-environments/node-workspace-transfer-service.ts const TRANSFER_TIMEOUT_MS = 6e5; const MANIFEST_REF_PATTERN = /^sha256:[a-f0-9]{64}$/u; function contextOwnerValid(context, owner) { const environment = owner?.environment; const credential = owner?.credential; return Boolean(!context.abortController.signal.aborted && context.isAuthorized() && environment && credential && environment.state === "attached" && environment.destroyRequestedAtMs === null && environment.ownerEpoch === context.ownerEpoch && environment.attachedSessionIds.length === 1 && environment.attachedSessionIds[0] === context.sessionId && credential.ownerEpoch === context.ownerEpoch && credential.sessionId === context.sessionId); } function capabilityMatchesContext(capability, context) { return capability.environmentId === context.environmentId && capability.ownerEpoch === context.ownerEpoch && capability.sessionId === context.sessionId && capability.generation === context.generation; } function entryPath(root, relative) { const candidate = path.join(root, ...relative.split("/")); if (candidate !== root && !isPathInside(root, candidate)) throw new Error("Workspace transfer entry escaped its staging root"); return candidate; } function createNodeWorkspaceTransferService(options) { const contexts = /* @__PURE__ */ new Map(); const contextOperations = new KeyedAsyncQueue(); const now = options.now ?? Date.now; const temporaryBaseRoot = options.temporaryRoot ?? path.join(resolveStateDir(), "tmp", "node-workspace-transfer"); let temporaryRootReady; const ensureTemporaryRoot = () => { temporaryRootReady ??= (async () => { await fs.rm(temporaryBaseRoot, { recursive: true, force: true }); await fs.mkdir(temporaryBaseRoot, { recursive: true, mode: 448 }); })(); return temporaryRootReady; }; const isCurrentContext = (context) => contexts.get(context.environmentId) === context && contextOwnerValid(context, options.getOwner(context.environmentId)); const closeContext = async (context) => { if (!context.abortController.signal.aborted) context.abortController.abort(/* @__PURE__ */ new Error("Node workspace transfer context closed")); context.stopWatchingOwnerSignal?.(); if (contexts.get(context.environmentId) === context) contexts.delete(context.environmentId); await context.pack?.catch(() => void 0); await fs.rm(context.temporaryRoot, { recursive: true, force: true }); }; const closeEnvironment = (environmentId) => contextOperations.enqueue(environmentId, async () => { const context = contexts.get(environmentId); if (context) await closeContext(context); }); const mintDownload = (context, manifestRef, isAuthorized, signal) => { signal?.throwIfAborted(); if (!isCurrentContext(context) || isAuthorized?.() === false) throw new Error("Node workspace transfer owner is no longer current"); const token = mintNodeWorkspaceTransferToken(); context.downloads.set(token, { direction: "download", token, environmentId: context.environmentId, ownerEpoch: context.ownerEpoch, sessionId: context.sessionId, generation: context.generation, manifestRef, expiresAtMs: now() + TRANSFER_TIMEOUT_MS, ...isAuthorized ? { isAuthorized } : {}, ...signal ? { signal } : {} }); return token; }; const pruneSnapshots = (context) => { const retained = /* @__PURE__ */ new Set([context.currentManifestRef, ...[...context.downloads.values()].map((download) => download.manifestRef)]); for (const manifestRef of context.snapshots.keys()) if (!retained.has(manifestRef)) context.snapshots.delete(manifestRef); }; const authorizationCurrent = (authorization) => { const { capability, context } = authorization; if (!isCurrentContext(context) || !capabilityMatchesContext(capability, context) || capability.expiresAtMs <= now()) return false; return capability.direction === "download" ? context.downloads.get(capability.token) === capability && !capability.signal?.aborted && capability.isAuthorized?.() !== false : context.upload === capability && (capability.state === "receiving" || capability.state === "completed"); }; const assertAuthorizationCurrent = (authorization) => { if (!authorizationCurrent(authorization)) throw new Error("Workspace transfer authority closed"); }; const routeMatchesDownload = (context, capability, route) => { if (route.direction !== "download" || route.environmentId !== context.environmentId) return false; return route.kind === "blob" ? Boolean(context.snapshots.get(capability.manifestRef)?.manifest.entries.some((entry) => entry.type === "file" && entry.sha256 === route.sha256)) : route.manifestRef === capability.manifestRef; }; return { initialize: ensureTemporaryRoot, async prepareAttachments(params) { params.signal.throwIfAborted(); const context = contexts.get(params.environmentId); if (!context || !isCurrentContext(context) || !params.isAuthorized()) throw new Error("Worker attachment transfer authority closed"); const root = await fs.realpath(params.localPath); params.signal.throwIfAborted(); const actual = await readActualWorkspaceManifest({ root, baseCommit: null }); params.signal.throwIfAborted(); if (!isCurrentContext(context) || !params.isAuthorized()) throw new Error("Worker attachment transfer authority closed"); const snapshot = { ...actual, root, rawManifest: serializeWorkerWorkspaceManifest(actual.manifest) }; context.snapshots.set(snapshot.manifestRef, snapshot); return { snapshot, token: mintDownload(context, snapshot.manifestRef, params.isAuthorized, params.signal) }; }, async prepareSync(params) { return await contextOperations.enqueue(params.environmentId, async () => { const previous = contexts.get(params.environmentId); if (previous) await closeContext(previous); await ensureTemporaryRoot(); const abortController = new AbortController(); const context = { ...params, localPath: await fs.realpath(params.localPath), temporaryRoot: await fs.mkdtemp(path.join(temporaryBaseRoot, "context-")), currentManifestRef: "", snapshots: /* @__PURE__ */ new Map(), baseCommit: null, downloads: /* @__PURE__ */ new Map(), abortController }; if (params.signal) { const abortFromOwner = () => abortController.abort(params.signal.reason); params.signal.addEventListener("abort", abortFromOwner, { once: true }); context.stopWatchingOwnerSignal = () => params.signal?.removeEventListener("abort", abortFromOwner); if (params.signal.aborted) abortFromOwner(); } try { const snapshot = await prepareNodeWorkspaceTransferSnapshot({ localPath: context.localPath, temporaryRoot: context.temporaryRoot, signal: AbortSignal.any([context.abortController.signal, AbortSignal.timeout(TRANSFER_TIMEOUT_MS)]) }); context.snapshots.set(snapshot.manifestRef, snapshot); context.baseCommit = snapshot.manifest.baseCommit; context.currentManifestRef = snapshot.manifestRef; contexts.set(context.environmentId, context); return { snapshot, token: mintDownload(context, snapshot.manifestRef) }; } catch (error) { await closeContext(context); throw error; } }); }, prepareUpload(environmentId, baseManifestRef) { const context = contexts.get(environmentId); if (!context || !MANIFEST_REF_PATTERN.test(baseManifestRef) || !isCurrentContext(context)) throw new Error("Node workspace transfer context is unavailable"); if (context.upload) throw new Error("Node workspace transfer upload is already active"); const token = mintNodeWorkspaceTransferToken(); context.upload = { direction: "upload", token, environmentId: context.environmentId, ownerEpoch: context.ownerEpoch, sessionId: context.sessionId, generation: context.generation, baseManifestRef, expiresAtMs: now() + TRANSFER_TIMEOUT_MS, state: "ready" }; return token; }, takeUpload(environmentId, baseManifestRef) { const context = contexts.get(environmentId); const operation = context?.upload; if (!context || !operation || operation.state !== "completed" || operation.baseManifestRef !== baseManifestRef || !operation.uploaded || !isCurrentContext(context)) throw new Error("Node workspace transfer upload did not complete"); context.upload = void 0; return operation.uploaded; }, getSnapshot(environmentId, manifestRef) { return contexts.get(environmentId)?.snapshots.get(manifestRef); }, publishSnapshot(environmentId, snapshot) { const context = contexts.get(environmentId); if (!context || !isCurrentContext(context)) throw new Error("Node workspace transfer context is unavailable"); context.snapshots.set(snapshot.manifestRef, snapshot); context.currentManifestRef = snapshot.manifestRef; pruneSnapshots(context); return mintDownload(context, snapshot.manifestRef); }, revoke(environmentId, token) { const context = contexts.get(environmentId); context?.downloads.delete(token); if (context) pruneSnapshots(context); if (context?.upload?.token === token && context.upload.state === "ready") context.upload = void 0; }, authorize(params) { const context = contexts.get(params.route.environmentId); if (!context) return; const download = context.downloads.get(params.token); if (download) { const authorization = { context, capability: download, route: params.route }; if (!authorizationCurrent(authorization) || !routeMatchesDownload(context, download, params.route)) return; return authorization; } const upload = context.upload; if (!upload || !isCurrentContext(context) || upload.token !== params.token || upload.state !== "ready" || upload.expiresAtMs <= now() || !capabilityMatchesContext(upload, context) || params.route.kind !== "reconcile" || params.route.environmentId !== context.environmentId || params.route.baseManifestRef !== upload.baseManifestRef) return; upload.state = "receiving"; return { context, capability: upload, route: params.route }; }, isAuthorizationCurrent: authorizationCurrent, authorizationSignal(authorization) { const signal = authorization.context.abortController.signal; const capability = authorization.capability; return capability.direction === "download" && capability.signal ? AbortSignal.any([signal, capability.signal]) : signal; }, snapshot(authorization) { if (authorization.capability.direction !== "download" || authorization.route.kind !== "manifest" && authorization.route.kind !== "pack" || !authorizationCurrent(authorization)) return; return authorization.context.snapshots.get(authorization.capability.manifestRef); }, async pack(authorization) { if (authorization.route.kind !== "pack" || !authorizationCurrent(authorization)) return; const { context, route } = authorization; const snapshot = context.snapshots.get(route.manifestRef); const { baseCommit } = context; if (!baseCommit || snapshot?.manifest.baseCommit !== baseCommit) return; if (!context.pack) context.pack = prepareWorkerWorkspaceGitPack({ root: context.localPath, baseCommit, temporaryRoot: context.temporaryRoot, signal: AbortSignal.any([context.abortController.signal, AbortSignal.timeout(TRANSFER_TIMEOUT_MS)]) }).catch((error) => { context.pack = void 0; throw error; }); const packPath = await context.pack; return authorizationCurrent(authorization) ? packPath : void 0; }, blob(authorization) { if (authorization.capability.direction !== "download" || authorization.route.kind !== "blob" || !authorizationCurrent(authorization)) return; const snapshot = authorization.context.snapshots.get(authorization.capability.manifestRef); const sha256 = authorization.route.sha256; const entry = snapshot?.manifest.entries.find((candidate) => candidate.type === "file" && candidate.sha256 === sha256); return snapshot && entry?.type === "file" ? { path: entryPath(snapshot.root, entry.path), size: entry.size, sha256: entry.sha256 } : void 0; }, async receiveUpload(params) { const { authorization } = params; const operation = authorization.capability; if (operation.direction !== "upload" || authorization.route.kind !== "reconcile" || operation.state !== "receiving") throw new Error("Workspace transfer upload owner is unavailable"); const assertCurrent = () => { params.signal.throwIfAborted(); assertAuthorizationCurrent(authorization); }; let stagingRoot; try { assertCurrent(); const contentLength = Number(params.request.headers["content-length"]); if (!Number.isSafeInteger(contentLength) || contentLength < 8 || contentLength > MAX_UPLOAD_BYTES) throw new NodeWorkspaceTransferLimitError("Workspace transfer upload exceeds its byte limit"); const reader = new RequestByteReader(params.request, params.signal, assertCurrent); const readManifest = async (expectedRef) => { const bytes = (await reader.readExactly(4)).readUInt32BE(); if (bytes < 2 || bytes > 67108864) throw new NodeWorkspaceTransferLimitError("Workspace transfer manifest exceeds its byte limit"); const raw = (await reader.readExactly(bytes)).toString("utf8"); const ref = expectedRef ?? `sha256:${createHash("sha256").update(raw).digest("hex")}`; try { return { raw, ref, manifest: parseWorkerWorkspaceManifest(raw, ref) }; } catch (error) { throw new NodeWorkspaceTransferInvalidError("manifest", "Workspace transfer manifest is invalid", { cause: error }); } }; const base = await readManifest(operation.baseManifestRef); assertCurrent(); const current = await readManifest(); assertCurrent(); let transferPaths; try { transferPaths = workerWorkspaceTransferPaths(current.manifest, base.manifest); } catch (error) { throw new NodeWorkspaceTransferInvalidError("manifest", "Workspace transfer manifests cannot be reconciled", { cause: error }); } const transferPathSet = new Set(transferPaths); stagingRoot = await fs.mkdtemp(path.join(authorization.context.temporaryRoot, "upload-")); const currentByPath = new Map(current.manifest.entries.map((entry) => [entry.path, entry])); for (const relative of transferPaths) { const entry = currentByPath.get(relative); if (!entry) continue; try { const destination = entryPath(stagingRoot, relative); await fs.mkdir(path.dirname(destination), { recursive: true, mode: 448 }); assertCurrent(); if (entry.type === "symlink") { await fs.symlink(entry.target, destination); assertCurrent(); } else { const handle = await fs.open(destination, "wx", entry.mode); try { await streamUploadFile({ reader, handle, entry, assertCurrent }); } finally { await handle.close(); } assertCurrent(); } } catch (error) { if (error instanceof NodeWorkspaceTransferInvalidError) throw error; if (params.signal.aborted || !authorizationCurrent(authorization)) throw error; throw new NodeWorkspaceTransferInvalidError("staging", "Workspace transfer payload could not be staged", { cause: error }); } } await reader.assertEnd(); assertCurrent(); if (reader.bytesRead !== contentLength) throw new NodeWorkspaceTransferInvalidError("content_length", "Workspace transfer upload length is inconsistent"); try { await assertWorkspaceMatchesManifest({ root: stagingRoot, manifest: current.manifest, entries: current.manifest.entries.filter((entry) => transferPathSet.has(entry.path)) }); } catch (error) { throw new NodeWorkspaceTransferInvalidError("staging", "Workspace transfer payload did not match its staged result", { cause: error }); } assertCurrent(); const context = authorization.context; if (!context.snapshots.has(base.ref)) { if (context.baseCommit !== base.manifest.baseCommit) { await context.pack?.catch(() => void 0); assertCurrent(); context.pack = void 0; context.baseCommit = base.manifest.baseCommit; } context.snapshots.set(base.ref, { manifest: base.manifest, manifestRef: base.ref, rawManifest: base.raw, root: context.localPath }); context.currentManifestRef = base.ref; } operation.uploaded = { base: base.manifest, baseManifestRef: operation.baseManifestRef, baseRaw: base.raw, current: current.manifest, currentManifestRef: current.ref, currentRaw: current.raw, stagingRoot }; operation.state = "completed"; return { manifestRef: current.ref }; } catch (error) { if (stagingRoot) await fs.rm(stagingRoot, { recursive: true, force: true }); if (authorization.context.upload === operation) authorization.context.upload = void 0; throw error; } }, async verifyBlob(params) { const snapshot = await readWorkspaceFileSnapshotWithLimit(params.path, Math.min(params.size, MAX_WORKSPACE_INVENTORY_TOTAL_BYTES)); return snapshot.type === "file" && snapshot.size === params.size && snapshot.sha256 === params.sha256; }, close: closeEnvironment, async closeAll() { await temporaryRootReady; const closed = await Promise.allSettled([...contexts.keys()].map(closeEnvironment)); closed.push(...await Promise.allSettled([fs.rm(temporaryBaseRoot, { recursive: true, force: true })])); const failure = closed.find((result) => result.status === "rejected"); if (failure) throw failure.reason; } }; } //#endregion export { isNodeWorkspaceTransferLimitError as n, nodeWorkspaceTransferInvalidReason as r, createNodeWorkspaceTransferService as t };