openclaw
Version:
Multi-channel AI gateway with extensible messaging integrations
1,973 lines • 88.1 kB
JavaScript
import { d as asPositiveSafeInteger } from "../../number-coercion-CLj0HTDM.js";
import { t as coerceErrorMessage } from "../../error-coercion-D_-xJ90S.js";
import { c as isRecord } from "../../record-coerce-DItp3I4t.js";
import { l as normalizeOptionalString } from "../../string-coerce-CIXf7egm.js";
import { n as sliceUtf16Safe, r as truncateUtf16Safe } from "../../utf16-slice-D_ngcYKd.js";
import { h as redactToolPayloadText, p as redactSensitiveText } from "../../redact-BtvPPfTi.js";
import { n as resolvePreferredOpenClawTmpDir } from "../../tmp-openclaw-dir-DnyL0lW9.js";
import { r as runCommandWithTimeout } from "../../exec-BIE-3oLG.js";
import { t as truncateUtf8Prefix } from "../../utf8-truncate-CCgJrv8A.js";
import { t as WorkerProviderError } from "../../capability-provider.types-cizOzEy5.js";
import "../../temp-path-DcDoMvSD.js";
import "../../error-runtime-Bz9Tw57Z.js";
import "../../string-coerce-runtime-GQa0ehRA.js";
import { t as definePluginEntry } from "../../plugin-entry-zfBGJaNO.js";
import "../../process-runtime-pcN9RjT8.js";
import "../../logging-core-yEitd9NN.js";
import "../../text-utility-runtime-BjzvUG99.js";
import { C as resolveCrabboxProvisionBaseTimeoutMs, S as resolveCrabboxLifecycleTimeoutMs, T as resolveCrabboxReadyPollIntervalMs, _ as CRABBOX_NODE_ENROLLMENT_TIMEOUT_MS, a as operationLeaseId, b as CRABBOX_WARMUP_TIMEOUT_MS, c as resolveCrabboxBinary, d as resolveOpenClawRoot, f as CRABBOX_COMMAND_SETTLEMENT_TIMEOUT_MS, g as CRABBOX_NODE_ENROLLMENT_DIAGNOSTIC_TIMEOUT_MS, h as CRABBOX_MACHINE_CATALOG_TIMEOUT_MS, i as listCrabboxMachineOptions, l as resolveCrabboxProvisionProfile, m as CRABBOX_LIFECYCLE_TIMEOUT_MS, n as buildCrabboxAllocationArgs, o as operationSlug, p as CRABBOX_DESKTOP_WARMUP_TIMEOUT_MS, s as parseCrabboxProfile, t as CRABBOX_WORKER_PROVIDER_ID, u as resolveCrabboxWarmImageProfile, v as CRABBOX_SETUP_TIMEOUT_MS, w as resolveCrabboxProvisionCallTimeoutMs, x as countCrabboxProvisionSetupPhases, y as CRABBOX_STOP_TIMEOUT_MS } from "../../crabbox-worker-profile-COqYDYGL.js";
import { a as crabboxWarmImageCaptureStatus, f as withoutCrabboxWarmImageOperation, n as assertCrabboxWarmImageMigrationReady, o as crabboxWarmImageRecoveryHint, r as clearCrabboxWarmImageCapture, s as isCrabboxWarmImageCapturePaused, u as openCrabboxWarmImageStore } from "../../crabbox-worker-warm-image-store-DUhnKJbP.js";
import fs from "node:fs";
import { fileURLToPath } from "node:url";
import { join } from "node:path";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { createHash, randomUUID } from "node:crypto";
import { setTimeout as setTimeout$1 } from "node:timers/promises";
//#region extensions/crabbox/src/crabbox-worker-command-error.ts
const MAX_COMMAND_DETAIL_CHARS = 512;
function crabboxCommandDetail(result) {
const raw = [result.stdout, result.stderr].filter(Boolean).join("\n").trim();
if (!raw) return "";
const compressed = redactSensitiveText(raw).replace(/\s+/gu, " ");
return compressed.length <= MAX_COMMAND_DETAIL_CHARS ? `: ${compressed}` : `: ... ${sliceUtf16Safe(compressed, -508)}`;
}
function crabboxCommandError(action, result) {
if (result.termination !== "exit") return /* @__PURE__ */ new Error(`Crabbox ${action} did not exit normally (${result.termination})${crabboxCommandDetail(result)}`);
const exitCode = result.code === null ? "unknown" : String(result.code);
return /* @__PURE__ */ new Error(`Crabbox ${action} failed with exit code ${exitCode}${crabboxCommandDetail(result)}`);
}
//#endregion
//#region extensions/crabbox/src/crabbox-worker-command.ts
const MAX_OUTPUT_BYTES = 65536;
async function runCrabboxCommand(params) {
params.signal?.throwIfAborted();
let result;
try {
result = await params.runCommand([params.binary, ...params.args], {
timeoutMs: params.timeoutMs,
maxOutputBytes: MAX_OUTPUT_BYTES,
killProcessTree: true,
...params.env === void 0 ? {} : { env: params.env },
...params.input === void 0 ? {} : { input: params.input },
...params.signal ? { signal: params.signal } : {}
});
} catch {
params.signal?.throwIfAborted();
throw new Error(`Crabbox ${params.action} could not start`);
}
params.signal?.throwIfAborted();
return result;
}
function isUnrecognizedLease(result, identifier) {
const output = `${result.stderr}\n${result.stdout}`;
if (!output.includes(identifier) || /\b(?:access\s+denied|authentication|authorization|credentials?|forbidden|permission|token|unauthorized)\b/iu.test(output)) return false;
return result.code === 4 && /\b(?:was\s+)?not found\b/iu.test(output) || result.code === 4 && /\bno longer exists\b/iu.test(output) || result.code === 4 && /\b(?:points to|is bound to) (?:a )?missing (?:instance|sandbox)\b/iu.test(output) || result.code === 4 && /\bdisappeared before release\b/iu.test(output) || result.code === 4 && /\bunknown blacksmith testbox(?:\s|:)/iu.test(output) || result.code === 4 && /\bis not claimed by Crabbox\b/iu.test(output) || result.code === 4 && /\bwandb sandbox "[^"\r\n]+" has no matching local ownership claim\b/iu.test(output) || result.code === 5 && /\bcoder workspace "[^"\r\n]+" not found\b/iu.test(output) || /\bcoordinator GET \S*\/v1\/leases\/\S+:\s*http 404\b/iu.test(output) || result.code === 4 && /\bunknown lease(?:\s|:)/iu.test(output);
}
async function stopCrabboxLease(params) {
const result = await runCrabboxCommand({
action: "stop",
args: [
"stop",
"--provider",
params.provider,
"--id",
params.id
],
binary: params.binary,
runCommand: params.runCommand,
timeoutMs: CRABBOX_STOP_TIMEOUT_MS
});
if (result.termination === "exit" && result.code === 0) return;
throw crabboxCommandError("stop", result);
}
//#endregion
//#region extensions/crabbox/src/crabbox-worker-desktop-setup.ts
const CRABBOX_WORKER_BROWSER_PATH = "/usr/local/bin/openclaw-worker-browser";
const CRABBOX_WORKER_TERMINAL_PATH = "/usr/local/bin/openclaw-worker-terminal";
const CRABBOX_WORKER_BROWSER_CDP_PORT = 9222;
function xfceDesktopEnvironment() {
return [
"[ -r /var/lib/crabbox/desktop.env ] || { echo \"Crabbox desktop environment is unavailable\" >&2; exit 1; }",
"grep -Fx 'CRABBOX_DESKTOP_ENV=xfce' /var/lib/crabbox/desktop.env >/dev/null || { echo \"Crabbox desktop environment is not XFCE\" >&2; exit 1; }",
"grep -Fx 'DISPLAY=:99' /var/lib/crabbox/desktop.env >/dev/null || { echo \"Crabbox XFCE display is not :99\" >&2; exit 1; }",
"export DISPLAY=:99"
];
}
function createCrabboxXfceSessionEnvironment() {
return [
...xfceDesktopEnvironment(),
"worker_uid=$(id -u)",
"read_xfce_process_environment() {",
" local process_pid=\"$1\"",
" exec 8<\"/proc/$process_pid/environ\" || return 1",
" process_display=",
" process_dbus=",
" process_runtime_dir=",
" while IFS= read -r -d '' process_variable; do",
" case \"$process_variable\" in",
" DISPLAY=*) process_display=\"${process_variable#*=}\" ;;",
" DBUS_SESSION_BUS_ADDRESS=*) process_dbus=\"${process_variable#*=}\" ;;",
" XDG_RUNTIME_DIR=*) process_runtime_dir=\"${process_variable#*=}\" ;;",
" esac",
" done <&8",
" exec 8<&-",
"}",
"# XFCE owns the D-Bus session; the image's original renderer may have been launched outside it.",
"mapfile -t session_pids < <(pgrep -u \"$worker_uid\" -x xfce4-session || true)",
"[ \"${#session_pids[@]}\" -eq 1 ] || { echo \"Expected exactly one worker-owned XFCE session; restart crabbox-desktop.service and retry\" >&2; exit 1; }",
"session_pid=\"${session_pids[0]}\"",
"read_xfce_process_environment \"$session_pid\" || { echo \"XFCE session changed while it was inspected; restart crabbox-desktop.service and retry\" >&2; exit 1; }",
"[ \"$process_display\" = \":99\" ] || { echo \"XFCE session does not use DISPLAY=:99; restart crabbox-desktop.service and retry\" >&2; exit 1; }",
"DBUS_SESSION_BUS_ADDRESS=\"$process_dbus\"",
"unset XDG_RUNTIME_DIR",
"XDG_RUNTIME_DIR=\"$process_runtime_dir\"",
"[ -n \"$DBUS_SESSION_BUS_ADDRESS\" ] || { echo \"XFCE session is missing its D-Bus binding; restart crabbox-desktop.service and retry\" >&2; exit 1; }",
"case \"$XDG_RUNTIME_DIR\" in \"\"|/*) ;; *) echo \"XFCE session has an invalid XDG_RUNTIME_DIR\" >&2; exit 1 ;; esac",
"export DBUS_SESSION_BUS_ADDRESS",
"[ -z \"$XDG_RUNTIME_DIR\" ] || export XDG_RUNTIME_DIR"
];
}
function browserLauncher(leaseId) {
return [
"#!/bin/bash",
"set -euo pipefail",
"[ \"$#\" -eq 0 ] || { echo \"openclaw-worker-browser does not accept arguments\" >&2; exit 64; }",
...xfceDesktopEnvironment(),
"[ -x /usr/local/bin/crabbox-browser ] || { echo \"Crabbox browser is unavailable\" >&2; exit 1; }",
"worker_home=$(getent passwd \"$(id -u)\" | cut -d: -f6)",
"case \"$worker_home\" in /*) ;; *) echo \"Crabbox worker home is invalid\" >&2; exit 1 ;; esac",
"export HOME=\"$worker_home\"",
`export CRABBOX_BROWSER_PROFILE="$worker_home/.cache/openclaw/worker-browser/${leaseId}"`,
"mkdir -p \"$CRABBOX_BROWSER_PROFILE\"",
"chmod 700 \"$CRABBOX_BROWSER_PROFILE\"",
"exec 9>\"$CRABBOX_BROWSER_PROFILE/.openclaw-launch.lock\"",
"flock -x 9",
`cdp_url=http://127.0.0.1:${CRABBOX_WORKER_BROWSER_CDP_PORT}/json/version`,
"if curl --fail --silent --show-error --max-time 1 \"$cdp_url\" >/dev/null; then",
" exit 0",
"fi",
"launch_log=\"$CRABBOX_BROWSER_PROFILE/launch.log\"",
": >\"$launch_log\"",
`nohup /usr/local/bin/crabbox-browser --remote-debugging-address=127.0.0.1 --remote-debugging-port=${CRABBOX_WORKER_BROWSER_CDP_PORT} about:blank >>"$launch_log" 2>&1 </dev/null &`,
"for _attempt in $(seq 1 40); do",
" if curl --fail --silent --show-error --max-time 1 \"$cdp_url\" >/dev/null; then",
" exit 0",
" fi",
" sleep 0.5",
"done",
`echo "Browser CDP did not become ready on 127.0.0.1:${CRABBOX_WORKER_BROWSER_CDP_PORT} within 20 seconds" >&2`,
"exit 1"
];
}
function terminalLauncher() {
return [
"#!/bin/bash",
"set -euo pipefail",
"[ \"$#\" -eq 0 ] || { echo \"openclaw-worker-terminal does not accept arguments\" >&2; exit 64; }",
...xfceDesktopEnvironment(),
"nohup /usr/bin/xfce4-terminal >/dev/null 2>&1 </dev/null &",
"terminal_pid=$!",
"sleep 0.2",
"if kill -0 \"$terminal_pid\" 2>/dev/null; then",
" exit 0",
"fi",
"wait \"$terminal_pid\""
];
}
function heredoc(target, marker, contents) {
return [
`cat >"$setup_dir/${target}" <<'${marker}'`,
...contents,
marker
];
}
function createCrabboxWorkerDesktopSetup(leaseId, wallpaperBase64) {
return [
"set -euo pipefail",
...createCrabboxXfceSessionEnvironment(),
"worker_user=$(id -un)",
"worker_group=$(id -gn)",
"worker_home=$(getent passwd \"$worker_uid\" | cut -d: -f6)",
"case \"$worker_home\" in /*) ;; *) echo \"Crabbox worker home is invalid\" >&2; exit 1 ;; esac",
"as_root() { if [ \"$worker_uid\" -eq 0 ]; then \"$@\"; else sudo -n -- \"$@\"; fi; }",
"for required_command in xfconf-query xfdesktop xrandr awk curl flock getent pgrep pkill python3; do command -v \"$required_command\" >/dev/null 2>&1 || { echo \"Required Crabbox desktop command is unavailable: $required_command\" >&2; exit 1; }; done",
"bind_xfdesktop_renderer() {",
" mapfile -t renderer_pids < <(pgrep -u \"$worker_uid\" -x xfdesktop || true)",
" [ \"${#renderer_pids[@]}\" -eq 1 ] || return 1",
" renderer_pid=\"${renderer_pids[0]}\"",
" read_xfce_process_environment \"$renderer_pid\" || return 1",
" [ \"$process_display\" = \"$DISPLAY\" ] && [ \"$process_dbus\" = \"$DBUS_SESSION_BUS_ADDRESS\" ]",
"}",
"setup_dir=$(mktemp -d)",
"trap 'rm -rf -- \"$setup_dir\"' EXIT",
...heredoc("browser", "WORKER_BROWSER_LAUNCHER_EOF", browserLauncher(leaseId)),
...heredoc("terminal", "WORKER_TERMINAL_LAUNCHER_EOF", terminalLauncher()),
`python3 -c 'import base64,pathlib,sys;pathlib.Path(sys.argv[1]).write_bytes(base64.b64decode(sys.stdin.buffer.read().strip(),validate=True))' "$setup_dir/wallpaper.png" <<'WORKER_WALLPAPER_B64_EOF'`,
wallpaperBase64,
"WORKER_WALLPAPER_B64_EOF",
`as_root install -o root -g root -m 0755 "$setup_dir/browser" ${CRABBOX_WORKER_BROWSER_PATH}`,
`as_root install -o root -g root -m 0755 "$setup_dir/terminal" ${CRABBOX_WORKER_TERMINAL_PATH}`,
"as_root install -d -o \"$worker_user\" -g \"$worker_group\" -m 0755 \"$worker_home/.cache\" \"$worker_home/.cache/openclaw\" \"$worker_home/.cache/openclaw/worker-browser\"",
`as_root install -d -o "$worker_user" -g "$worker_group" -m 0700 "$worker_home/.cache/openclaw/worker-browser/${leaseId}"`,
"as_root install -d -o \"$worker_user\" -g \"$worker_group\" -m 0755 \"$worker_home/.local\" \"$worker_home/.local/share\" \"$worker_home/.local/share/backgrounds\"",
"wallpaper_path=\"$worker_home/.local/share/backgrounds/openclaw-worker.png\"",
"as_root install -o \"$worker_user\" -g \"$worker_group\" -m 0644 \"$setup_dir/wallpaper.png\" \"$wallpaper_path\"",
"# Setup precedes node enrollment, so re-home only this worker's renderer before publishing it.",
"pkill -TERM -u \"$worker_uid\" -x xfdesktop || true",
"for _attempt in $(seq 1 20); do pgrep -u \"$worker_uid\" -x xfdesktop >/dev/null || break; sleep 0.1; done",
"pkill -KILL -u \"$worker_uid\" -x xfdesktop || true",
"nohup xfdesktop >\"$worker_home/.cache/openclaw/xfdesktop.log\" 2>&1 </dev/null &",
"for _attempt in $(seq 1 40); do bind_xfdesktop_renderer && break; sleep 0.1; done",
"bind_xfdesktop_renderer || { echo \"XFCE desktop renderer did not converge on the worker session\" >&2; exit 1; }",
"mapfile -t backdrop_roots < <(",
" {",
" xfconf-query -c xfce4-desktop -l | sed -n 's#\\(/backdrop/[^/]*/[^/]*/workspace[^/]*\\)/.*#\\1#p'",
" while read -r monitor; do for workspace in 0 1 2 3; do printf \"/backdrop/screen0/monitor%s/workspace%s\\n\" \"$monitor\" \"$workspace\"; done; done < <(xrandr --listmonitors | awk 'NR > 1 { print $NF }')",
" } | sort -u",
")",
"[ \"${#backdrop_roots[@]}\" -gt 0 ] || { echo \"XFCE did not advertise any desktop backdrops\" >&2; exit 1; }",
"for backdrop in \"${backdrop_roots[@]}\"; do",
" xfconf-query -c xfce4-desktop -p \"$backdrop/last-image\" -s \"$wallpaper_path\" || xfconf-query -c xfce4-desktop -p \"$backdrop/last-image\" -n -t string -s \"$wallpaper_path\"",
" xfconf-query -c xfce4-desktop -p \"$backdrop/image-style\" -s 5 || xfconf-query -c xfce4-desktop -p \"$backdrop/image-style\" -n -t int -s 5",
"done",
"renderer_pid_before_reload=\"$renderer_pid\"",
"xfdesktop --reload",
"bind_xfdesktop_renderer || { echo \"XFCE desktop renderer lost its worker session during reload\" >&2; exit 1; }",
"[ \"$renderer_pid\" = \"$renderer_pid_before_reload\" ] || { echo \"XFCE desktop renderer changed during reload; restart crabbox-desktop.service and retry\" >&2; exit 1; }"
].join("\n");
}
function createCrabboxWorkerDesktopEndpoint() {
return {
protocol: "rfb",
port: 5900,
passwordFilePath: "/var/lib/crabbox/vnc.password",
apps: [{
id: "browser",
executablePath: CRABBOX_WORKER_BROWSER_PATH,
cdpPort: CRABBOX_WORKER_BROWSER_CDP_PORT
}, {
id: "terminal",
executablePath: CRABBOX_WORKER_TERMINAL_PATH
}]
};
}
//#endregion
//#region extensions/crabbox/src/crabbox-worker-heartbeat.ts
const CRABBOX_HEARTBEAT_UPGRADE = "upgrade Crabbox to v0.44.0 or newer for `crabbox heartbeat`";
function permanentHeartbeatFailure(result) {
const output = `${result.stderr}\n${result.stdout}`;
if (result.termination === "exit" && result.code === 2 && /\bprovider=\S+ does not support lease heartbeat\b/iu.test(output)) return "provider";
return /\b(?:unexpected argument|unknown command|unrecognized command)[^\r\n]*\bheartbeat\b/iu.test(output) || /\bheartbeat\b[^\r\n]*\b(?:unknown|unrecognized)\b/iu.test(output) || result.termination === "exit" && result.code === 2 ? "command" : void 0;
}
function createCrabboxHeartbeatManager(dependencies) {
const entries = /* @__PURE__ */ new Map();
let disposed = false;
const isCurrent = (entry) => !disposed && entries.get(entry.id) === entry && !entry.controller.signal.aborted;
const warn = (entry, message) => dependencies.warn(`${message}; cloud worker machines may be reaped after ${entry.idleTimeout} of coordinator-idle time`);
const schedule = (entry, delayMs = entry.heartbeatIntervalMs) => {
if (!isCurrent(entry)) return;
entry.timer = setTimeout(() => {
entry.pending = heartbeat(entry);
}, delayMs);
entry.timer.unref?.();
};
const heartbeat = async (entry) => {
if (!isCurrent(entry)) return;
let result;
const startedAt = Date.now();
try {
result = await dependencies.run(entry, entry.controller.signal);
} catch (error) {
if (isCurrent(entry) && !entry.failureWarned) {
entry.failureWarned = true;
warn(entry, error instanceof Error ? error.message : "Crabbox heartbeat failed");
}
schedule(entry);
return;
}
if (!isCurrent(entry)) return;
if (result.termination === "exit" && result.code === 0) {
entry.failureWarned = false;
schedule(entry);
return;
}
const permanentFailure = permanentHeartbeatFailure(result);
if (permanentFailure) {
const message = permanentFailure === "command" ? `Crabbox heartbeat is unavailable for worker lease ${entry.id}; ${CRABBOX_HEARTBEAT_UPGRADE}` : `Crabbox provider ${entry.provider} does not support heartbeat for worker lease ${entry.id}`;
warn(entry, message);
return;
}
if (!entry.failureWarned) {
entry.failureWarned = true;
const message = crabboxCommandError("heartbeat", result).message;
warn(entry, message.replace("(timeout)", `(timeout after ${Date.now() - startedAt} ms)`));
}
schedule(entry);
};
const stop = async (leaseId) => {
const entry = entries.get(leaseId);
if (!entry) return;
entry.controller.abort();
clearTimeout(entry.timer);
try {
await entry.pending;
} finally {
if (entries.get(leaseId) === entry) entries.delete(leaseId);
}
};
return {
start(context) {
if (disposed || entries.has(context.id)) return;
const entry = {
...context,
failureWarned: false,
controller: new AbortController()
};
entries.set(context.id, entry);
schedule(entry, 0);
},
stop,
async dispose() {
disposed = true;
await Promise.all([...entries.keys()].map(stop));
}
};
}
//#endregion
//#region extensions/crabbox/src/crabbox-worker-machine-options.ts
function parseCrabboxMachineShapes(stdout) {
const parsed = JSON.parse(stdout);
if (!Array.isArray(parsed)) throw new Error("Crabbox providers returned invalid JSON");
return new Map(parsed.flatMap((entry) => {
if (!isRecord(entry) || !isRecord(entry.classCatalog) || entry.classCatalog.disposition !== "mapped") return [];
const classes = (Array.isArray(entry.classCatalog.profiles) ? entry.classCatalog.profiles : []).flatMap((raw) => {
if (!isRecord(raw) || raw.target !== "linux" || raw.architecture !== "amd64" || !isRecord(raw.primary)) return [];
const machineClass = normalizeOptionalString(raw.class);
if (!machineClass) return [];
const cpu = asPositiveSafeInteger(raw.primary.vcpu);
const memory = raw.primary.memory;
const memoryGb = isRecord(memory) && (memory.unit === "GB" || memory.unit === "GiB") ? asPositiveSafeInteger(memory.value) : void 0;
return [{
class: machineClass,
...cpu ? { cpu } : {},
...memoryGb ? { memoryGb } : {}
}];
});
const provider = normalizeOptionalString(entry.provider)?.toLowerCase();
return provider && classes.length > 0 ? [[provider, classes]] : [];
}));
}
function createCrabboxMachineOptionsResolver(dependencies) {
const machineShapesByBinary = /* @__PURE__ */ new Map();
const loadMachineShapes = async (binary) => {
const result = await dependencies.runCommand([
binary,
"providers",
"--json"
], {
maxOutputBytes: 1048576,
killProcessTree: true,
timeoutMs: CRABBOX_MACHINE_CATALOG_TIMEOUT_MS
});
if (result.termination !== "exit" || result.code !== 0) throw new Error(`Crabbox providers command failed (${result.termination}, code ${result.code})`);
return parseCrabboxMachineShapes(result.stdout);
};
return async (profile) => {
const parsed = parseCrabboxProfile(profile);
const binary = dependencies.resolveBinary(parsed.binary);
let shapes = machineShapesByBinary.get(binary);
if (!shapes) {
shapes = loadMachineShapes(binary).catch((error) => {
machineShapesByBinary.delete(binary);
dependencies.warn(`Crabbox machine shapes unavailable: ${error instanceof Error ? error.message : String(error)}`);
return /* @__PURE__ */ new Map();
});
machineShapesByBinary.set(binary, shapes);
}
return listCrabboxMachineOptions(parsed.class, (await shapes).get(parsed.provider));
};
}
//#endregion
//#region extensions/crabbox/src/crabbox-worker-node-enrollment-diagnostics.ts
const MAX_NODE_ENROLLMENT_EVIDENCE_BYTES = 2048;
async function collectCrabboxNodeEnrollmentEvidence(params) {
let label = "box evidence";
let detail;
try {
const result = await runCrabboxCommand({
action: "enrollment diagnostics",
args: params.args,
binary: params.binary,
input: [
`state_dir="$HOME/.openclaw/cloud-workers/${params.id}"`,
"printf \"node-runtime=\"",
"if [ -L \"$state_dir/runtime\" ]; then readlink \"$state_dir/runtime\"; else printf absent; fi",
"printf \" node-pid=\"",
"if [ -s \"$state_dir/node.pid\" ] && kill -0 \"$(head -c 32 \"$state_dir/node.pid\")\" 2>/dev/null; then printf alive; else printf dead-or-absent; fi",
"printf \" node.log tail: \"",
"if [ -r \"$state_dir/node.log\" ]; then tail -c 2000 \"$state_dir/node.log\"; else printf absent; fi"
].join("\n"),
runCommand: params.runCommand,
...params.signal ? { signal: params.signal } : {},
timeoutMs: CRABBOX_NODE_ENROLLMENT_DIAGNOSTIC_TIMEOUT_MS
});
if (result.termination !== "exit" || result.code !== 0) throw crabboxCommandError("enrollment diagnostics", result);
detail = result.stdout.trim();
if (!detail) throw new Error("diagnostic command returned no output");
} catch (error) {
label = "box evidence unavailable";
detail = error instanceof Error ? error.message : "diagnostic command failed";
}
const prefix = `${label}: `;
const safeDetail = redactToolPayloadText(detail).replace(/\s+/gu, " ").trim();
return `${prefix}${truncateUtf8Prefix(safeDetail, MAX_NODE_ENROLLMENT_EVIDENCE_BYTES - prefix.length)}`;
}
//#endregion
//#region extensions/crabbox/src/crabbox-worker-node-enrollment.ts
const CLOUD_SETUP_CODE_ENV = "CRABBOX_WORKER_SETUP_CODE";
const CLOUD_BOOTSTRAP_TOKEN_ENV = "CRABBOX_WORKER_BOOTSTRAP_TOKEN";
function createCrabboxNodeEnrollmentSetup(params) {
return createCrabboxNodeSetup({
...params,
nodeBootstrap: params.enrollment.nodeBootstrap
});
}
function createCrabboxNodeRuntimeSetup(params) {
return createCrabboxNodeSetup(params);
}
function createCrabboxNodeSetup(params) {
const { enrollment, leaseId } = params;
const { token, ...nodeBootstrap } = params.nodeBootstrap;
const workerBundle = params.workerBundle ? (({ token: _token, ...artifact }) => artifact)(params.workerBundle) : void 0;
const desktopEnvironment = params.desktop ? [
"set -eu",
...createCrabboxXfceSessionEnvironment(),
`exec "$1" -e 'process.stdout.write(JSON.stringify({DISPLAY:process.env.DISPLAY,DBUS_SESSION_BUS_ADDRESS:process.env.DBUS_SESSION_BUS_ADDRESS,XDG_RUNTIME_DIR:process.env.XDG_RUNTIME_DIR}))'`
].join("\n") : null;
return {
command: `set -eu
node <<'CRABBOX_NODE_ENROLLMENT_SCRIPT'
const fs = require("node:fs");
const fsp = fs.promises;
const path = require("node:path");
const os = require("node:os");
const crypto = require("node:crypto");
const http = require("node:http");
const https = require("node:https");
const { spawn, spawnSync } = require("node:child_process");
const { once } = require("node:events");
const bootstrap = ${JSON.stringify(nodeBootstrap)};
const workerBundle = ${JSON.stringify(workerBundle)};
const leaseId = ${JSON.stringify(leaseId)};
const displayName = ${JSON.stringify(enrollment?.displayName)};
const mode = ${JSON.stringify(enrollment?.mode)};
const desktopEnvironment = ${JSON.stringify(desktopEnvironment)};
const credentials = process.env.${CLOUD_BOOTSTRAP_TOKEN_ENV};
const setupCode = process.env.${CLOUD_SETUP_CODE_ENV};
delete process.env.${CLOUD_BOOTSTRAP_TOKEN_ENV};
delete process.env.${CLOUD_SETUP_CODE_ENV};
process.umask(0o077);
let phase = "preparation";
(async () => {
let tokens;
try { tokens = JSON.parse(credentials || "{}"); }
catch { throw new Error("Cloud worker bootstrap credential format is invalid"); }
const stateDir = path.join(os.homedir(), ".openclaw", "cloud-workers", leaseId);
const runtimeRoot = path.join(os.homedir(), ".openclaw-worker", "node-runtimes");
const runtimeDir = path.join(runtimeRoot, bootstrap.sha256);
const cli = path.join(runtimeDir, "node_modules", "openclaw", "openclaw.mjs");
const pidFile = path.join(stateDir, "node.pid");
const setupFile = path.join(stateDir, "setup-code");
const runtimeLink = path.join(stateDir, "runtime");
const nodeEnv = { ...process.env, ...(mode ? { OPENCLAW_STATE_DIR: stateDir } : {}) };
if (desktopEnvironment) {
// Inspect XFCE only after stripping forwarded credentials from every child environment.
const desktop = spawnSync("bash", ["-c", desktopEnvironment, "bash", process.execPath], { env: nodeEnv, encoding: "utf8", timeout: 60000 });
if (desktop.status !== 0) throw new Error(desktop.stderr?.trim() || "Cloud worker XFCE session is unavailable");
delete nodeEnv.XDG_RUNTIME_DIR;
Object.assign(nodeEnv, JSON.parse(desktop.stdout));
}
if (mode) {
fs.mkdirSync(stateDir, { recursive: true, mode: 0o700 });
fs.chmodSync(stateDir, 0o700);
}
if (mode && fs.existsSync(pidFile)) {
const pidText = fs.readFileSync(pidFile, "utf8").trim();
if (!/^[1-9][0-9]*$/.test(pidText)) throw new Error("Cloud worker node PID is invalid; release and reprovision the worker");
const pid = Number(pidText);
let alive = true;
try { process.kill(pid, 0); } catch (error) { if (error.code !== "ESRCH") throw error; alive = false; }
if (alive) {
const args = fs.readFileSync(path.join("/proc", pidText, "cmdline"), "utf8").split("\\0");
const env = fs.readFileSync(path.join("/proc", pidText, "environ"), "utf8").split("\\0");
// OpenClaw changes process.title; the immutable install cwd survives that argv rewrite.
const title = args[0];
const nodeInvocation = args[1] === cli || ["openclaw", "openclaw-connect", "openclaw-node"].includes(title);
if (!nodeInvocation || fs.realpathSync(path.join("/proc", pidText, "cwd")) !== runtimeDir || !env.includes("OPENCLAW_STATE_DIR=" + stateDir)) {
throw new Error("Cloud worker node is running a different bootstrap artifact or invocation; release and reprovision the worker");
}
return;
}
fs.unlinkSync(pidFile);
}
const verifyRuntime = (root) => {
phase = "runtime verification";
if (!fs.lstatSync(root).isDirectory() || fs.realpathSync(root) !== root) throw new Error("Cloud worker bootstrap runtime path is unsafe");
const packageRoot = path.join(root, "node_modules", "openclaw");
const manifest = JSON.parse(fs.readFileSync(path.join(packageRoot, "package.json"), "utf8"));
if (manifest.name !== "openclaw" || manifest.version !== bootstrap.openclawVersion) throw new Error("Cloud worker bootstrap package identity does not match the Gateway");
const probe = spawnSync(process.execPath, [path.join(packageRoot, "openclaw.mjs"), "--version"], { env: nodeEnv, encoding: "utf8", timeout: 60000 });
const version = probe.stdout?.trim();
const expected = "OpenClaw " + bootstrap.openclawVersion;
if (probe.status !== 0 || (version !== expected && !version?.startsWith(expected + " "))) throw new Error("Cloud worker bootstrap CLI could not verify its Gateway version");
};
const verifyArchive = async (source, artifact, output) => {
const hash = crypto.createHash("sha256");
let bytes = 0;
for await (const chunk of source) {
bytes += chunk.byteLength;
if (bytes > artifact.bytes) throw new Error("Cloud worker bootstrap archive exceeds its declared size");
hash.update(chunk);
if (output) await output.writeFile(chunk);
}
if (bytes !== artifact.bytes || hash.digest("hex") !== artifact.sha256) throw new Error("Cloud worker bootstrap archive failed integrity verification");
};
const downloadArchive = async (artifact, token, archive) => {
phase = "download connection";
if (!token) throw new Error("Cloud worker bootstrap download authority is unavailable");
const url = new URL(artifact.url);
if (!["http:", "https:"].includes(url.protocol) || url.username || url.password || url.search || url.hash || (artifact.tlsFingerprint && url.protocol !== "https:")) throw new Error("Cloud worker bootstrap artifact transport is invalid");
const normalizePin = (value) => value.trim().replace(/^sha256:/i, "").replaceAll(":", "").toLowerCase();
const pin = artifact.tlsFingerprint ? normalizePin(artifact.tlsFingerprint) : undefined;
if (pin && !/^[a-f0-9]{64}$/.test(pin)) throw new Error("Cloud worker bootstrap TLS fingerprint is invalid");
const transport = url.protocol === "https:" ? https : http;
const request = transport.request(url, {
agent: false, headers: { authorization: "Bearer " + token }, signal: AbortSignal.timeout(600000),
...(pin ? { rejectUnauthorized: false, session: Buffer.alloc(0) } : {}),
});
// Observe transport progress without changing when the pinned request may send credentials.
request.once("socket", (socket) => {
socket.once("connect", () => { phase = url.protocol === "https:" ? "download TLS" : "download HTTP response"; });
socket.once("secureConnect", () => { if (!pin) phase = "download HTTP response"; });
});
const pendingResponse = once(request, "response").then(([response]) => response);
// Pinned private certificates authenticate the socket before any bearer bytes leave.
void (async () => {
if (pin) {
const [socket] = await once(request, "socket");
await once(socket, "secureConnect");
if (normalizePin(socket.getPeerCertificate().fingerprint256 ?? "") !== pin) throw new Error("Cloud worker bootstrap TLS fingerprint mismatch");
phase = "download HTTP response";
}
request.end();
})().catch((error) => request.destroy(error));
const response = await pendingResponse;
try {
if (response.statusCode !== 200) throw new Error("Cloud worker bootstrap download failed with HTTP " + response.statusCode);
if (response.headers["content-length"] !== undefined && Number(response.headers["content-length"]) !== artifact.bytes) throw new Error("Cloud worker bootstrap archive length does not match the Gateway");
phase = "download body";
const output = await fsp.open(archive, "wx", 0o600);
try { await verifyArchive(response, artifact, output); }
finally { await output.close(); }
} finally { response.destroy(); }
};
const workerArchivePath = (root) => {
const relative = workerBundle.packageRelativePath;
const parts = relative.split("/");
if (parts.length !== 2 || !/^[a-z][a-z-]*$/.test(parts[0]) || parts[1] !== workerBundle.sha256 + ".tgz" || !/^[a-f0-9]{64}$/.test(workerBundle.sha256)) throw new Error("Cloud worker archive package path is invalid");
return path.join(root, "node_modules", "openclaw", ...parts);
};
const verifyWorkerArchive = async (root) => {
const archive = workerArchivePath(root);
let handle;
// A non-regular artifact (including a FIFO) must reject without blocking preparation.
try { handle = await fsp.open(archive, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK); }
catch (error) { if (error.code === "ENOENT") return false; throw error; }
try {
const stat = await handle.stat();
if (!stat.isFile() || stat.nlink !== 1 || stat.size !== workerBundle.bytes || fs.realpathSync(archive) !== archive) throw new Error("Cloud worker prepared archive path or length is unsafe");
const source = handle.createReadStream({ autoClose: false });
try { await verifyArchive(source, workerBundle); }
finally { source.destroy(); }
} finally { await handle.close(); }
return true;
};
const publishWorkerArchive = (root, downloaded) => {
if (!workerBundle) return;
const archive = workerArchivePath(root);
const directory = path.dirname(archive);
fs.mkdirSync(directory, { recursive: true, mode: 0o700 });
if (fs.realpathSync(directory) !== directory || !fs.lstatSync(directory).isDirectory()) throw new Error("Cloud worker prepared archive directory is unsafe");
if (downloaded) fs.renameSync(downloaded, archive);
// This package owns one immutable source archive; ordinary node installs never write here.
for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
if (entry.name !== path.basename(archive) && /^[a-f0-9]{64}\\.tgz$/.test(entry.name)) {
if (!entry.isFile() || entry.isSymbolicLink()) throw new Error("Cloud worker prepared archive directory contains an unsafe artifact");
fs.unlinkSync(path.join(directory, entry.name));
}
}
};
fs.mkdirSync(runtimeRoot, { recursive: true, mode: 0o700 });
const existingRuntime = fs.existsSync(runtimeDir);
if (existingRuntime) verifyRuntime(runtimeDir);
if (existingRuntime && (!workerBundle || await verifyWorkerArchive(runtimeDir))) {
publishWorkerArchive(runtimeDir);
} else {
const stage = fs.mkdtempSync(path.join(runtimeRoot, "node-bootstrap-"));
try {
const archive = path.join(stage, "openclaw.tgz");
if (!existingRuntime) await downloadArchive(bootstrap, tokens.nodeBootstrap, archive);
let downloadedWorker;
if (workerBundle) {
downloadedWorker = path.join(stage, "worker.tgz");
await downloadArchive(workerBundle, tokens.workerBundle, downloadedWorker);
}
const installDir = existingRuntime ? runtimeDir : path.join(stage, "runtime");
if (!existingRuntime) {
phase = "installation";
fs.mkdirSync(installDir, { mode: 0o700 });
// npm 12 requires a project policy even when ignore-scripts is false.
// Trust only the verified artifact; dependency script policy stays unchanged.
fs.writeFileSync(path.join(installDir, "package.json"), JSON.stringify({ private: true, allowScripts: { ["file:" + archive]: true } }), { mode: 0o600 });
const logPath = path.join(stage, "install.log");
const log = fs.openSync(logPath, "w", 0o600);
let installed;
try {
installed = spawnSync("npm", ["install", "--prefix", installDir, "--omit=dev", "--no-save", "--package-lock=false", "--no-audit", "--no-fund", "--ignore-scripts=false", archive], { cwd: stage, env: nodeEnv, stdio: ["ignore", log, log], timeout: 600000 });
} finally { fs.closeSync(log); }
if (installed.status !== 0) {
const tail = fs.readFileSync(logPath, "utf8").slice(-2048);
throw new Error("Cloud worker bootstrap package installation failed: " + tail);
}
verifyRuntime(installDir);
}
publishWorkerArchive(installDir, downloadedWorker);
// Archive publication completes before a fresh runtime becomes reusable or capture can begin.
if (!existingRuntime) fs.renameSync(installDir, runtimeDir);
} finally { fs.rmSync(stage, { recursive: true, force: true }); }
}
// A project snapshot contains only verified runtime bytes, never enrollment state.
if (!mode) return;
phase = "activation";
try {
if (!fs.lstatSync(runtimeLink).isSymbolicLink()) throw new Error("Cloud worker runtime pointer is occupied");
fs.unlinkSync(runtimeLink);
} catch (error) { if (error.code !== "ENOENT") throw error; }
fs.symlinkSync(runtimeDir, runtimeLink);
for (const pluginId of new Set([...bootstrap.enabledPluginIds, ...${JSON.stringify(params.desktop ? ["cua-computer"] : [])}])) {
const enabled = spawnSync(process.execPath, [cli, "plugins", "enable", pluginId], { env: nodeEnv, encoding: "utf8", timeout: 60000 });
if (enabled.status !== 0) throw new Error("Cloud worker bootstrap could not enable plugin " + pluginId);
}
if (mode === "connect") {
if (!setupCode) throw new Error("Cloud worker enrollment credential is unavailable");
fs.writeFileSync(setupFile, setupCode + "\\n", { mode: 0o600 });
}
const args = mode === "connect" ? ["connect", "--target-file", setupFile] : ["node", "run"];
phase = "node launch";
const log = fs.openSync(path.join(stateDir, "node.log"), "a", 0o600);
let child;
try {
child = spawn(process.execPath, [cli, ...args, "--ephemeral", "--display-name", displayName], { cwd: runtimeDir, env: nodeEnv, detached: true, stdio: ["ignore", log, log] });
await once(child, "spawn");
try { fs.writeFileSync(pidFile, String(child.pid) + "\\n", { mode: 0o600 }); }
catch (error) { process.kill(-child.pid, "SIGTERM"); throw error; }
child.unref();
} finally { fs.closeSync(log); }
})().catch((error) => { console.error("Cloud worker node bootstrap " + phase + " failed" + (error.code ? " (" + error.code + ")" : "") + ": " + error.message); process.exitCode = 1; });
CRABBOX_NODE_ENROLLMENT_SCRIPT`,
forwardedEnv: {
[CLOUD_BOOTSTRAP_TOKEN_ENV]: JSON.stringify({
nodeBootstrap: token,
workerBundle: params.workerBundle?.token
}),
...enrollment?.mode === "connect" ? { [CLOUD_SETUP_CODE_ENV]: enrollment.setupCode } : {}
}
};
}
//#endregion
//#region extensions/crabbox/src/crabbox-worker-project.ts
/** Core owns Git contents; this adapter owns only the existing lease's transport. */
async function prepareCrabboxProjectFiles(params) {
const run = async (args, signal, input) => {
params.project.assertCurrent();
const result = await runCrabboxCommand({
action: "project preparation",
args,
binary: params.binary,
runCommand: params.runCommand,
signal: params.signal ? AbortSignal.any([signal, params.signal]) : signal,
input,
timeoutMs: params.timeoutMs()
});
params.project.assertCurrent();
if (result.termination !== "exit" || result.code !== 0) throw crabboxCommandError("project preparation", result);
return result.stdout;
};
await params.project.prepare({
runScript: (input, signal) => run(params.runArgs, signal, input),
upload: async (localPath, remotePath, signal) => {
await run([
"cp",
"--provider",
params.provider,
"--id",
params.id,
localPath,
`SANDBOX:${remotePath}`
], signal);
}
});
}
//#endregion
//#region extensions/crabbox/src/crabbox-worker-env-profile.ts
async function withCrabboxWorkerEnvProfile(values, run) {
const entries = Object.entries(values ?? {});
const names = entries.map(([name]) => name);
let directory;
try {
let profilePath;
if (entries.length > 0) {
const profile = entries.map(([name, value]) => {
if ([
"\0",
"\r",
"\n",
"`",
"$("
].some((unsafe) => value.includes(unsafe))) throw new WorkerProviderError(`Crabbox setup environment value cannot be represented safely: ${name}`);
return `${name}="${value.replaceAll("\\", "\\\\").replaceAll("\"", "\\\"")}"`;
}).join("\n");
directory = await mkdtemp(join(resolvePreferredOpenClawTmpDir(), "openclaw-crabbox-env-"));
profilePath = join(directory, "setup.env");
await writeFile(profilePath, `${profile}\n`, {
mode: 384,
flag: "wx"
});
}
return await run(names, profilePath, {
...Object.fromEntries(names.map((name) => [name, void 0])),
CRABBOX_ENV_ALLOW: ","
});
} finally {
if (directory) await rm(directory, {
force: true,
recursive: true
});
}
}
//#endregion
//#region extensions/crabbox/src/crabbox-worker-inspect.ts
function parseInspectJson(stdout) {
let value;
try {
const parsed = JSON.parse(stdout);
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw new Error("inspect output is not an object");
value = parsed;
} catch {
throw new Error("Crabbox inspect returned invalid JSON");
}
const id = normalizeOptionalString(value.id);
const state = normalizeOptionalString(value.state)?.toLowerCase();
if (!id || !/^\S{1,128}$/u.test(id) || !state) throw new Error("Crabbox inspect returned an invalid lease identity or state");
if (value.ready !== void 0 && typeof value.ready !== "boolean") throw new Error("Crabbox inspect returned an invalid ready state");
if (value.tailscale !== void 0 && (value.tailscale === null || typeof value.tailscale !== "object" || Array.isArray(value.tailscale))) throw new Error("Crabbox inspect returned invalid Tailscale state");
const tailscaleEnabled = value.tailscale !== void 0;
let awsInstanceProfileAttached;
if (value.providerMetadata !== void 0) {
if (value.providerMetadata === null || typeof value.providerMetadata !== "object" || Array.isArray(value.providerMetadata)) throw new Error("Crabbox inspect returned invalid provider metadata");
const attached = value.providerMetadata["instanceProfileAttached"];
if (attached !== void 0 && typeof attached !== "boolean") throw new Error("Crabbox inspect returned invalid AWS instance profile metadata");
awsInstanceProfileAttached = attached;
}
return {
id,
state,
tailscaleEnabled,
...awsInstanceProfileAttached !== void 0 ? { awsInstanceProfileAttached } : {},
...typeof value.ready === "boolean" ? { ready: value.ready } : {}
};
}
//#endregion
//#region extensions/crabbox/src/crabbox-worker-provision-commands.ts
const NON_RUNNABLE_STATES = /* @__PURE__ */ new Set([
"archived",
"deleted",
"deleting",
"destroyed",
"expired",
"failed",
"missing",
"released",
"stopped",
"stopped_with_code",
"terminated"
]);
async function inspectWithContext(params) {
const action = params.waitForReady ? "status" : "inspect";
const result = await runCrabboxCommand({
action,
args: [
action,
"--provider",
params.context.provider,
"--network",
"public",
"--id",
params.id,
...params.waitForReady ? [
"--wait",
"--wait-timeout",
"4m"
] : [],
"--json"
],
binary: params.context.binary,
runCommand: params.runCommand,
signal: params.signal,
timeoutMs: params.timeoutMs ?? resolveCrabboxLifecycleTimeoutMs(params.context.provider)
});
if (result.termination === "exit" && result.code === 0) {
let inspect;
try {
inspect = parseInspectJson(result.stdout);
} catch (error) {
throw new WorkerProviderError(error instanceof Error ? error.message : "Crabbox inspect returned invalid output");
}
if (params.expectedLeaseId && inspect.id !== params.expectedLeaseId) throw new WorkerProviderError("Crabbox inspect returned a different lease id");
return {
status: "found",
inspect
};
}
if (result.termination === "exit" && isUnrecognizedLease(result, params.id)) return { status: "unknown" };
throw crabboxCommandError(action, result);
}
function remainingProvisionTimeout(deadline, maximum) {
const remaining = deadline - Date.now();
if (remaining <= 0) throw new Error("Crabbox provision exceeded its provider deadline");
return Math.min(maximum, remaining);
}
const isNonRunnableState = (state) => NON_RUNNABLE_STATES.has(state.toLowerCase());
function leaseRunArgs(context, forwardedEnvNames = [], envProfilePath) {
return [
"run",
"--provider",
context.provider,
"--network",
"public",
"--tailscale=false",
"--id",
context.id,
"--keep=true",
"--no-sync",
...forwardedEnvNames.flatMap((name) => ["--allow-env", name]),
...envProfilePath ? ["--env-from-profile", envProfilePath] : [],
"--script-stdin"
];
}
function assertProvisionSecurityPolicy(params) {
if (params.inspect.tailscaleEnabled) throw new WorkerProviderError("Crabbox cloud worker lease must not have Tailscale enabled");
const attached = params.inspect.awsInstanceProfileAttached;
const pending = !params.inspect.ready && !isNonRunnableState(params.inspect.state);
if (params.provider === "aws" && attached !== false && (attached || !pending)) throw new WorkerProviderError("Crabbox AWS inspect must attest that no instance profile is attached");
}
async function waitForProvisionReady(params) {
let inspect = params.inspect;
const inspectAgain = async () => {
params.signal?.throwIfAborted();
const replay = await inspectWithContext({
context: {
binary: params.binary,
provider: params.provider
},
expectedLeaseId: inspect.id,
id: inspect.id,
runCommand: params.runCommand,
signal: params.signal,
timeoutMs: remainingProvisionTimeout(params.deadline, resolveCrabboxLifecycleTimeoutMs(params.provider)),
waitForReady: params.provider === "machine0"
});
if (replay.status === "unknown") throw new Error("Crabbox operation lease disappeared while waiting for SSH readiness");
params.signal?.throwIfAborted();
return replay.inspect;
};
try {
inspect = params.refresh ? await inspectAgain() : params.inspect;
params.signal?.throwIfAborted();
assertProvisionSecurityPolicy({
inspect,
provider: params.provider
});
while (inspect.ready !== true && !isNonRunnableState(inspect.state)) {
params.signal?.throwIfAborted();
const remaining = remainingProvisionTimeout(params.deadline, CRABBOX_LIFECYCLE_TIMEOUT_MS);
await params.sleep(Math.min(resolveCrabboxReadyPollIntervalMs(params.provider), remaining), params.signal);
params.signal?.throwIfAborted();
inspect = await inspectAgain();
assertProvisionSecurityPolicy({
inspect,
provider: params.provider
});
}
if (isNonRunnableState(inspect.state)) throw new WorkerProviderError("Crabbox operation lease entered a terminal state while waiting for SSH");
return inspect;
} catch (error) {
params.signal?.throwIfAborted();
if (error instanceof WorkerProviderError) return await failProvisionAfterCleanup({
...params,
id: inspect.id
}, error);
throw error;
}
}
async function runProvisionSetup(params) {
try {
const result = await withCrabboxWorkerEnvProfile(params.forwardedEnv, (names, profilePath, childEnv) => runCrabboxCommand({
action: params.phase,
args: leaseRunArgs({
...params,
id: params.inspect.id
}, names, profilePath),
binary: params.binary,
env: childEnv,
input: params.setup,
runCommand: params.runCommand,
signal: params.signal,
timeoutMs: remainingProvisionTimeout(params.deadline, params.timeoutMs ?? 9e5)
}));
if (result.termination !== "exit" || result.code !== 0) throw new WorkerProviderError(crabboxCommandError(params.phase, result).message);
} catch (error) {
params.signal?.throwIfAborted();
return await failProvisionAfterCleanup({
...params,
id: params.inspect.id
}, error);
}
params.signal?.throwIfAborted();
}
async function runProvisionSetupAndWaitReady(params) {
await runProvisionSetup(params);
return await waitForProvisionReady({
...params,
refresh: true
});
}
async function failProvisionAfterCleanup(params, provisionError) {
try {
await params.stopLease(params);
} catch (cleanupError) {
throw WorkerProviderError.cleanupIndeterminate(params.id, provisionError, cleanupError);
}
throw provisionError;
}
//#endregion
//#region extensions/crabbox/src/crabbox-worker-wallpaper.ts
const WORKER_WALLPAPER_WIDTH = 1024;
const WORKER_WALLPAPER_HEIGHT = 576;
const PNG_SIGNATURE = Buffer.from([
137,
80,
78,
71,
13,
10,
26,
10
]);
function loadCrabboxWorkerWallpaperBase64(wallpaperPath) {
let wallpaper;
try {
wallpaper = fs.readFileSync(wallpaperPath);
} catch (cause) {
throw new Error(`Crabbox worker wallpaper could not be read: ${wallpaperPath}`, { cause });
}
if (wallpaper.length < 33 || !wallpaper.subarray(0, PNG_SIGNATURE.length).equals(PNG_SIGNATURE) || wallpaper.readUInt32BE(8) !== 13 || wallpaper.toString("ascii", 12, 16) !== "IHDR") throw new Error(`Crabbox worker wallpaper is not a PNG: ${wallpaperPath}`);
const width = wallpaper.readUInt32BE(16);
const height = wallpaper.readUInt32BE(20);
if (width !== WORKER_WALLPAPER_WIDTH || height !== WORKER_WALLPAPER_HEIGHT) throw new Error(`Crabbox worker wallpaper must be ${WORKER_WALLPAPER_WIDTH}x${WORKER_WALLPAPER_HEIGHT}; got ${width}x${height}: ${wallpaperPath}`);
return wallpaper.toString("base64");
}
//#endregion
//#region extensions/crabbox/src/crabbox-worker-warm-image-checkpoint.ts
const CHECKPOINT_ID_PATTERN = /^chk_[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/u;
function parseCheckpointJson(stdout, action) {
let parsed;
try {
parsed = JSON.parse(stdout);
} catch {
throw new Error(`Crabbox checkpoint ${action} returned invalid JSON`);
}
if (!isRecord(parsed)) throw new Error(`Crabbox checkpoint ${action} returned an invalid record`);
return parsed;
}
function parseCreatedCheckpoint(stdout, leaseId) {
const record = parseCheckpointJson(stdout, "create");
const checkpointId = normalizeOptionalString(record.id);
const kind = normalizeOptionalString(record.kind);
const nativeState = isRecord(record.native) ? normalizeOptionalString(record.native.state) : void 0;
if (!checkpointId || !CHECKPOINT_ID_PATTERN.test(checkpointId) || !kind || record.leaseId !== leaseId || !nativeState) throw new Error("Crabbox checkpoint create returned an invalid native checkpoint");
return {
checkpointId,
kind,
state: nativeState === "available" ? "available" : "pending"
};
}
function parseCheckpointAvailability(stdout) {
const record = parseCheckpointJson(stdout, "inspect");
if (!normalizeOptionalString(record.localState) || !normalizeOptionalString(record.nextAction)) throw new Error("Crabbox checkpoint inspect returned an invalid verification record");
if (record.providerState === void 0 || record.providerState === "missing") return "missing";
if (typeof record.providerState !== "string") throw new Error("Crabbox checkpoint inspect returned an invalid provider state");
return record.providerState === "available" || record.nextAction === "fork_or_delete" || record.nextAction === "fork_restore_or_delete" ? "available" : "pending";
}
//#endregion
//#region extensions/crabbox/src/crabbox-worker-warm-image-scrub.ts
const SCRUB_WORKER_STATE = `set -eu
worker_root="$HOME/.openclaw/cloud-workers"
node <<'CRABBOX_SCRUB_NODE_SCRIPT'
const fs = require("node:fs");
const path = require("node:path");
const os = require("node:os");
const root = path.join(os.homedir(), ".openclaw", "cloud-workers");
const runtimeRoot = path.join(os.homedir(), ".openclaw-worker", "node-runtimes") + path.sep;
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
(async () => {
if (!fs.existsSync(root)) return;
for (const entry of fs.readdirSync(root, { withFileTypes: true })) {
if (!entry.isDirectory()) continue;
const stateDir = path.join(root, entry.name);
const pidFile = path.join(stateDir, "node.pid");
if (!fs.existsSync(pidFile)) continue;
const pidText = fs.readFileSync(pidFile, "utf8").trim();
if (!/^[1-9][0-9]*$/.test(pidText)) throw new Error("Cannot scrub a worker with an invalid node PID");
const pid = Number(pidText);
const owned = () => {
let stat;
try { stat = fs.readFileSync(path.join("/proc", pidText, "stat"), "utf8"); }
catch (error) { if (error.code === "ENOENT") return false; throw error; }
const fields = stat.slice(stat.lastIndexOf(")") + 2).split(" ");
if (fields[0] === "Z") return false;
const runtime = fs.realpathSync(path.join(stateDir, "runtime"));
const cwd = fs.realpathSync(path.join("/proc", pidText, "cwd"));
const env = fs.readFileSync(path.join("/proc", pidText, "environ"), "utf8").split("\\0");
if (!runtime.startsWith(runtimeRoot) || cwd !== runtime || Number(fields[2]) !== pid || !env.includes("OPENCLAW_STATE_DIR=" + stateDir)) throw new Error("Cannot scrub a worker whose live node ownership does not match");
return true;
};
if (!owned()) continue;
process.kill(-pid, "SIGTERM");
await delay(1000);
if (owned()) process.kill(-pid, "SIGKILL");
for (let attempt = 0; attempt < 50 && owned(); attempt++) await delay(20);
if (owned()) throw new Error("Cloud worker node did not exit before image capture");
}
})().catch((error) => { console.error(error.message); process.exitCode = 1; });
CRABBOX_SCRUB_NODE_SCRIPT
rm -rf "$worker_root"
rm -rf "$HOME/.openclaw-worker/workspaces"
# Crabbox's forwarded-env cleanup only warns on failure; native images include its workdir.
# Replace the shell for this final command so deleting its uploaded script cannot interrupt cleanup.
exec rm -rf -- .crabbox/env .crabbox/scripts
`;
//#endregion
//#region extensions/crabbox/src/crabbox-worker-warm-image.ts
const WARM_IMAGE_RETENTION_MS = 12096e5;
const WARM_IMAGE_REFRESH_MS = 864e5;
const WARM_IMAGE_COMMAND_TIMEOUT_MS = 6e4;
const WARM_IMAGE_CAPTURE_TIMEOUT_MS = 18e4;
const WARM_IMAGE_MACHINE0_CAPTURE_TIMEOUT_MS = 6e5;
const checkpointCaptureTimeoutMs = (provider) => provider === "machine0" ? WARM_IMAGE_MACHINE0_CAPTURE_TIMEOUT_MS : WARM_IMAGE_CAPTURE_TIMEOUT_MS;
function resolveCrabboxWarmImageCaptureTimeoutMs(provider) {
return 48e4 + checkpointCaptureTimeoutMs(provider);
}
function resolveCrabboxWarmImageProfileKey(profile, projectKey) {
return createHash("sha256").update(JSON.stringify({
backendProvider: profile.provider,
setup: profile.setup ?? "",
setupEnvKeys: [...profile.setupEnv ?? []].toSorted(),
desktop: profile.desktop ?? false,
machineClass: profile.class,
...projectKey ? { projectKey } : {}
})).digest("hex");
}
function createCrabboxWarmImageManager(dependencies) {
let store;
const warned = /* @__PURE__ */ new Set();
const openStore = () => store ??= openCrabboxWarmImageStore();
const assertCurrent = (context) => {
context.assertCurrent?.();
context.signal?.throwIfAborted();
};
const warnOnce = (action, error) => {
const message = `Crabbox warm image ${action} failed: ${coerceErrorMessage(error)}`;
if (!warned.has(message)) {
if (warned.size >= 128) warned.clear();
warned.add(message);
dependencies.warn(message);
}
};
const checkpointCommand = async (context, action, args, timeoutMs = WARM_IMAGE_COMMAND_TIMEOUT_MS, input) => {
assertCurrent(context);
const result = await runCrabboxCommand({
action: action === "scrub" ? action : `checkpoint ${action}`,
args,
binary: context.binary,
runCommand: dependencies.runCommand,
timeoutMs,
...context.signal ? { signal: context.signal } : {},
...input === void 0 ? {} : { input }
});
if (result.termination !== "exit" || result.code !== 0) throw crabboxCommandError(action === "scrub" ? action : `checkpoint ${action}`, result);
return result.stdout;
};
const sameImage = (left, right) => left?.checkpointId === right?.checkpointId && left?.createdAtMs === right?.createdAtMs;
const pinned = (record, checkpointId) => Object.values(record.allocations).some(({ choice }) => choice.kind === "checkpoint" && choice.checkpointId === checkpointId);
const retiringCurrent = (record) => record.operation?.type === "retire" && record.operation.checkpointId === record.image?.checkpointId;
const deleteEmptyProfile = (key) => openStore().deleteIf(key, (record) => !record.image && !record.operation && Object.keys(record.allocations).length === 0);
const lookupLease = (id) => {
const entries = openStore().entries().filter(({ value }) => Object.hasOwn(value.allocations, id));
if (entries.length > 1) throw new Error(`Crabbox lease ${id} has conflicting warm-image owners; run openclaw doctor --fix.`);
const entry = entries[0];
return entry ? {
key: entry.key,
projectKey: entry.value.projectKey,
...entry.value.allocations[id]
} : void 0;
};
const retireImage = async (context, key, record, timeoutMs = WARM_IMAGE_COMMAND_TIMEOUT_MS) => {
const operation = record.operation;
if (operation?.type !== "retire" || pinned(record, operation.checkpointId)) return;
const matches = (current) => current?.operation?.type === "retire" && current.operation.checkpointId === operation.checkpointId && sameImage(current.image, record.image) && !pinned(current, operation.checkpointId);
if (!matches(openStore().lookup(key))) return;
try {
await checkpointCommand(context, "delete", [
"checkpoint",
"delete",
operation.checkpointId
], timeoutMs);
} catch (error) {
assertCurrent(context);
if (matches(openStore().lookup(key))) warnOnce(`checkpoint retirement (${operation.checkpointId} deletion obligation retained; retry during periodic maintenance or next warm-image-enabled worker teardown; inspect with openclaw crabbox warm-images)`, error);
return;
}
openStore().update(key, (current) => {
assertCurrent(context);
if (!current || !matches(current)) return;
const next = withoutCrabboxWarmImageOperation(current);
if (next.image?.checkpointId === operation.checkpointId) delete next.image;
return next;
});
deleteEmptyProfile(key);
};
const deleteImage = async (context, key, record, timeoutMs = WARM_IMAGE_COMMAND_TIMEOUT_MS) => {
if (!record.image || record.operation || pinned(record, record.image.checkpointId)) return;
assertCurrent(context);
const retiring = {
...record,
operation: {
type: "retire",
checkpointId: record.image.checkpointId
}
};
if (openStore().update(key, (current) => JSON.stringify(current) === JSON.stringify(record) ? retiring : void 0)) await retireImage(context, key, retiring, timeoutMs);
};
const collectImages = async (context, phase) => {
const deadline = Date.now() + WARM_IMAGE_COMMAND_TIMEOUT_MS;
for (const { key, value } of openStore().entries()) {
assertCurrent(context);
const capture = crabboxWarmImageCaptureStatus(key, value);
if (capture) {
if (isCrabboxWarmImageCapturePaused(capture)) warnOnce("capture paused", crabboxWarmImageRecoveryHint(capture.selector));
continue;
}
if (value.operation ? phase === "allocation" : !value.image || Date.now() - value.image.lastUsedAtMs < WARM_IMAGE_RETENTION_MS) continue;
const remaining = () => deadline - Date.now();
if (remaining() <= 0) break;
await retireImage(context, key, value, remaining());
const current = openStore().lookup(key);
if (current?.image && sameImage(current.image, value.image) && !current.operation && Date.now() - current.image.lastUsedAtMs >= WARM_IMAGE_RETENTION_MS && remaining() > 0) await deleteImage(context, key, current, remaining());
}
};
const makeRoom = async (context) => {
const deadline = Date.now() + WARM_IMAGE_COMMAND_TIMEOUT_MS;
const candidates = openStore().entries().filter(({ value }) => !value.operation && Object.keys(value.allocations).length === 0).toSorted((a, b) => (a.value.image?.lastUsedAtMs ?? 0) - (b.value.image?.lastUsedAtMs ?? 0));
for (const { key, value } of candidates) {
if (openStore().entries().length < 128) return;
const remaining = deadline - Date.now();
if (remaining <= 0) break;
if (value.image) await deleteImage(context, key, value, remaining);
else deleteEmptyProfile(key);
}
if (openStore().entries().length >= 128) throw new Error("Crabbox warm-image profile capacity is full; stop outstanding workers or resolve cleanup with openclaw crabbox warm-images before retrying.");
};
const verifyImage = async (context, checkpointId) => {
return parseCheckpointAvailability(await checkpointCommand(context, "inspect", [
"checkpoint",
"inspect",
checkpointId,
"--verify",
"--json"
]));
};
const selectAllocation = async (context, profile) => {
const key = resolveCrabboxWarmImageProfileKey(profile, context.projectKey);
const replay = lookupLease(context.id);
if (replay) {
if (replay.key !== key || replay.machineClass !== profile.class) throw new Error("Crabbox provision retry changed its recorded profile or project identity.");
return replay;
}
await collectImages(context, "allocation");
const observed = openStore().lookup(key);
let available = Boolean(observed?.image && !retiringCurrent(observed));
if (available && observed?.image?.state === "pending") try {
const state = await verifyImage(context, observed.image.checkpointId);
available = state === "available";
if (state === "missing") await deleteImage(context, key, observed);
} catch (error) {
assertCurrent(context);
available = false;
warnOnce("verification", error);
}
if (!openStore().lookup(key)) await makeRoom(context);
assertCurrent(context);
let rejection;
openStore().update(key, (current) => {
const record = current ?? {
version: 2,
allocations: {},
...context.projectKey ? { projectKey: context.projectKey } : {}
};
if (Object.hasOwn(record.allocations, context.id)) return;
if (Object.keys(record.allocations).length >= 256) {
rejection = "Crabbox warm-image allocation capacity is full; stop outstanding workers before retrying.";
return;
}
const choice = available && record.image && sameImage(record.image, observed?.image) && !retiringCurrent(record) ? {
kind: "checkpoint",
checkpointId: record.image.checkpointId
} : { kind: "cold" };
return {
...record,
allocations: {
...record.allocations,
[context.id]: {
choice,
machineClass: profile.class,
phase: "pending"
}
}
};
});
if (rejection) throw new Error(rejection);
return lookupLease(context.id);
};
const markPhase = (id, phase, baseCommit) => {
const owner = lookupLease(id);
if (!owner) return;
if (phase === "prepared" && (!baseCommit || !/^[a-f0-9]{40}(?:[a-f0-9]{24})?$/u.test(baseCommit))) throw new Error("Crabbox project preparation requires a verified Git commit.");
let rejection;
openStore().update(owner.key, (record) => {
const allocation = record?.allocations[id];
if (!record || !allocation) {
rejection = "Crabbox allocation closed before preparation completed.";
return;
}
if (record.operation?.type === "capture" && record.operation.leaseId === id) {
rejection = "Crabbox allocation cannot enroll while its image capture is unresolved.";
return;
}
if (baseCommit && allocation.baseCommit && baseCommit !== allocation.baseCommit) {
rejection = "Crabbox provision retry changed its prepared Git commit.";
return;
}
if (phase === "enrolled" && record.projectKey && allocation.phase === "pending") {
rejection = "Crabbox project allocation must be prepared before enrollment.";
return;
}
return {
...record,
allocations: {
...record.allocations,
[id]: {
...allocation,
phase: allocation.phase === "enrolled" ? "enrolled" : phase,
...baseCommit ? { baseCommit } : {}
}
}
};
});
if (rejection) throw new Error(rejection);
};
return {
maintain: async (context) => {
assertCurrent(context);
assertCrabboxWarmImageMigrationReady();
await collectImages(context, "teardown");
},
lookupLease,
markPrepared: (id, baseCommit) => markPhase(id, "prepared", baseCommit),
markEnrolled: (id) => markPhase(id, "enrolled"),
async release(context) {
const owner = lookupLease(context.id);
if (!owner) return;
openStore().update(owner.key, (record) => {
if (!record?.allocations[context.id]) return;
const allocations = { ...record.allocations };
delete allocations[context.id];
return {
...record,
allocations
};
});
const current = openStore().lookup(owner.key);
if (current) await retireImage(context, owner.key, current);
deleteEmptyProfile(owner.key);
},
async capture(context, prepareSource) {
assertCurrent(context);
const captureId = randomUUID();
const owner = lookupLease(context.id);
const key = owner?.key;
let claimed = false;
let creating = false;
let preparing = false;
let captured = false;
const attemptCapture = async () => {
try {
await collectImages(context, "teardown");
if (!owner || !key || (owner.projectKey ? owner.phase !== "prepared" : owner.phase !== "enrolled")) return;
if (key !== resolveCrabboxWarmImageProfileKey({
...context.profile,
class: owner.machineClass
}, owner.projectKey)) throw new Error("Crabbox capture profile does not match its recorded allocation.");
let existing = openStore().lookup(key);
if (existing.operation) return;
if (existing.image) {
const state = context.forkedCheckpointId === existing.image.checkpointId ? "available" : await verifyImage(context, existing.image.checkpointId);
if (state === "missing" && !pinned(existing, existing.image.checkpointId)) {
await deleteImage(context, key, existing);
existing = openStore().lookup(key);
if (existing.image || existing.operation) return;
} else if (state !== "missing" && Date.now() - existing.image.createdAtMs < WARM_IMAGE_REFRESH_MS && (!owner.projectKey || existing.image.baseCommit === owner.baseCommit)) return;
}
const now = Date.now();
assertCurrent(context);
claimed = openStore().update(key, (current) => {
if (!current || JSON.stringify(current) !== JSON.stringify(existing) || current.allocations[context.id]?.phase !== owner.phase) return;
return {
...current,
operation: {
type: "capture",
id: captureId,
startedAtMs: now,
leaseId: context.id,
provider: context.provider,
phase: "scrubbing"
}
};
});
if (!claimed) return;
assertCurrent(context);
preparing = true;
await prepareSource?.();
preparing = false;
await checkpointCommand(context, "scrub", dependencies.runArgs(context), WARM_IMAGE_CAPTURE_TIMEOUT_MS, SCRUB_WORKER_STATE);
assertCurrent(context);
creating = openStore().update(key, (current) => current?.operation?.type === "capture" && current.operation.id === captureId && current.allocations[context.id]?.phase === owner.phase ? {
...current,
operation: {
...current.operation,
phase: "creating"
}
} : void 0);
if (!creating) {
clearCrabboxWarmImageCapture(key, captureId);
return;
}
const created = parseCreatedCheckpoint(await checkpointCommand(context, "create", [
"checkpoint",
"create",
"--provider",
context.provider,
"--id",
context.id,
"--mode",
"native",
"--wait",
"--json",
...context.provider === "daytona" ? ["--no-reboot=false"] : [],
...context.provider === "machine0" ? ["--strategy", "image"] : []
], checkpointCaptureTimeoutMs(context.provider)), context.id);
captured = true;
if (!openStore().update(key, (current) => {
if (current?.operation?.type !== "capture" || current.operation.id !== captureId) return;
return {
...withoutCrabboxWarmImageOperation(current),
image: {
...created,
createdAtMs: now,
lastUsedAtMs: Math.max(now, current.image?.lastUsedAtMs ?? 0),
...owner.baseCommit ? { baseCommit: owner.baseCommit } : {}
},
...current.image && current.image.checkpointId !== created.checkpointId ? { operation: {
type: "retire",
checkpointId: current.image.checkpointId
} } : {}
};
})) {
warnOnce("capture ownership changed", `Checkpoint ${created.checkpointId} returned after recovery of ${captureId}; reconcile it in the Crabbox catalog before resuming captures.`);
return;
}
creating = false;
claimed = false;
const replacement = openStore().lookup(key);
if (replacement) await retireImage(context, key, replacement);
} catch (error) {
if (claimed && key) try {
if (creating) openStore().update(key, (current) => current?.operation?.type === "capture" && current.operation.id === captureId ? {
...current,
operation: {
...current.operation,
phase: "uncertain"
}
} : void 0);
else clearCrabboxWarmImageCapture(key, captureId);
} catch {}
if (preparing) throw error;
warnOnce("capture", creating ? `${coerceErrorMessage(error)}. ${crabboxWarmImageRecoveryHint(captureId)}` : error);
}
};
await attemptCapture();
const operation = key && owner?.projectKey ? openStore().lookup(key)?.operation : void 0;
if (operation?.type === "capture" && operation.leaseId === context.id) throw new Error(`Crabbox project image capture is unresolved. ${crabboxWarmImageRecoveryHint(operation.id)}`);
assertCurrent(context);
return captured;
},
async allocate(context) {
assertCurrent(context);
if (context.profile.warmImage) {
assertCrabboxWarmImageMigrationReady();
const owner = await selectAllocation(context, context.profile);
if (owner.choice.kind === "checkpoint") {
const checkpointId = owner.choice.checkpointId;
const fork = parseCheckpointJson(await checkpointCommand(context, "fork", [
"checkpoint",
"fork",
checkpointId,
...buildCrabboxAllocationArgs(context.profile, context.id, context.slug),
"--json"
], context.timeoutMs()), "fork");
if (fork.checkpointId !== checkpointId || fork.leaseId !== context.id || fork.provider !== context.provider || fork.slug !== context.slug || !normalizeOptionalString(fork.workdir)) throw new Error("Crabbox checkpoint fork returned an invalid lease identity");
openStore().update(owner.key, (current) => current?.image?.checkpointId === checkpointId ? {
...current,
image: {
...current.image,
state: "available",
lastUsedAtMs: Date.now()
}
} : void 0);
return owner.choice;
}
}
assertCurrent(context);
const result = await runCrabboxCommand({
action: "warmup",
args: ["warmup", ...buildCrabboxAllocationArgs(context.profile, context.id, context.slug)],
binary: context.binary,
runCommand: dependencies.runCommand,
timeoutMs: context.timeoutMs(),
...context.signal ? { signal: context.signal } : {}
});
if (result.termination !== "exit" || result.code !== 0) throw crabboxCommandError("warmup", result);
return { kind: "cold" };
}
};
}
//#endregion
//#region extensions/crabbox/src/crabbox-worker-provider.ts
const MAX_ERROR_DETAIL_CHARS = 512;
const CRABBOX_PROJECT_PREPARATION_TIMEOUT_MS = 4 * CRABBOX_SETUP_TIMEOUT_MS + CRABBOX_NODE_ENROLLMENT_TIMEOUT_MS;
const LEASE_ID_PATTERN = /^(?:cbx_|tbx_)[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/u;
async function loadCrabboxConfigShow(params) {
const result = await runCrabboxCommand({
action: "config show",
args: [
"config",
"show",
"--json"
],
binary: params.binary,
runCommand: params.runCommand,
signal: params.signal,
timeoutMs: CRABBOX_LIFECYCLE_TIMEOUT_MS
});
if (result.termination !== "exit" || result.code !== 0) throw crabboxCommandError("config show", result);
try {
return JSON.parse(result.stdout);
} catch {
throw new Error("Crabbox config show returned invalid JSON");
}
}
async function assertAwsWorkerHasNoInstanceProfile(params) {
const config = await loadCrabboxConfigShow(params);
const instanceProfile = config && typeof config === "object" && !Array.isArray(config) ? config.aws?.instanceProfile : void 0;
if (typeof instanceProfile !== "string") throw new WorkerProviderError("Crabbox config show returned an invalid AWS instance profile");
if (normalizeOptionalString(instanceProfile)) throw new WorkerProviderError("Crabbox AWS instance profile must be empty for cloud workers");
}
async function assertHetznerDesktopHasManagedCoordinator(params) {
const config = await loadCrabboxConfigShow(params);
const view = isRecord(config) ? config : void 0;
if (normalizeOptionalString(view?.coordinator) && view?.brokerMode === "managed") return;
throw new Error("Crabbox Hetzner desktop profiles require a managed coordinator");
}
function transientAwsProfileCleanupError(profileError, action, cleanupError) {
const message = `Crabbox AWS profile rejection cleanup is indeterminate during ${action}: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}; rejection: ${profileError.message}`;
return new Error(truncateUtf16Safe(redactSensitiveText(message).replace(/\s+/gu, " "), MAX_ERROR_DETAIL_CHARS), { cause: cleanupError });
}
async function rejectAwsProfileAfterLeaseReconciliation(context, profileError, runCommand, stopLease) {
let inspected;
let invalidInspect;
try {
inspected = await inspectWithContext({
context,
expectedLeaseId: context.id,
id: context.id,
runCommand
});
} catch (error) {
if (!(error instanceof WorkerProviderError)) throw transientAwsProfileCleanupError(profileError, "inspect", error);
invalidInspect = error;
}
try {
await stopLease(context);
} catch (error) {
if (!invalidInspect && inspected?.status === "found") throw WorkerProviderError.cleanupIndeterminate(context.id, profileError, error);
throw transientAwsProfileCleanupError(profileError, "stop", invalidInspect ? new AggregateError([invalidInspect, error], "invalid inspect and stop failed") : error);
}
throw profileError;
}
function createCrabboxWorkerProvider(dependencies) {
const wallpaperBase64 = loadCrabboxWorkerWallpaperBase64(dependencies.wallpaperPath);
const runCommand = dependencies.runCommand ?? runCommandWithTimeout;
const warn = dependencies.warn ?? (() => {});
const sleep = dependencies.sleep ?? ((milliseconds, signal) => setTimeout$1(milliseconds, void 0, { signal }));
const openclawRoot = dependencies.openclawRoot ?? process.cwd();
const heartbeats = createCrabboxHeartbeatManager({
run: (context, signal) => runCrabboxCommand({
action: "heartbeat",
args: [
"heartbeat",
"--provider",
context.provider,
"--id",
context.id,
"--idle-timeout",
context.idleTimeout,
"--json"
],
binary: context.binary,
runCommand,
signal,
timeoutMs: context.heartbeatTimeoutMs
}),
warn
});
let defaultBinary;
const resolveBinary = (explicit) => {
if (explicit) return explicit;
defaultBinary ??= resolveCrabboxBinary({
explicit,
isExecutable: dependencies.isExecutable,
openclawRoot,
pathEnv: dependencies.pathEnv ?? process.env.PATH,
platform: dependencies.platform
});
return defaultBinary;
};
const listMachineOptions = createCrabboxMachineOptionsResolver({
resolveBinary,
runCommand,
warn
});
const warmImages = createCrabboxWarmImageManager({
runCommand,
runArgs: leaseRunArgs,
warn
});
const maintenanceAbort = new AbortController();
let maintenanceInFlight;
const stopLease = async (context) => {
await heartbeats.stop(context.id);
await stopCrabboxLease({
...context,
runCommand
});
await warmImages.release(context);
};
const resolveLeaseContext = (lease) => {
const profile = parseCrabboxProfile(lease.profile);
if (!LEASE_ID_PATTERN.test(lease.leaseId)) throw new Error("Crabbox lease id is invalid");
return {
context: {
binary: resolveBinary(profile.binary),
heartbeatIntervalMs: profile.heartbeatIntervalMs,
heartbeatTimeoutMs: profile.heartbeatTimeoutMs,
id: lease.leaseId,
idleTimeout: profile.idleTimeout,
provider: profile.provider
},
profile
};
};
const resolveAllocation = async (_profile, operationId) => ({
leaseId: operationLeaseId(operationId),
sharedHost: false
});
return {
id: CRABBOX_WORKER_PROVIDER_ID,
async dispose() {
maintenanceAbort.abort();
await Promise.all([heartbeats.dispose(), maintenanceInFlight?.catch(() => {})]);
},
maintain(context) {
context.assertCurrent();
maintenanceAbort.signal.throwIfAborted();
return maintenanceInFlight ??= Promise.resolve().then(async () => {
const signal = AbortSignal.any([context.signal, maintenanceAbort.signal]);
const assertCurrent = () => {
signal.throwIfAborted();
context.assertCurrent();
};
assertCurrent();
const binaries = new Set(context.profiles.map((profile) => resolveBinary(parseCrabboxProfile(profile).binary)));
if (binaries.size !== 1) {
warn("Crabbox warm-image maintenance requires one configured CLI executable; retained images were not changed. Check cloud worker profile binary settings.");
return;
}
await warmImages.maintain({
binary: [...binaries][0],
signal,
assertCurrent
});
}).finally(() => {
maintenanceInFlight = void 0;
});
},
listMachineOptions,
supportedExecutionModes: ["worker-turn", "remote-exec"],
provisionBeforeInstallation: true,
requiresNodeEnrollment: true,
supportsProjectPreparation(profile, machineClass) {
const parsed = parseCrabboxProfile(profile);
return resolveCrabboxWarmImageProfile(parsed, machineClass ?? parsed.class).warmImage;
},
resolveAllocation,
resolveProvisionTimeoutMs(profile) {
const parsed = parseCrabboxProfile(profile);
return resolveCrabboxProvisionCallTimeoutMs(parsed) + (parsed.warmImage === false ? 0 : CRABBOX_PROJECT_PREPARATION_TIMEOUT_MS + resolveCrabboxWarmImageCaptureTimeoutMs(parsed.provider));
},
resolveDestroyTimeoutMs(profile) {
const parsed = parseCrabboxProfile(profile);
return CRABBOX_STOP_TIMEOUT_MS + 2 * CRABBOX_COMMAND_SETTLEMENT_TIMEOUT_MS + (parsed.warmImage === false ? 0 : resolveCrabboxWarmImageCaptureTimeoutMs(parsed.provider));
},
async provision(profile, operationId, options) {
const signal = options?.signal;
signal?.throwIfAborted();
const executionMode = options?.executionMode;
if (executionMode !== void 0 && executionMode !== "worker-turn" && executionMode !== "remote-exec") throw new WorkerProviderError("Crabbox execution mode is unsupported");
const { profile: parsed, forwardedEnv } = resolveCrabboxProvisionProfile(profile, options?.machineClass);
const warmupTimeoutMs = parsed.desktop ? CRABBOX_DESKTOP_WARMUP_TIMEOUT_MS : CRABBOX_WARMUP_TIMEOUT_MS;
const deadline = Date.now() + resolveCrabboxProvisionBaseTimeoutMs(parsed);
const project = parsed.warmImage ? options?.project : void 0;
const preparationSignal = signal && project ? AbortSignal.any([signal, project.signal]) : signal ?? project?.signal;
const setupDeadline = deadline + countCrabboxProvisionSetupPhases(parsed) * CRABBOX_SETUP_TIMEOUT_MS + CRABBOX_NODE_ENROLLMENT_TIMEOUT_MS + (project ? CRABBOX_PROJECT_PREPARATION_TIMEOUT_MS + resolveCrabboxWarmImageCaptureTimeoutMs(parsed.provider) : 0);
const allocation = await resolveAllocation(profile, operationId);
signal?.throwIfAborted();
const binary = resolveBinary(parsed.binary);
const context = {
binary,
provider: parsed.provider
};
const leaseId = allocation.leaseId;
if (parsed.desktop && parsed.provider === "hetzner") await assertHetznerDesktopHasManagedCoordinator({
binary,
runCommand,
signal
});
if (parsed.provider === "aws") try {
await assertAwsWorkerHasNoInstanceProfile({
binary,
runCommand,
signal
});
} catch (error) {
signal?.throwIfAborted();
if (!(error instanceof WorkerProviderError)) throw error;
await rejectAwsProfileAfterLeaseReconciliation({
binary,
id: leaseId,
provider: parsed.provider
}, error, runCommand, stopLease);
}
const allocationChoice = await warmImages.allocate({
...context,
id: leaseId,
profile: parsed,
...project ? { projectKey: project.key } : {},
...project ? { assertCurrent: project.assertCurrent } : {},
signal: preparationSignal,
slug: operationSlug(operationId),
timeoutMs: () => remainingProvisionTimeout(deadline, warmupTimeoutMs)
});
let inspected;
try {
inspected = await inspectWithContext({
context,
expectedLeaseId: leaseId,
id: leaseId,
runCommand,
timeoutMs: remainingProvisionTimeout(deadline, resolveCrabboxLifecycleTimeoutMs(parsed.provider)),
waitForReady: parsed.provider === "machine0",
signal: preparationSignal
});
signal?.throwIfAborted();
} catch (error) {
signal?.throwIfAborted();
if (error instanceof WorkerProviderError) return await failProvisionAfterCleanup({
...context,
id: leaseId,
stopLease
}, error);
throw error;
}
if (inspected.status === "unknown") throw new Error("Crabbox warmup lease was not found during inspection");
const inspectedParams = {
...context,
deadline,
inspect: inspected.inspect,
profile: parsed,
runCommand,
stopLease,
signal: preparationSignal
};
if (isNonRunnableState(inspected.inspect.state)) return await failProvisionAfterCleanup({
...inspectedParams,
id: leaseId
}, new WorkerProviderError("Crabbox warmup lease entered a terminal state"));
inspectedParams.inspect = await waitForProvisionReady({
...inspectedParams,
sleep
});
inspectedParams.deadline = setupDeadline;
if (parsed.setup) inspectedParams.inspect = await runProvisionSetupAndWaitReady({
...inspectedParams,
phase: "profile setup",
setup: parsed.setup,
forwardedEnv,
sleep
});
if (parsed.desktop) inspectedParams.inspect = await runProvisionSetupAndWaitReady({
...inspectedParams,
phase: "desktop setup",
setup: createCrabboxWorkerDesktopSetup(leaseId, wallpaperBase64),
sleep
});
if (project && warmImages.lookupLease(leaseId)?.phase !== "enrolled") {
let preparationFailed = false;
let captured;
try {
await prepareCrabboxProjectFiles({
...context,
id: leaseId,
project,
runArgs: leaseRunArgs({
...context,
id: leaseId
}),
runCommand,
signal: preparationSignal,
timeoutMs: () => remainingProvisionTimeout(setupDeadline, CRABBOX_SETUP_TIMEOUT_MS)
});
project.assertCurrent();
warmImages.markPrepared(leaseId, project.baseCommit);
captured = await warmImages.capture({
...context,
id: leaseId,
profile: parsed,
signal: preparationSignal,
assertCurrent: project.assertCurrent,
...allocationChoice.kind === "checkpoint" ? { forkedCheckpointId: allocationChoice.checkpointId } : {}
}, async () => {
if (!options?.prepareNodeRuntime) throw new Error("Crabbox project snapshots require node runtime preparation");
const runtime = await options.prepareNodeRuntime();
signal?.throwIfAborted();
project.assertCurrent();
const setup = createCrabboxNodeRuntimeSetup({
nodeBootstrap: runtime.nodeBootstrap,
workerBundle: runtime.workerBundle,
leaseId
});
try {
await runProvisionSetup({
...inspectedParams,
phase: "node runtime preparation",
setup: setup.command,
forwardedEnv: setup.forwardedEnv,
timeoutMs: CRABBOX_NODE_ENROLLMENT_TIMEOUT_MS,
signal: runtime.signal && preparationSignal ? AbortSignal.any([preparationSignal, runtime.signal]) : preparationSignal ?? runtime.signal
});
} catch (error) {
preparationFailed = true;
throw error;
}
});
} catch (error) {
signal?.throwIfAborted();
project.assertCurrent();
if (preparationFailed) throw error;
return await failProvisionAfterCleanup({
...context,
id: leaseId,
stopLease
}, error);
}
if (captured) inspectedParams.inspect = await waitForProvisionReady({
...inspectedParams,
refresh: true,
sleep
});
}
signal?.throwIfAborted();
const beginNodeEnrollment = options?.beginNodeEnrollment;
if (!beginNodeEnrollment) return await failProvisionAfterCleanup({
...inspectedParams,
id: leaseId
}, /* @__PURE__ */ new Error("Crabbox worker node enrollment is unavailable"));
let enrollment;
try {
enrollment = await beginNodeEnrollment();
signal?.throwIfAborted();
} catch (error) {
signal?.throwIfAborted();
if (error instanceof Error && error.name === "AbortError") throw error;
return await failProvisionAfterCleanup({
...inspectedParams,
id: leaseId
}, error);
}
const nodeEnrollmentSetup = createCrabboxNodeEnrollmentSetup({
enrollment,
desktop: parsed.desktop,
leaseId
});
const enrollmentSignal = preparationSignal && enrollment.signal ? AbortSignal.any([preparationSignal, enrollment.signal]) : preparationSignal ?? enrollment.signal;
await runProvisionSetup({
...inspectedParams,
phase: "node enrollment setup",
signal: enrollmentSignal,
setup: nodeEnrollmentSetup.command,
timeoutMs: CRABBOX_NODE_ENROLLMENT_TIMEOUT_MS,
...nodeEnrollmentSetup.forwardedEnv ? { forwardedEnv: nodeEnrollmentSetup.forwardedEnv } : {}
});
let deviceId;
try {
deviceId = await enrollment.waitForDeviceId();
signal?.throwIfAborted();
} catch (error) {
signal?.throwIfAborted();
if (enrollment.signal?.aborted) throw error;
const leaseContext = {
...inspectedParams,
id: leaseId
};
const evidence = await collectCrabboxNodeEnrollmentEvidence({
...leaseContext,
args: leaseRunArgs(leaseContext),
...enrollmentSignal ? { signal: enrollmentSignal } : {}
});
signal?.throwIfAborted();
enrollment.signal?.throwIfAborted();
const message = error instanceof Error ? error.message : "Worker node enrollment failed";
return await failProvisionAfterCleanup(leaseContext, new Error(`${message}; ${evidence}`, { cause: error }));
}
if (parsed.warmImage) warmImages.markEnrolled(leaseId);
heartbeats.start({
binary,
heartbeatIntervalMs: parsed.heartbeatIntervalMs,
heartbeatTimeoutMs: parsed.heartbeatTimeoutMs,
id: leaseId,
idleTimeout: parsed.idleTimeout,
provider: parsed.provider
});
return {
...allocation,
node: { deviceId },
...parsed.desktop ? { desktop: createCrabboxWorkerDesktopEndpoint() } : {}
};
},
async inspect(lease) {
const { context } = resolveLeaseContext(lease);
const inspected = await inspectWithContext({
context,
expectedLeaseId: context.id,
id: context.id,
runCommand
});
if (inspected.status === "unknown" || isNonRunnableState(inspected.inspect.state)) {
await heartbeats.stop(context.id);
return { status: "unknown" };
}
heartbeats.start(context);
return { status: "active" };
},
async destroy(lease) {
const { context, profile } = resolveLeaseContext(lease);
await heartbeats.stop(context.id);
let captureError;
try {
const allocation = warmImages.lookupLease(context.id);
const captureProfile = resolveCrabboxWarmImageProfile(profile, allocation?.machineClass ?? profile.class);
if (captureProfile.warmImage) await warmImages.capture({
...context,
profile: captureProfile
});
} catch (error) {
captureError = error;
}
await stopLease(context);
if (captureError) throw captureError instanceof Error ? captureError : new Error(coerceErrorMessage(captureError));
}
};
}
//#endregion
//#region extensions/crabbox/index.ts
const workerWallpaperPath = fileURLToPath(new URL("./assets/openclaw-worker-wallpaper.png", import.meta.url));
var crabbox_default = definePluginEntry({
id: "crabbox",
name: "Crabbox Worker Provider",
description: "Cloud worker provider backed by the Crabbox CLI",
register(api) {
api.registerCli(async ({ program }) => {
const { registerCrabboxWarmImageCommands } = await import("../../crabbox-worker-warm-image-cli-DhKnriXI.js");
registerCrabboxWarmImageCommands(program);
}, { descriptors: [{
name: "crabbox",
description: "Inspect and recover Crabbox warm images",
hasSubcommands: true
}] });
const provider = createCrabboxWorkerProvider({
openclawRoot: resolveOpenClawRoot(api.rootDir),
wallpaperPath: workerWallpaperPath,
warn: (message) => api.logger.warn(message)
});
api.registerWorkerProvider(provider);
api.registerService({
id: "crabbox-worker-cleanup",
start() {},
stop() {
return provider.dispose();
}
});
}
});
//#endregion
export { crabbox_default as default };