skills-ws
Version:
87 agent skills for AI coding assistants — marketing, growth, web3, dev, design & operations. Built for OpenClaw, Claude Code, Cursor, and Codex.
776 lines (689 loc) • 33.2 kB
Markdown
## Contents
- Multi-Node Fleet Management
- Fleet Deployment Orchestrator
- Fleet Management Commands
## Multi-Node Fleet Management
### Fleet Deployment Orchestrator
**Master Deployment Script:**
**Before you run this:** generate ONE persistent `FLEET_API_KEY` locally and export it. Both the deploy script and the fleet manager must use the *same* key, and it must survive restarts (the manager must not invent a new random key each boot).
```bash
# Generate once and store it safely (NOT in git, NOT in shell history files):
export FLEET_API_KEY="$(openssl rand -hex 32)"
echo "FLEET_API_KEY=$FLEET_API_KEY" >> ~/.aleph-deploy/configs/fleet.env # chmod 600 this file
chmod 600 ~/.aleph-deploy/configs/fleet.env
```
```bash
#!/bin/bash
# deploy-fleet.sh
set -euo pipefail
# Fleet Configuration
FLEET_NAME="${1:-openclaw-fleet}"
NODE_COUNT="${2:-5}"
SSH_KEY="${ALEPH_SSH_KEY:-$HOME/.aleph-deploy/keys/aleph_ed25519}"
SSH_USER="${ALEPH_SSH_USER:-root}"
: "${FLEET_API_KEY:?Set FLEET_API_KEY (see fleet.env) before deploying}"
# Pin CRNs you have ACTUALLY verified with crn-discovery.sh. Leave empty to let
# the CLI auto-select. Never list non-compute services here (a storage gateway
# or NFT pinning API is NOT a CRN and cannot host an instance).
PRIMARY_CRN="${PRIMARY_CRN:-}" # e.g. https://<verified-crn-host>
WORKER_CRNS=(${WORKER_CRNS:-}) # e.g. ("https://<crn-a>" "https://<crn-b>")
echo "Deploying fleet: $FLEET_NAME with $NODE_COUNT nodes"
# Fleet configuration. worker_nodes entries WILL record ip + item_hash (added at
# create time) so networking/backup/security scripts can find every worker.
cat > ~/.aleph-deploy/configs/fleet.json << EOF
{
"fleet_name": "$FLEET_NAME",
"deployment_date": "$(date -Iseconds)",
"node_count": $NODE_COUNT,
"ssh_user": "$SSH_USER",
"primary_node": null,
"worker_nodes": [],
"network": {
"ssh_tunnel_port": 2222,
"load_balancer_port": 8080
},
"replication": {
"enabled": true,
"sync_interval": 300,
"backup_retention": 7
}
}
EOF
deploy_primary_node() {
echo "📊 Deploying Primary Node (Orchestrator)..."
local node_name="${FLEET_NAME}-primary"
# The primary setup script is parameterized with the (persistent) fleet key so
# the manager and workers share ONE key. We export it into the heredoc env.
local setup_script
setup_script=$(FLEET_API_KEY="$FLEET_API_KEY" envsubst '$FLEET_API_KEY' << 'PRIMARY_SETUP'
#!/bin/bash
set -euo pipefail
export DEBIAN_FRONTEND=noninteractive
# Standard VM setup. Modern package set: Docker Engine + Compose v2 plugin
# (installed via get.docker.com below), Node 22 via NodeSource, iproute2 for `ss`.
apt-get update && apt-get -y upgrade
apt-get install -y curl wget git htop jq fail2ban ufw ca-certificates iproute2 gettext-base
curl -fsSL https://get.docker.com -o /tmp/get-docker.sh && sh /tmp/get-docker.sh
installer_1="$(mktemp)"
curl -fsSL https://deb.nodesource.com/setup_22.x -o "$installer_1"
less "$installer_1" # Review before execution; verify the release checksum/signature when published.
bash "$installer_1" - && apt-get install -y nodejs
# Create a dedicated non-root user for fleet services. Running all services as
rm -f "$installer_1"
# root is a security risk — a compromise in any service gives full system access.
useradd -r -s /usr/sbin/nologin -d /opt/fleet-manager fleetmgr || true
# Install fleet management tools
mkdir -p /opt/fleet-manager
cd /opt/fleet-manager
# Fleet Manager Application
cat > fleet-manager.js << 'FLEET_MANAGER'
const express = require('express');
const fs = require('fs');
const app = express();
app.use(express.json());
// API key auth. The key MUST be provided via the environment (root-owned
// EnvironmentFile, see below) so it is stable across restarts and is never
// generated/logged. Fail fast if it is missing rather than minting a random one.
const FLEET_API_KEY = process.env.FLEET_API_KEY;
if (!FLEET_API_KEY || FLEET_API_KEY.length < 32) {
console.error('FATAL: FLEET_API_KEY env var missing or too short. Refusing to start.');
process.exit(1);
}
// Constant-time comparison; header-only (never accept keys in the query string —
// URLs are logged and cached, leaking the secret).
const crypto = require('crypto');
function keyMatches(provided) {
if (typeof provided !== 'string') return false;
const a = Buffer.from(provided);
const b = Buffer.from(FLEET_API_KEY);
return a.length === b.length && crypto.timingSafeEqual(a, b);
}
function requireAuth(req, res, next) {
if (!keyMatches(req.headers['x-api-key'])) {
return res.status(401).json({ error: 'Unauthorized' });
}
next();
}
// Health check FIRST and UNAUTHENTICATED — HAProxy/`option httpchk` calls this
// without an API key. Keep it non-sensitive (no node data).
app.get('/health', (req, res) => {
res.json({ status: 'healthy', timestamp: new Date().toISOString() });
});
// Everything below requires the API key.
app.use(requireAuth);
// Fleet status endpoint
app.get('/fleet/status', (req, res) => {
try {
const data = fs.readFileSync('/opt/fleet-manager/nodes.json', 'utf8');
res.json(JSON.parse(data));
} catch (err) {
if (err.code === 'ENOENT') {
res.json({ nodes: [] });
} else {
res.status(500).json({ error: err.message });
}
}
});
// (Health check is defined above, before requireAuth, so HAProxy/httpchk can
// reach it without an API key. Do not re-add an authenticated /health here.)
// Node registration endpoint
app.post('/fleet/register', (req, res) => {
const { node_id, ip_address, capabilities, item_hash } = req.body;
let fleet;
try {
fleet = JSON.parse(fs.readFileSync('/opt/fleet-manager/nodes.json', 'utf8'));
} catch {
fleet = { nodes: [] };
}
// Update or add node. We persist item_hash (the Aleph instance hash captured
// at create time) so the autoscale/auto-recreate paths can delete/recreate
// this exact instance later. Heartbeats omit item_hash, so on re-register we
// preserve whatever hash we already stored for this node.
const existingIndex = fleet.nodes.findIndex(n => n.node_id === node_id);
const prior = existingIndex >= 0 ? fleet.nodes[existingIndex] : {};
const nodeData = {
node_id,
ip_address,
capabilities,
item_hash: item_hash || prior.item_hash || null,
last_seen: new Date().toISOString(),
status: 'active'
};
if (existingIndex >= 0) {
fleet.nodes[existingIndex] = nodeData;
} else {
fleet.nodes.push(nodeData);
}
fs.writeFileSync('/opt/fleet-manager/nodes.json', JSON.stringify(fleet, null, 2));
res.json({ success: true });
});
// Load distribution endpoint
app.get('/fleet/distribute/:task', (req, res) => {
const task = req.params.task;
let nodes;
try {
nodes = JSON.parse(fs.readFileSync('/opt/fleet-manager/nodes.json', 'utf8'));
} catch {
nodes = { nodes: [] };
}
// Simple round-robin distribution
const activeNodes = nodes.nodes.filter(n => n.status === 'active');
if (activeNodes.length === 0) {
return res.status(503).json({ error: 'No active nodes available' });
}
const assignedNode = activeNodes[Math.floor(Math.random() * activeNodes.length)];
res.json({
task,
assigned_node: assignedNode.node_id,
node_ip: assignedNode.ip_address
});
});
const PORT = process.env.PORT || 8080;
// Bind to the Tailscale interface (or localhost) — NEVER 0.0.0.0. The systemd
// unit sets BIND_HOST to the node's Tailscale IP so workers on the mesh can
// register, while the public internet cannot reach the control plane.
const BIND_HOST = process.env.BIND_HOST || '127.0.0.1';
app.listen(PORT, BIND_HOST, () => {
console.log(`Fleet Manager listening on ${BIND_HOST}:${PORT}`);
});
FLEET_MANAGER
# Install dependencies and start fleet manager
npm init -y
npm install express
chmod +x fleet-manager.js
# Provision the SHARED, PERSISTENT FLEET_API_KEY via a root-owned EnvironmentFile.
# The key was injected into this setup script by deploy-fleet.sh (envsubst) and is
# never logged. BIND_HOST is resolved to the Tailscale IP after the mesh is up
# (a drop-in updates it; until then it stays on localhost).
install -o root -g root -m 600 /dev/null /etc/fleet-manager.env
{
echo "FLEET_API_KEY=${FLEET_API_KEY}"
echo "PORT=8080"
echo "BIND_HOST=127.0.0.1"
} > /etc/fleet-manager.env
# Create systemd service
cat > /etc/systemd/system/fleet-manager.service << 'SERVICE'
[Unit]
Description=OpenClaw Fleet Manager
After=network.target
[Service]
Type=simple
User=fleetmgr
WorkingDirectory=/opt/fleet-manager
EnvironmentFile=/etc/fleet-manager.env
ExecStart=/usr/bin/node fleet-manager.js
Restart=always
RestartSec=10
# Harden: no new privileges, read-only system except its own dir.
NoNewPrivileges=true
ProtectSystem=strict
ReadWritePaths=/opt/fleet-manager
[Install]
WantedBy=multi-user.target
SERVICE
# Set ownership so fleetmgr user can read/write
chown -R fleetmgr:fleetmgr /opt/fleet-manager
# Initialize nodes registry BEFORE starting fleet-manager.
# fleet-manager.js reads this file on startup — if it doesn't exist,
# the readFileSync call will throw ENOENT and crash the service.
echo '{"nodes": []}' > /opt/fleet-manager/nodes.json
chown fleetmgr:fleetmgr /opt/fleet-manager/nodes.json
systemctl daemon-reload
systemctl enable fleet-manager
systemctl start fleet-manager
# Install OpenClaw on the primary (official installer + onboarding daemon).
# Docs: https://docs.openclaw.ai/install . Requires Node >= 22.19 (installed above).
installer_2="$(mktemp)"
curl -fsSL https://openclaw.ai/install.sh -o "$installer_2"
less "$installer_2" # Review before execution; verify the release checksum/signature when published.
bash "$installer_2"
# `openclaw onboard --install-daemon` is interactive; run it manually (or with a
rm -f "$installer_2"
# pre-seeded config/secret store) to create the systemd daemon. Do NOT hand-write
# /opt/openclaw/config/*.json — OpenClaw manages its own config via onboard.
echo "Primary node base setup complete (fleet-manager active on Tailscale)."
PRIMARY_SETUP
)
# Create the instance with CURRENT flags, then provision over SSH (the Python
# aleph-client has no --setup-script / --image-ref / --disk-size / --crn; the
# rewritten aleph-cli does have --disk-size). See `... create --help`.
local create_args=(
--name "$node_name"
--compute-units 4 --memory 8192
--rootfs-size 81920
--ssh-pubkey-file "$SSH_KEY.pub"
--payment-type credit --payment-chain BASE
--persistent-volume "name=fleet,mount=/opt/fleet-manager,size_mib=10240"
)
[[ -n "$PRIMARY_CRN" ]] && create_args+=(--crn-url "$PRIMARY_CRN" --crn-auto-tac)
local out item_hash primary_ip
out="$(aleph instance create "${create_args[@]}")"
echo "$out"
item_hash="$(printf '%s\n' "$out" | grep -oE '[0-9a-f]{64}' | head -1)"
primary_ip="$(wait_for_ip "$node_name")" || { echo "Primary got no IP"; return 1; }
# Provision over SSH using the injected, persistent FLEET_API_KEY.
ssh -i "$SSH_KEY" -o StrictHostKeyChecking=accept-new "$SSH_USER@$primary_ip" \
"FLEET_API_KEY='$FLEET_API_KEY' bash -s" <<< "$setup_script"
# Record name, IP, and item_hash so every later script can reach/destroy it.
local tmpfile; tmpfile="$(mktemp)"
jq --arg n "$node_name" --arg ip "$primary_ip" --arg h "$item_hash" \
'.primary_node = {name:$n, ip:$ip, item_hash:$h}' \
~/.aleph-deploy/configs/fleet.json > "$tmpfile"
mv "$tmpfile" ~/.aleph-deploy/configs/fleet.json
echo "Primary node deployed: $primary_ip ($item_hash)"
return 0
}
# Poll the REAL `aleph instance list` for a named instance's IP (no fake commands).
wait_for_ip() {
local name="$1" ip=""
for _ in $(seq 1 30); do
ip="$(aleph instance list --json \
| jq -r --arg n "$name" '.[] | select(.name==$n) | (.ipv4 // .ipv6 // empty)' \
| head -1)"
[[ -n "$ip" ]] && { echo "$ip"; return 0; }
sleep 10
done
return 1
}
deploy_worker_node() {
local node_id="$1" crn_url="$2" primary_ip="$3"
local node_name="${FLEET_NAME}-worker-${node_id}"
echo "Deploying worker node $node_id ($node_name)..."
# Worker provisioning script. The worker JOINS the Tailscale mesh FIRST, then
# registers with the primary over that mesh (primary_tailscale_ip), so the
# address it registers is always its reachable Tailscale IP — never a
# firewalled public/private address. We pass primary's Tailscale IP, the
# shared key, the Tailscale auth key, and the instance ITEM_HASH in as env
# vars at SSH time.
local setup_script
setup_script=$(cat <<'WORKER_SETUP'
#!/bin/bash
set -euo pipefail
export DEBIAN_FRONTEND=noninteractive
apt-get update && apt-get -y upgrade
apt-get install -y curl wget git htop jq ca-certificates iproute2
curl -fsSL https://get.docker.com -o /tmp/get-docker.sh && sh /tmp/get-docker.sh
installer_3="$(mktemp)"
curl -fsSL https://deb.nodesource.com/setup_22.x -o "$installer_3"
less "$installer_3" # Review before execution; verify the release checksum/signature when published.
bash "$installer_3" - && apt-get install -y nodejs
# Join the Tailscale mesh BEFORE registering, so `tailscale ip -4` returns a
rm -f "$installer_3"
# reachable mesh address (the control plane is only reachable over the mesh).
# Official Tailscale installer; review first via:
# curl -fsSL https://tailscale.com/install.sh -o /tmp/ts-install.sh && less /tmp/ts-install.sh
installer_4="$(mktemp)"
curl -fsSL https://tailscale.com/install.sh -o "$installer_4"
less "$installer_4" # Review before execution; verify the release checksum/signature when published.
sh "$installer_4"
rm -f "$installer_4"
: "${TAILSCALE_AUTH_KEY:?TAILSCALE_AUTH_KEY required to join the mesh before registering}"
printf '%s' "$TAILSCALE_AUTH_KEY" > /tmp/ts && chmod 600 /tmp/ts
tailscale up --auth-key="file:/tmp/ts" --hostname="$NODE_ID"
rm -f /tmp/ts
# Confirm we actually have a mesh IP before going any further.
for _ in $(seq 1 12); do
TS_IP="$(tailscale ip -4 2>/dev/null || true)"
[[ -n "$TS_IP" ]] && break
sleep 5
done
[[ -n "${TS_IP:-}" ]] || { echo "Worker never obtained a Tailscale IP — aborting"; exit 1; }
# Install OpenClaw (official installer; Node already present).
installer_5="$(mktemp)"
curl -fsSL https://openclaw.ai/install.sh -o "$installer_5"
less "$installer_5" # Review before execution; verify the release checksum/signature when published.
bash "$installer_5"
# Run `openclaw onboard --install-daemon` to set up the daemon (see docs).
rm -f "$installer_5"
# Registration: POST to the primary over Tailscale, key from EnvironmentFile.
# NODE_ID / PRIMARY_TS_IP / FLEET_API_KEY / ITEM_HASH are provided via /etc/worker.env.
install -o root -g root -m 600 /dev/null /etc/worker.env
cat > /etc/worker.env <<ENVF
NODE_ID=${NODE_ID}
PRIMARY_TS_IP=${PRIMARY_TS_IP}
FLEET_API_KEY=${FLEET_API_KEY}
ITEM_HASH=${ITEM_HASH}
ENVF
cat > /opt/register-worker.sh <<'REGISTER'
#!/bin/bash
set -euo pipefail
set -a; . /etc/worker.env; set +a
# Use the Tailscale IP as our reachable address. We already joined the mesh in
# the setup phase, so this must succeed; bail rather than register an
# unreachable public/private fallback address.
LOCAL_IP="$(tailscale ip -4 2>/dev/null || true)"
[[ -n "$LOCAL_IP" ]] || { echo "No Tailscale IP yet — not registering an unreachable address"; exit 1; }
curl -fsS -X POST "http://${PRIMARY_TS_IP}:8080/fleet/register" \
-H "Content-Type: application/json" \
-H "x-api-key: ${FLEET_API_KEY}" \
-d "{\"node_id\":\"${NODE_ID}\",\"ip_address\":\"${LOCAL_IP}\",\"item_hash\":\"${ITEM_HASH}\",\"capabilities\":[\"compute\",\"openclaw\"]}"
REGISTER
chmod +x /opt/register-worker.sh
# Register once now (Tailscale is already up), then keep a heartbeat going.
/opt/register-worker.sh
# Heartbeat as a supervised systemd timer (re-registers every 30s; updates last_seen).
cat > /etc/systemd/system/heartbeat.service <<'HB_SVC'
[Unit]
Description=Worker node heartbeat
After=network-online.target tailscaled.service
[Service]
Type=oneshot
EnvironmentFile=/etc/worker.env
ExecStart=/opt/register-worker.sh
HB_SVC
cat > /etc/systemd/system/heartbeat.timer <<'HB_TIMER'
[Unit]
Description=Run worker heartbeat every 30s
[Timer]
OnBootSec=30
OnUnitActiveSec=30
[Install]
WantedBy=timers.target
HB_TIMER
systemctl daemon-reload
systemctl enable --now heartbeat.timer
echo "Worker node setup complete (joined mesh, registered over Tailscale)."
WORKER_SETUP
)
local create_args=(
--name "$node_name"
--compute-units 2 --memory 4096
--rootfs-size 40960
--ssh-pubkey-file "$SSH_KEY.pub"
--payment-type credit --payment-chain BASE
)
[[ -n "$crn_url" ]] && create_args+=(--crn-url "$crn_url" --crn-auto-tac)
local out item_hash worker_ip
out="$(aleph instance create "${create_args[@]}")"
echo "$out"
item_hash="$(printf '%s\n' "$out" | grep -oE '[0-9a-f]{64}' | head -1)"
worker_ip="$(wait_for_ip "$node_name")" || { echo "Worker $node_id got no IP"; return 1; }
# primary_ip here is the primary's TAILSCALE IP (resolved by the caller after
# the mesh is up). Provision over SSH with NODE_ID/PRIMARY_TS_IP/FLEET_API_KEY/
# TAILSCALE_AUTH_KEY (so the worker joins the mesh first) and ITEM_HASH (so the
# primary's registry records the instance hash for later delete/recreate).
ssh -i "$SSH_KEY" -o StrictHostKeyChecking=accept-new "$SSH_USER@$worker_ip" \
"NODE_ID='$node_name' PRIMARY_TS_IP='$primary_ip' FLEET_API_KEY='$FLEET_API_KEY' \
TAILSCALE_AUTH_KEY='$TAILSCALE_AUTH_KEY' ITEM_HASH='$item_hash' bash -s" \
<<< "$setup_script"
# Record name, id, crn, IP, and item_hash. IP is REQUIRED by Tailscale/backup/
# security scripts — never omit it.
local worker_info tmpfile
worker_info="$(jq -n --arg n "$node_name" --argjson id "$node_id" \
--arg crn "$crn_url" --arg ip "$worker_ip" --arg h "$item_hash" \
'{name:$n, id:$id, crn:$crn, ip:$ip, item_hash:$h}')"
tmpfile="$(mktemp)"
jq --argjson w "$worker_info" '.worker_nodes += [$w]' \
~/.aleph-deploy/configs/fleet.json > "$tmpfile"
mv "$tmpfile" ~/.aleph-deploy/configs/fleet.json
echo "Worker node $node_id deployed on ${crn_url:-auto-selected CRN}: $worker_ip"
}
# Main deployment sequence
echo "Starting fleet deployment sequence..."
# 1. Deploy + provision the primary (installs the fleet manager on its Tailscale IP).
deploy_primary_node
primary_public_ip="$(jq -r '.primary_node.ip' ~/.aleph-deploy/configs/fleet.json)"
# 2. Bring the primary onto Tailscale and capture its mesh IP. Workers register
# against THIS address (the control plane is never reachable on the public IP).
# Requires TAILSCALE_AUTH_KEY in the environment (see "Tailscale Mesh" section).
: "${TAILSCALE_AUTH_KEY:?Set TAILSCALE_AUTH_KEY before deploying the fleet}"
ssh -i "$SSH_KEY" -o StrictHostKeyChecking=accept-new "$SSH_USER@$primary_public_ip" \
"TAILSCALE_AUTH_KEY='$TAILSCALE_AUTH_KEY' bash -s" <<'TS_BOOT'
set -euo pipefail
installer_6="$(mktemp)"
curl -fsSL https://tailscale.com/install.sh -o "$installer_6"
less "$installer_6" # Review before execution; verify the release checksum/signature when published.
sh "$installer_6" # official, OS-detecting installer
rm -f "$installer_6"
printf '%s' "$TAILSCALE_AUTH_KEY" > /tmp/ts && chmod 600 /tmp/ts
tailscale up --auth-key="file:/tmp/ts" --hostname="$(hostname)"
rm -f /tmp/ts
# Re-point the fleet manager at the Tailscale interface and restart it.
TS_IP="$(tailscale ip -4)"
sed -i "s/^BIND_HOST=.*/BIND_HOST=${TS_IP}/" /etc/fleet-manager.env
systemctl restart fleet-manager
echo "PRIMARY_TS_IP=${TS_IP}"
TS_BOOT
primary_ts_ip="$(ssh -i "$SSH_KEY" "$SSH_USER@$primary_public_ip" "tailscale ip -4")"
jq --arg ip "$primary_ts_ip" '.primary_node.tailscale_ip=$ip' \
~/.aleph-deploy/configs/fleet.json > /tmp/fleet.$$ && \
mv /tmp/fleet.$$ ~/.aleph-deploy/configs/fleet.json
echo "Primary Tailscale IP: $primary_ts_ip"
# 3. Deploy workers. Each worker's setup script JOINS the Tailscale mesh FIRST
# and only THEN registers — so it registers against the primary's Tailscale IP
# using its own reachable Tailscale IP (never a firewalled public address).
worker_total=$((NODE_COUNT - 1))
for i in $(seq 1 "$worker_total"); do
if (( ${#WORKER_CRNS[@]} > 0 )); then
crn_url="${WORKER_CRNS[$(((i - 1) % ${#WORKER_CRNS[@]}))]}"
else
crn_url="" # let the CLI auto-select a CRN
fi
deploy_worker_node "$i" "$crn_url" "$primary_ts_ip" &
sleep 30 # stagger to avoid overwhelming CRNs
done
wait
echo "Fleet deployment complete."
echo "Fleet manager (PRIVATE, Tailscale only): http://$primary_ts_ip:8080"
echo "Status: curl -H \"x-api-key: \$FLEET_API_KEY\" http://$primary_ts_ip:8080/fleet/status"
echo "Next: run setup-tailscale-mesh.sh to verify the mesh, then setup-load-balancer.sh."
jq . ~/.aleph-deploy/configs/fleet.json
```
> **Ordering note.** Workers reach the fleet manager over Tailscale, so the primary joins the mesh *before* workers are provisioned (step 2). Each worker's setup script then joins Tailscale **first** and only **then** registers, so the address it registers is always its reachable Tailscale IP. This requires `TAILSCALE_AUTH_KEY` in the environment (passed through to each worker at SSH time). You can still run `setup-tailscale-mesh.sh` (next section) afterward to verify mesh connectivity. The public IPs are used only for the initial SSH provisioning hop.
> **Primary needs the fleet SSH key (one-time).** Several primary-resident services (replication, backups, node monitor, key rotation) SSH from the primary to workers, so the primary must hold the **private** key. Copy it once, locked down, after the primary is up — prefer Tailscale for the hop:
>
> ```bash
> PRIMARY_TS_IP="$(jq -r '.primary_node.tailscale_ip' ~/.aleph-deploy/configs/fleet.json)"
> scp -i "$SSH_KEY" "$SSH_KEY" "$SSH_USER@$PRIMARY_TS_IP:/root/.ssh/aleph_ed25519"
> ssh -i "$SSH_KEY" "$SSH_USER@$PRIMARY_TS_IP" "chmod 600 /root/.ssh/aleph_ed25519"
> ```
>
> Primary-side scripts read `ALEPH_SSH_KEY` (default `/root/.ssh/aleph_ed25519`). Treat this key as sensitive: it grants root on every worker. Rotate it (see the rotation tool) if the primary is ever compromised, and never bake the private key into an instance setup message.
### Fleet Management Commands
**Fleet Control Script.** Run this from a machine that is **on the tailnet** (the control plane lives on the primary's Tailscale IP). It reads SSH user/key and the manager host from config/env.
```bash
#!/bin/bash
# fleet-control.sh
set -euo pipefail
FLEET_CONFIG="$HOME/.aleph-deploy/configs/fleet.json"
SSH_KEY="${ALEPH_SSH_KEY:-$HOME/.aleph-deploy/keys/aleph_ed25519}"
SSH_USER="$(jq -r '.ssh_user // "root"' "$FLEET_CONFIG" 2>/dev/null || echo root)"
SSH_OPTS=(-i "$SSH_KEY" -o StrictHostKeyChecking=accept-new)
# All fleet manager endpoints require x-api-key auth. Keep the key in fleet.env
# (chmod 600), source it before running, or export it; never hardcode it.
FLEET_API_KEY="${FLEET_API_KEY:?FLEET_API_KEY env var is required (see fleet.env)}"
# Control-plane base URL = primary's TAILSCALE IP:8080 (NOT the public IP).
MGR_HOST="$(jq -r '.primary_node.tailscale_ip // .primary_node.ip' "$FLEET_CONFIG")"
mgr() { # mgr <path> [curl args...]
curl -fsS -H "x-api-key: $FLEET_API_KEY" "http://$MGR_HOST:8080$1" "${@:2}"
}
fleet_status() {
echo "Fleet status:"
mgr /fleet/status | jq '.' || { echo "Unable to reach fleet manager (on tailnet?)"; return 1; }
}
fleet_health() {
echo "Fleet health check:"
local nodes; nodes="$(mgr /fleet/status | jq -r '.nodes[].ip_address')"
for node_ip in $nodes; do
echo "Checking node: $node_ip"
if ssh "${SSH_OPTS[@]}" -o ConnectTimeout=5 "$SSH_USER@$node_ip" \
"systemctl is-active openclaw" &>/dev/null; then
echo " OK $node_ip - OpenClaw running"
else
echo " DOWN $node_ip - OpenClaw not responding"
fi
done
}
fleet_restart() {
local service_name=$1
[[ "$service_name" =~ ^[a-zA-Z0-9_.-]+$ ]] || { echo "Invalid service: $service_name"; return 1; }
echo "Restarting $service_name on all nodes..."
for node_ip in $(mgr /fleet/status | jq -r '.nodes[].ip_address'); do
echo " $node_ip"
ssh "${SSH_OPTS[@]}" "$SSH_USER@$node_ip" "sudo systemctl restart $service_name"
done
}
fleet_deploy() {
local script_path=$1
echo "Deploying script to all nodes: $script_path"
[[ -f "$script_path" ]] || { echo "Script not found: $script_path"; return 1; }
for node_ip in $(mgr /fleet/status | jq -r '.nodes[].ip_address'); do
echo " $node_ip"
scp "${SSH_OPTS[@]}" "$script_path" "$SSH_USER@$node_ip":/tmp/deploy-script.sh
ssh "${SSH_OPTS[@]}" "$SSH_USER@$node_ip" "chmod +x /tmp/deploy-script.sh && sudo /tmp/deploy-script.sh"
done
}
# Real scale operation. Up: create+provision new workers, register them, let
# haproxy-fleet-sync pick them up. Down: drain HAProxy backend, deregister, then
# DELETE the Aleph instance (irreversible for non-persistent volumes — confirmed).
# Requires FLEET_API_KEY, TAILSCALE_AUTH_KEY, and the deploy/worker helpers; the
# simplest robust approach is to re-invoke deploy-fleet.sh's worker function. Here
# we implement it inline so fleet-control.sh is self-contained.
fleet_scale() {
local target=$1
[[ "$target" =~ ^[0-9]+$ ]] || { echo "Target must be an integer"; return 1; }
local cur; cur="$(jq '.worker_nodes | length' "$FLEET_CONFIG")" # worker count
local want=$((target - 1)) # minus the primary
(( want < 0 )) && { echo "Target must be >= 1 (includes primary)"; return 1; }
echo "Scaling workers from $cur to $want (fleet total $((cur+1)) -> $target)..."
local primary_ts; primary_ts="$(jq -r '.primary_node.tailscale_ip' "$FLEET_CONFIG")"
local fleet_name; fleet_name="$(jq -r '.fleet_name' "$FLEET_CONFIG")"
if (( want > cur )); then
: "${TAILSCALE_AUTH_KEY:?TAILSCALE_AUTH_KEY required to add workers}"
command -v aleph >/dev/null || { echo "aleph CLI required on this host to add workers"; return 1; }
for ((i=cur+1; i<=want; i++)); do
local wname="${fleet_name}-worker-${i}" wout whash wip
echo "Adding worker $i ($wname)..."
# Real create (current flags), then poll the REAL `aleph instance list`.
wout="$(aleph instance create --name "$wname" --compute-units 2 --rootfs-size 40960 \
--ssh-pubkey-file "$SSH_KEY.pub" --payment-type credit --payment-chain BASE 2>&1)"
echo "$wout"
whash="$(printf '%s\n' "$wout" | grep -oE '[0-9a-f]{64}' | head -1)"
wip=""
for _ in $(seq 1 30); do
wip="$(aleph instance list --json \
| jq -r --arg n "$wname" '.[]|select(.name==$n)|(.ipv4//.ipv6//empty)' | head -1)"
[[ -n "$wip" ]] && break; sleep 10
done
[[ -z "$wip" ]] && { echo " $wname got no IP — skipping"; continue; }
# Provision over SSH: Tailscale join + register with the primary over the mesh.
# ITEM_HASH is passed through so the primary's registry records this
# instance's hash for later delete/recreate.
ssh "${SSH_OPTS[@]}" "$SSH_USER@$wip" \
"NODE_ID='$wname' PRIMARY_TS_IP='$primary_ts' FLEET_API_KEY='$FLEET_API_KEY' \
TAILSCALE_AUTH_KEY='$TAILSCALE_AUTH_KEY' ITEM_HASH='$whash' bash -s" <<'REPROV'
set -euo pipefail; export DEBIAN_FRONTEND=noninteractive
apt-get update && apt-get install -y curl jq iproute2
installer_7="$(mktemp)"
curl -fsSL https://get.docker.com -o "$installer_7"
less "$installer_7" # Review before execution; verify the release checksum/signature when published.
sh "$installer_7"
rm -f "$installer_7"
installer_8="$(mktemp)"
curl -fsSL https://deb.nodesource.com/setup_22.x -o "$installer_8"
less "$installer_8" # Review before execution; verify the release checksum/signature when published.
bash "$installer_8" - && apt-get install -y nodejs
rm -f "$installer_8"
installer_9="$(mktemp)"
curl -fsSL https://tailscale.com/install.sh -o "$installer_9"
less "$installer_9" # Review before execution; verify the release checksum/signature when published.
sh "$installer_9"
# file: pattern keeps the auth key out of the process list (see Tailscale section)
rm -f "$installer_9"
[[ -n "${TAILSCALE_AUTH_KEY:-}" ]] && { printf '%s' "$TAILSCALE_AUTH_KEY" > /tmp/ts && chmod 600 /tmp/ts && tailscale up --auth-key="file:/tmp/ts" --hostname="$NODE_ID"; rm -f /tmp/ts; }
installer_10="$(mktemp)"
curl -fsSL https://openclaw.ai/install.sh -o "$installer_10"
less "$installer_10" # Review before execution; verify the release checksum/signature when published.
bash "$installer_10"
rm -f "$installer_10"
TS_IP="$(tailscale ip -4 2>/dev/null || hostname -I | awk '{print $1}')"
curl -fsS -X POST "http://$PRIMARY_TS_IP:8080/fleet/register" -H "x-api-key: $FLEET_API_KEY" \
-H 'Content-Type: application/json' \
-d "{\"node_id\":\"$NODE_ID\",\"ip_address\":\"$TS_IP\",\"item_hash\":\"${ITEM_HASH:-}\",\"capabilities\":[\"compute\",\"openclaw\"]}"
REPROV
# Append {name,id,ip,item_hash} to fleet.json atomically.
local wtmp; wtmp="$(mktemp)"
jq --arg n "$wname" --argjson id "$i" --arg ip "$wip" --arg h "$whash" \
'.worker_nodes += [{name:$n, id:$id, crn:"", ip:$ip, item_hash:$h}]' \
"$FLEET_CONFIG" > "$wtmp" && mv "$wtmp" "$FLEET_CONFIG"
echo " Added $wname ($wip)."
done
echo "New workers register automatically; haproxy-fleet-sync adds them within 60s."
elif (( want < cur )); then
local n=$((cur - want))
echo "Removing $n least-recently-active worker(s)..."
# Pick workers to remove (last in the list = most recently added).
local victims; victims="$(jq -r '.worker_nodes[-'"$n"':][] | .name + " " + .ip + " " + .item_hash' "$FLEET_CONFIG")"
while read -r name ip hash; do
[[ -z "$name" ]] && continue
echo "Draining $name ($ip)..."
# 1. Drain in HAProxy so no new requests go to it, then deregister.
ssh "${SSH_OPTS[@]}" "$SSH_USER@$primary_ts" \
"sudo /opt/manage-haproxy-backends.sh remove '$name' || true"
# 2. Confirm before the irreversible delete.
echo "About to DELETE Aleph instance $name ($hash). Non-persistent data is lost."
read -r -p "Type the item-hash to confirm: " ans
if [[ "$ans" == "$hash" ]]; then
aleph instance delete "$hash"
# 3. Atomically drop it from fleet.json.
local tmp; tmp="$(mktemp)"
jq --arg n "$name" '.worker_nodes |= map(select(.name != $n))' \
"$FLEET_CONFIG" > "$tmp" && mv "$tmp" "$FLEET_CONFIG"
echo "Removed $name."
else
echo "Skipped $name (hash mismatch)."
fi
done <<< "$victims"
else
echo "Fleet already at target size."
fi
# Keep node_count in sync with reality.
local tmp; tmp="$(mktemp)"
jq --argjson c "$target" '.node_count=$c' "$FLEET_CONFIG" > "$tmp" && mv "$tmp" "$FLEET_CONFIG"
}
fleet_logs() {
local service_name="${1:-openclaw}" lines="${2:-50}"
[[ "$service_name" =~ ^[a-zA-Z0-9_.-]+$ ]] || { echo "Invalid service: $service_name"; return 1; }
[[ "$lines" =~ ^[0-9]+$ ]] || { echo "Invalid line count: $lines"; return 1; }
echo "Collecting logs from all nodes..."
for node_ip in $(mgr /fleet/status | jq -r '.nodes[].ip_address'); do
echo "=== $node_ip ==="
ssh "${SSH_OPTS[@]}" "$SSH_USER@$node_ip" "sudo journalctl -u $service_name -n $lines --no-pager"
echo ""
done
}
# Command dispatcher
case "${1:-status}" in
"status")
fleet_status
;;
"health")
fleet_health
;;
"restart")
fleet_restart "${2:-openclaw}"
;;
"deploy")
fleet_deploy "$2"
;;
"scale")
fleet_scale "$2"
;;
"logs")
fleet_logs "$2" "$3"
;;
*)
echo "Usage: $0 {status|health|restart|deploy|scale|logs}"
echo ""
echo "Commands:"
echo " status - Show fleet status"
echo " health - Check health of all nodes"
echo " restart [svc] - Restart service on all nodes"
echo " deploy <script> - Deploy script to all nodes"
echo " scale <count> - Scale fleet to N nodes"
echo " logs [svc] [n] - Collect logs from all nodes"
exit 1
;;
esac
```
---