UNPKG

openclaw

Version:

Multi-channel AI gateway with extensible messaging integrations

2,238 lines 95.9 kB
import { c as isRecord, l as isStringRecord } from "./record-coerce-DItp3I4t.js";
import { r as defaultRuntime } from "./runtime-CF2WjnNZ.js";
import { t as attachChildProcessBridge } from "./child-process-bridge-CFJsa4sQ.js";
import { t as FsSafeError, w as root } from "./fs-safe-B6pvPGnf.js";
import { r as isPathInside } from "./path-guards-Cp-mGr3-.js";
import "./utils-P__uGsPB.js";
import { w as resolveStateDir } from "./paths-D2sRr1a_.js";
import { t as formatErrorMessage } from "./errors-Db3Ymjlb.js";
import { n as replaceFileAtomic } from "./replace-file-BAJ-TWzD.js";
import { a as getNodeSqliteKysely, i as executeSqliteQueryTakeFirstSync, r as executeSqliteQuerySync } from "./kysely-sync-COmh4HWh.js";
import { E as tableExists } from "./openclaw-state-db-cache-C7ljO0xP.js";
import { r as withExistingOpenClawStateDatabaseReadOnly } from "./openclaw-state-db-readonly-BRgmrGHt.js";
import { s as runOpenClawStateWriteTransaction } from "./openclaw-state-db-BRTnL-D8.js";
import { r as withContainerEnvFile } from "./container-env-file-BC1o9FW_.js";
import { r as runCommandWithTimeout } from "./exec-BIE-3oLG.js";
import { n as getPublishFileExclusiveFailureDetails, o as publishFileNoClobber } from "./directory-durability-CINgXRM4.js";
import { i as ArchiveSecurityError, n as ArchiveFormatError, r as ArchiveLimitError, t as ARCHIVE_LIMIT_ERROR_CODE, u as extractArchive } from "./archive-BfjFIxAJ.js";
import { n as renderTable, t as getTerminalTableWidth } from "./table-DzN9OC44.js";
import { _ as parseEnvAssignments, a as FLEET_GATEWAY_PORT, b as validateFleetImage, c as allocateHostPort, d as buildCellRunArgs, f as cellAuthSecretDir, g as cellOwnerId, h as cellNetworkName, i as FLEET_ENV_KEYS_LABEL, l as buildCellCreateArgs, m as cellDataDir, n as FLEET_ATTEMPT_LABEL, o as FLEET_OWNER_LABEL, p as cellContainerName, r as FLEET_DISK_LIMIT_LABEL, s as FLEET_TENANT_LABEL, u as buildCellEnvironment, v as validateCellContainerProfile, x as validateTenantId, y as validateDiskSize } from "./cell-profile-C89hQfv6.js";
import { t as createBackupLinkCache } from "./backup-volatile-stat-cache-CCe7X9Xy.js";
import { createWriteStream } from "node:fs";
import path from "node:path";
import fs$1 from "node:fs/promises";
import { spawn } from "node:child_process";
import { createServer } from "node:net";
import os from "node:os";
import { StringDecoder } from "node:string_decoder";
import crypto, { randomUUID } from "node:crypto";
import JSON5 from "json5";
import { pipeline } from "node:stream/promises";
import * as tar from "tar";
//#region src/fleet/registry.ts
const FLEET_OPERATION_LEASE_SCOPE = "fleet-cell-operation";
const FLEET_OPERATION_LEASE_TTL_MS = 3e5;
function kyselyFor(db) {
	return getNodeSqliteKysely(db);
}
function parseRuntime(runtime) {
	if (runtime === "docker" || runtime === "podman") return runtime;
	throw new Error(`Unsupported fleet runtime in state database: ${runtime}`);
}
function rowToRecord(row) {
	return {
		tenantId: row.tenant_id,
		createdAtMs: row.created_at_ms,
		image: row.image,
		runtime: parseRuntime(row.runtime),
		hostPort: row.host_port,
		containerName: row.container_name,
		dataDir: row.data_dir
	};
}
function recordToRow(record) {
	return {
		tenant_id: record.tenantId,
		created_at_ms: record.createdAtMs,
		image: record.image,
		runtime: record.runtime,
		host_port: record.hostPort,
		container_name: record.containerName,
		data_dir: record.dataDir
	};
}
function listFleetCells(env = process.env) {
	return withExistingOpenClawStateDatabaseReadOnly(({ db }) => {
		if (!tableExists(db, "fleet_cells")) return [];
		return executeSqliteQuerySync(db, kyselyFor(db).selectFrom("fleet_cells").selectAll().orderBy("tenant_id", "asc")).rows.map(rowToRecord);
	}, { env }) ?? [];
}
function getFleetCell(env, tenantId) {
	return withExistingOpenClawStateDatabaseReadOnly(({ db }) => {
		if (!tableExists(db, "fleet_cells")) return;
		const row = executeSqliteQueryTakeFirstSync(db, kyselyFor(db).selectFrom("fleet_cells").selectAll().where("tenant_id", "=", tenantId));
		return row ? rowToRecord(row) : void 0;
	}, { env });
}
function reserveFleetCell(env, params) {
	return runOpenClawStateWriteTransaction(({ db }) => {
		const kysely = kyselyFor(db);
		if (executeSqliteQueryTakeFirstSync(db, kysely.selectFrom("fleet_cells").select("tenant_id").where("tenant_id", "=", params.tenantId))) throw new Error(`Fleet cell already exists: ${params.tenantId}`);
		const usedPorts = executeSqliteQuerySync(db, kysely.selectFrom("fleet_cells").select("host_port")).rows.map((row) => row.host_port);
		const hostPort = allocateHostPort(usedPorts, params.requestedPort);
		const record = {
			tenantId: params.tenantId,
			createdAtMs: params.createdAtMs,
			image: params.image,
			runtime: params.runtime,
			hostPort,
			containerName: params.containerName,
			dataDir: params.dataDir
		};
		executeSqliteQuerySync(db, kysely.insertInto("fleet_cells").values(recordToRow(record)));
		return record;
	}, { env });
}
function updateFleetCellImage(env, tenantId, image) {
	runOpenClawStateWriteTransaction(({ db }) => {
		if (executeSqliteQuerySync(db, kyselyFor(db).updateTable("fleet_cells").set({ image }).where("tenant_id", "=", tenantId)).numAffectedRows !== 1n) throw new Error(`Fleet cell disappeared before its image could be updated: ${tenantId}`);
	}, { env });
}
function acquireFleetCellOperation(params) {
	const nowMs = params.nowMs ?? Date.now();
	const expiresAt = nowMs + FLEET_OPERATION_LEASE_TTL_MS;
	const owner = params.owner ?? crypto.randomUUID();
	runOpenClawStateWriteTransaction(({ db }) => {
		const kysely = kyselyFor(db);
		executeSqliteQuerySync(db, kysely.deleteFrom("state_leases").where("scope", "=", FLEET_OPERATION_LEASE_SCOPE).where("lease_key", "=", params.tenantId).where("expires_at", "<=", nowMs));
		const existing = executeSqliteQueryTakeFirstSync(db, kysely.selectFrom("state_leases").select(["expires_at", "payload_json"]).where("scope", "=", FLEET_OPERATION_LEASE_SCOPE).where("lease_key", "=", params.tenantId));
		if (existing) {
			let operation = "fleet operation";
			try {
				const payload = existing.payload_json ? JSON.parse(existing.payload_json) : void 0;
				if (typeof payload === "object" && payload !== null && "operation" in payload && typeof payload.operation === "string") operation = `fleet ${payload.operation}`;
			} catch {}
			throw new Error(`Another ${operation} is already running for ${params.tenantId}; retry after ${new Date(existing.expires_at ?? expiresAt).toISOString()}.`);
		}
		executeSqliteQuerySync(db, kysely.insertInto("state_leases").values({
			scope: FLEET_OPERATION_LEASE_SCOPE,
			lease_key: params.tenantId,
			owner,
			expires_at: expiresAt,
			heartbeat_at: nowMs,
			payload_json: JSON.stringify({ operation: params.operation }),
			created_at: nowMs,
			updated_at: nowMs
		}));
	}, { env: params.env });
	return {
		owner,
		heartbeat: (heartbeatNowMs = Date.now()) => {
			const heartbeatExpiresAt = heartbeatNowMs + FLEET_OPERATION_LEASE_TTL_MS;
			runOpenClawStateWriteTransaction(({ db }) => {
				if (executeSqliteQuerySync(db, kyselyFor(db).updateTable("state_leases").set({
					expires_at: heartbeatExpiresAt,
					heartbeat_at: heartbeatNowMs,
					updated_at: heartbeatNowMs
				}).where("scope", "=", FLEET_OPERATION_LEASE_SCOPE).where("lease_key", "=", params.tenantId).where("owner", "=", owner).where("expires_at", ">", heartbeatNowMs)).numAffectedRows !== 1n) throw new Error(`Fleet operation lease was lost for ${params.tenantId}.`);
			}, { env: params.env });
		},
		release: () => {
			runOpenClawStateWriteTransaction(({ db }) => {
				executeSqliteQuerySync(db, kyselyFor(db).deleteFrom("state_leases").where("scope", "=", FLEET_OPERATION_LEASE_SCOPE).where("lease_key", "=", params.tenantId).where("owner", "=", owner));
			}, { env: params.env });
		}
	};
}
function deleteFleetCell(env, tenantId) {
	runOpenClawStateWriteTransaction(({ db }) => {
		executeSqliteQuerySync(db, kyselyFor(db).deleteFrom("fleet_cells").where("tenant_id", "=", tenantId));
	}, { env });
}
//#endregion
//#region src/fleet/service-support.runtime.ts
const CELL_CONFIG_FILENAME = "openclaw.json";
const HEALTH_TIMEOUT_MS = 1e3;
const CELL_CONFIG_MAX_BYTES = 4194304;
const FLEET_OPERATION_HEARTBEAT_MS = 6e4;
function requiredRecord(value, label) {
	if (!isRecord(value)) throw new Error(`${label} must be an object.`);
	return value;
}
function optionalRecord(value, label) {
	if (value === void 0) return {};
	return requiredRecord(value, label);
}
function readAllowedOrigins(value) {
	if (value === void 0) return [];
	if (!Array.isArray(value) || !value.every((origin) => typeof origin === "string")) throw new Error("gateway.controlUi.allowedOrigins must be an array of strings.");
	return value;
}
async function ensurePrivateDirectory(dir) {
	await fs$1.mkdir(dir, {
		recursive: true,
		mode: 448
	});
	const stat = await fs$1.lstat(dir);
	if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error(`Refusing to use non-directory fleet data path: ${dir}`);
	await fs$1.chmod(dir, 448);
}
async function prepareCellDirectories(record, authSecretDir, owner) {
	await Promise.all([ensurePrivateDirectory(record.dataDir), ensurePrivateDirectory(authSecretDir)]);
	if (owner) await Promise.all([fs$1.chown(record.dataDir, owner.uid, owner.gid), fs$1.chown(authSecretDir, owner.uid, owner.gid)]);
}
async function prepareCellConfig(record, owner) {
	const configPath = path.join(record.dataDir, CELL_CONFIG_FILENAME);
	let rootConfig;
	const cellRoot = await root(record.dataDir, {
		hardlinks: "reject",
		maxBytes: CELL_CONFIG_MAX_BYTES,
		nonBlockingRead: true,
		symlinks: "reject"
	});
	try {
		const read = await cellRoot.read(CELL_CONFIG_FILENAME);
		rootConfig = requiredRecord(JSON5.parse(read.buffer.toString("utf8")), "Cell config");
	} catch (error) {
		if (error instanceof FsSafeError && error.code === "not-found") rootConfig = {};
		else if (error instanceof FsSafeError) throw new Error(`Refusing to read unsafe cell config: ${configPath}`, { cause: error });
		else throw error;
	}
	const gateway = optionalRecord(rootConfig.gateway, "gateway");
	const auth = optionalRecord(gateway.auth, "gateway.auth");
	const controlUi = optionalRecord(gateway.controlUi, "gateway.controlUi");
	const nextAuth = {
		...auth,
		mode: "token"
	};
	delete nextAuth.token;
	const origins = new Set(readAllowedOrigins(controlUi.allowedOrigins));
	origins.add(`http://localhost:${record.hostPort}`);
	origins.add(`http://127.0.0.1:${record.hostPort}`);
	const nextConfig = {
		...rootConfig,
		gateway: {
			...gateway,
			mode: "local",
			bind: "lan",
			auth: nextAuth,
			controlUi: {
				...controlUi,
				allowedOrigins: [...origins]
			}
		}
	};
	await replaceFileAtomic({
		filePath: configPath,
		content: `${JSON.stringify(nextConfig, null, 2)}\n`,
		dirMode: 448,
		mode: 384,
		tempPrefix: CELL_CONFIG_FILENAME,
		copyFallbackOnPermissionError: true
	});
	if (owner) await fs$1.chown(configPath, owner.uid, owner.gid);
}
function readHostIdentity(getuid, getgid) {
	const uid = getuid();
	const gid = getgid();
	if (uid === void 0 || gid === void 0) return;
	if (!Number.isSafeInteger(uid) || uid < 0 || !Number.isSafeInteger(gid) || gid < 0) throw new Error("Host uid and gid must be non-negative integers.");
	return {
		uid,
		gid
	};
}
async function resolveContainerUser(params) {
	const match = params.user?.match(/^(\d+):(\d+)$/u);
	if (match) {
		const uid = Number(match[1]);
		const gid = Number(match[2]);
		return params.runtime === "podman" ? {
			mode: "podman-keep-id",
			uid,
			gid
		} : {
			mode: "numeric",
			uid,
			gid
		};
	}
	if (!params.hostIdentity) return;
	if (params.runtime === "podman") return params.hostIdentity.uid === 0 ? void 0 : {
		mode: "podman-keep-id",
		...params.hostIdentity
	};
	if (await params.containers.isDockerRootless()) return {
		mode: "numeric",
		uid: 0,
		gid: 0
	};
	return params.hostIdentity.uid === 0 ? void 0 : {
		mode: "numeric",
		...params.hostIdentity
	};
}
async function detectHostSelinux() {
	if (process.platform !== "linux") return false;
	try {
		await fs$1.access("/sys/fs/selinux");
		return true;
	} catch {
		return false;
	}
}
function inspectionState(record, inspection) {
	if (inspection.kind !== "ok") return inspection.state;
	return inspection.labels["openclaw.fleet.tenant"] === record.tenantId && inspection.labels["openclaw.fleet.owner"] === cellOwnerId(record.dataDir) ? inspection.state : "unknown";
}
function assertManagedInspection(record, inspection) {
	if (inspection.kind === "missing") throw new Error(`Fleet container is missing for tenant ${record.tenantId}.`);
	if (inspection.kind === "unavailable") throw new Error(`Cannot inspect ${record.runtime} container for tenant ${record.tenantId}: ${inspection.error}`);
	if (inspection.labels["openclaw.fleet.tenant"] !== record.tenantId || inspection.labels["openclaw.fleet.owner"] !== cellOwnerId(record.dataDir)) throw new Error(`Refusing to manage ${record.containerName}: fleet ownership labels do not match tenant ${record.tenantId}.`);
	return inspection;
}
async function probeCellHealth(params) {
	const url = `http://127.0.0.1:${params.port}/healthz`;
	const controller = new AbortController();
	const timeout = setTimeout(() => controller.abort(), HEALTH_TIMEOUT_MS);
	let response;
	try {
		response = await params.fetchImpl(url, {
			method: "GET",
			redirect: "manual",
			signal: controller.signal
		});
		if (response.ok) return {
			status: "ok",
			url,
			httpStatus: response.status
		};
		return {
			status: "failed",
			url,
			httpStatus: response.status,
			error: `HTTP ${response.status}`
		};
	} catch (error) {
		return {
			status: "failed",
			url,
			error: error instanceof Error ? error.message : String(error)
		};
	} finally {
		clearTimeout(timeout);
		await response?.body?.cancel().catch(() => void 0);
	}
}
async function resolvePurgeTarget(rootDir, targetDir, tenantId) {
	const expectedTarget = path.resolve(rootDir, tenantId);
	if (path.resolve(targetDir) !== expectedTarget) throw new Error(`Refusing to purge data outside its fleet-owned directory: ${targetDir}`);
	let targetStat;
	try {
		targetStat = await fs$1.lstat(expectedTarget);
	} catch (error) {
		if (error.code === "ENOENT") return;
		throw error;
	}
	if (targetStat.isSymbolicLink()) throw new Error(`Refusing to purge a symlinked fleet tenant directory: ${targetDir}`);
	const root = await fs$1.realpath(rootDir);
	const target = await fs$1.realpath(targetDir);
	const relative = path.relative(root, target);
	if (target !== path.join(root, tenantId) || !relative || relative === ".." || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) throw new Error(`Refusing to purge data outside its fleet-owned directory: ${targetDir}`);
	return target;
}
function requireCell(env, tenant) {
	const tenantId = validateTenantId(tenant);
	const record = getFleetCell(env, tenantId);
	if (!record) throw new Error(`Fleet cell not found: ${tenantId}`);
	return record;
}
function assertCurrentReservation(env, expected) {
	const current = getFleetCell(env, expected.tenantId);
	if (!current || current.createdAtMs !== expected.createdAtMs || current.image !== expected.image || current.runtime !== expected.runtime || current.hostPort !== expected.hostPort || current.containerName !== expected.containerName || current.dataDir !== expected.dataDir) throw new Error(`Fleet create reservation changed while provisioning ${expected.tenantId}.`);
}
function requirePositiveResource(value, label, context = "upgrade") {
	const parsed = Number(value);
	if (!value.trim() || !Number.isFinite(parsed) || parsed <= 0) throw new Error(`Cannot ${context} cell: inspected ${label} limit is missing or invalid.`);
	return value;
}
function requireInspectedGatewayToken(inspection, context) {
	const gatewayCredential = inspection.environment.OPENCLAW_GATEWAY_TOKEN;
	if (!gatewayCredential) throw new Error(`Cannot ${context} cell: existing container has no Gateway token environment.`);
	return gatewayCredential;
}
function requireInspectedAttemptId(inspection, context) {
	const attemptId = inspection.labels[FLEET_ATTEMPT_LABEL];
	if (!attemptId || !/^[a-f0-9]{32}$/u.test(attemptId)) throw new Error(`Cannot ${context} cell: container attempt label is missing or invalid.`);
	return attemptId;
}
function requirePidsLimit(value, context = "upgrade") {
	if (value === void 0 || !Number.isSafeInteger(value) || value < 1) throw new Error(`Cannot ${context} cell: inspected PID limit is missing or invalid.`);
	return value;
}
function rebuildInspectedEnvironment(environment, labels, token, context = "upgrade") {
	const encodedKeys = labels[FLEET_ENV_KEYS_LABEL];
	if (encodedKeys === void 0) throw new Error(`Cannot ${context} cell: user environment provenance label is missing.`);
	const keys = encodedKeys ? encodedKeys.split(",") : [];
	if (new Set(keys).size !== keys.length || keys.toSorted().join(",") !== encodedKeys) throw new Error(`Cannot ${context} cell: user environment provenance label is invalid.`);
	const assignments = keys.map((key) => {
		const value = environment[key];
		if (value === void 0) throw new Error(`Cannot ${context} cell: inspected environment is missing ${key}.`);
		return `${key}=${value}`;
	});
	return buildCellEnvironment(token, parseEnvAssignments(assignments));
}
function buildProfileBaseFromInspection(params) {
	return {
		tenantId: params.record.tenantId,
		containerName: params.record.containerName,
		networkName: cellNetworkName(params.record.tenantId),
		runtime: params.record.runtime,
		hostPort: params.record.hostPort,
		dataDir: params.record.dataDir,
		authSecretDir: cellAuthSecretDir(params.stateDir, params.record.tenantId),
		ownerId: cellOwnerId(params.record.dataDir),
		memory: requirePositiveResource(params.inspection.memory, "memory", params.context),
		cpus: requirePositiveResource(params.inspection.cpus, "CPU", params.context),
		pidsLimit: requirePidsLimit(params.inspection.pidsLimit, params.context),
		...params.inspection.labels["openclaw.fleet.disk-limit"] !== void 0 ? { diskSize: params.inspection.labels[FLEET_DISK_LIMIT_LABEL] } : {},
		environment: rebuildInspectedEnvironment(params.inspection.environment, params.inspection.labels, params.token, params.context),
		...params.containerUser ? { containerUser: params.containerUser } : {},
		selinuxRelabel: params.selinuxRelabel
	};
}
async function verifyReplacementHealthy(params) {
	const deadline = params.now() + params.timeoutMs;
	for (;;) {
		const replacement = await params.containers.inspect(params.record.runtime, params.record.containerName);
		if (replacement.kind !== "ok" || replacement.labels["openclaw.fleet.attempt"] !== params.attemptId || !replacement.running) throw new Error(replacement.kind === "ok" ? `Replacement cell container is not running after ${params.context}.` : `Replacement cell container could not be verified after ${params.context}.`);
		if ((await probeCellHealth({
			port: params.record.hostPort,
			fetchImpl: params.fetchImpl
		})).status === "ok") return;
		if (params.now() >= deadline) throw new Error(`Replacement cell container did not become healthy after ${params.context}.`);
		params.checkpoint();
		await params.sleep(params.pollMs);
	}
}
async function cleanupFailedCreateContainer(record, containers, attemptId, checkpoint) {
	const inspection = await containers.inspect(record.runtime, record.containerName);
	if (inspection.kind === "missing") return true;
	if (inspection.kind === "unavailable") return false;
	const tenantLabel = inspection.labels[FLEET_TENANT_LABEL];
	const ownerLabel = inspection.labels[FLEET_OWNER_LABEL];
	if (tenantLabel !== record.tenantId || ownerLabel !== cellOwnerId(record.dataDir)) return true;
	if (inspection.labels["openclaw.fleet.attempt"] !== attemptId) return false;
	checkpoint();
	await containers.remove(record.runtime, record.containerName, true);
	return (await containers.inspect(record.runtime, record.containerName)).kind === "missing";
}
async function cleanupFailedCreateNetwork(record, containers, attemptId, checkpoint) {
	const networkName = cellNetworkName(record.tenantId);
	const inspection = await containers.inspectNetwork(record.runtime, networkName);
	if (inspection.kind === "missing") return true;
	if (inspection.kind === "unavailable") return false;
	const tenantLabel = inspection.labels[FLEET_TENANT_LABEL];
	const ownerLabel = inspection.labels[FLEET_OWNER_LABEL];
	if (tenantLabel !== record.tenantId || ownerLabel !== cellOwnerId(record.dataDir)) return true;
	if (inspection.labels["openclaw.fleet.attempt"] !== attemptId || inspection.attachedContainers.length > 0) return false;
	checkpoint();
	await containers.removeNetwork(record.runtime, networkName);
	return (await containers.inspectNetwork(record.runtime, networkName)).kind === "missing";
}
function inspectionHasFleetOwner(record, inspection) {
	return inspection.labels["openclaw.fleet.tenant"] === record.tenantId && inspection.labels["openclaw.fleet.owner"] === cellOwnerId(record.dataDir);
}
function assertManagedNetwork(record, inspection) {
	if (inspection.kind === "missing") throw new Error(`Fleet network is missing for tenant ${record.tenantId}.`);
	if (inspection.kind === "unavailable") throw new Error(`Cannot inspect ${record.runtime} network for tenant ${record.tenantId}: ${inspection.error}`);
	if (inspection.labels["openclaw.fleet.tenant"] !== record.tenantId || inspection.labels["openclaw.fleet.owner"] !== cellOwnerId(record.dataDir)) throw new Error(`Refusing to manage ${cellNetworkName(record.tenantId)}: fleet ownership labels do not match tenant ${record.tenantId}.`);
	if (inspection.attachedContainers.filter((container) => container.name !== record.containerName).length > 0 || inspection.attachedContainers.length > 1) throw new Error(`Refusing to manage ${cellNetworkName(record.tenantId)}: unexpected containers are attached.`);
	return inspection;
}
async function restorePreviousCell(params) {
	const current = await params.containers.inspect(params.record.runtime, params.record.containerName);
	if (current.kind === "unavailable") throw new Error(current.error);
	if (current.kind === "ok") {
		if (!inspectionHasFleetOwner(params.record, current)) throw new Error("container ownership changed during upgrade recovery");
		const currentAttemptId = current.labels[FLEET_ATTEMPT_LABEL];
		if (currentAttemptId === params.previousAttemptId) {
			if (current.running !== params.wasRunning) {
				params.checkpoint();
				await params.containers[current.running ? "stop" : "start"](params.record.runtime, params.record.containerName);
			}
			return;
		}
		if (currentAttemptId !== params.nextAttemptId) throw new Error("container generation changed during upgrade recovery");
		params.checkpoint();
		await params.containers.remove(params.record.runtime, params.record.containerName, true);
	}
	params.checkpoint();
	await params.containers.run(params.oldProfile, params.wasRunning);
}
async function withFleetCellOperation(params) {
	const lease = acquireFleetCellOperation({
		env: params.env,
		tenantId: params.tenantId,
		operation: params.operationName
	});
	let heartbeatError;
	const checkpoint = () => {
		try {
			lease.heartbeat();
			heartbeatError = void 0;
		} catch (error) {
			heartbeatError = error;
			throw error;
		}
	};
	const heartbeat = setInterval(() => {
		try {
			lease.heartbeat();
			heartbeatError = void 0;
		} catch (error) {
			heartbeatError = error;
		}
	}, FLEET_OPERATION_HEARTBEAT_MS);
	heartbeat.unref();
	let result;
	try {
		result = await params.operation(checkpoint);
		if (heartbeatError) checkpoint();
		else lease.heartbeat();
	} catch (error) {
		clearInterval(heartbeat);
		try {
			lease.release();
		} catch {}
		throw error;
	}
	clearInterval(heartbeat);
	lease.release();
	return result;
}
//#endregion
//#region src/fleet/backup.runtime.ts
const DEFAULT_FLEET_BACKUP_MAX_BYTES = 16 * 1024 ** 3;
const FLEET_BACKUP_MAX_ENTRIES = 1e6;
const MANIFEST_MAX_BYTES = 4194304;
const BACKUP_LEASE_PROBE_INTERVAL_MS = 3e4;
const RESTORE_VERIFY_TIMEOUT_MS = 6e4;
const RESTORE_VERIFY_POLL_MS = 1e3;
const RESTORE_EXTRACT_TIMEOUT_MS = 18e5;
function timestampBasename(tenant, nowMs) {
	return `openclaw-fleet-backup-${tenant}-${new Date(nowMs).toISOString().replaceAll(":", "-").replace(/\.\d{3}Z$/u, "Z")}.tgz`;
}
async function resolveOutputPath(out, basename) {
	if (!out) return path.resolve(process.cwd(), basename);
	const resolved = path.resolve(out);
	if (out.endsWith(path.sep) || out.endsWith("/") || out.endsWith("\\")) return path.join(resolved, basename);
	try {
		return (await fs$1.stat(resolved)).isDirectory() ? path.join(resolved, basename) : resolved;
	} catch {
		return resolved;
	}
}
async function canonicalizeForContainment(targetPath) {
	const resolved = path.resolve(targetPath);
	const suffix = [];
	let probe = resolved;
	for (;;) try {
		const real = await fs$1.realpath(probe);
		return path.join(real, ...suffix.toReversed());
	} catch {
		const parent = path.dirname(probe);
		if (parent === probe) return resolved;
		suffix.push(path.basename(probe));
		probe = parent;
	}
}
function remapArchivePath(entryPath, manifestPath, dataTarget, authTarget) {
	const resolved = path.resolve(entryPath);
	if (resolved === manifestPath) return "manifest.json";
	if (isPathInside(dataTarget, resolved)) {
		const relative = path.relative(dataTarget, resolved).split(path.sep).join(path.posix.sep);
		return relative ? path.posix.join("data", relative) : "data";
	}
	if (isPathInside(authTarget, resolved)) {
		const relative = path.relative(authTarget, resolved).split(path.sep).join(path.posix.sep);
		return relative ? path.posix.join("auth", relative) : "auth";
	}
	throw new Error(`Fleet backup encountered a path outside the cell roots: ${entryPath}`);
}
async function backupFleetCell(params) {
	await params.containers.assertLocal(params.record.runtime);
	const inspection = await params.containers.inspect(params.record.runtime, params.record.containerName);
	if (inspection.kind === "unavailable") throw new Error(`Cannot inspect ${params.record.runtime} container for tenant ${params.record.tenantId}: ${inspection.error}`);
	if (inspection.kind === "ok") {
		assertManagedInspection(params.record, inspection);
		if (inspection.running) throw new Error(`Fleet cell ${params.record.tenantId} is running; stop it first (openclaw fleet stop ${params.record.tenantId}) so SQLite state is captured consistently.`);
	}
	const dataTarget = await resolvePurgeTarget(path.join(params.stateDir, "fleet", "cells"), params.record.dataDir, params.record.tenantId);
	if (!dataTarget) throw new Error(`Fleet cell ${params.record.tenantId} has no cell data to back up.`);
	const authTarget = await resolvePurgeTarget(path.join(params.stateDir, "fleet", "auth-profile-secrets"), cellAuthSecretDir(params.stateDir, params.record.tenantId), params.record.tenantId);
	if (!authTarget) throw new Error(`Fleet cell ${params.record.tenantId} has no auth-secret directory to back up.`);
	const nowMs = params.now();
	const archivePath = await resolveOutputPath(params.out, timestampBasename(params.record.tenantId, nowMs));
	const canonicalOutput = await canonicalizeForContainment(archivePath);
	if ([dataTarget, authTarget].some((root) => isPathInside(root, canonicalOutput))) throw new Error("Fleet backup output must not be written inside the cell data or auth directory.");
	try {
		await fs$1.lstat(archivePath);
		throw new Error(`Refusing to overwrite existing fleet backup archive: ${archivePath}`);
	} catch (error) {
		if (error.code !== "ENOENT") throw error;
	}
	await fs$1.mkdir(path.dirname(archivePath), { recursive: true });
	const tempArchivePath = `${archivePath}.${randomUUID()}.tmp`;
	const tempRoot = await fs$1.realpath(os.tmpdir());
	const tempDir = await fs$1.mkdtemp(path.join(tempRoot, "openclaw-fleet-backup-"));
	const manifestPath = path.join(tempDir, "manifest.json");
	const manifest = {
		schemaVersion: 1,
		kind: "openclaw-fleet-cell-backup",
		tenant: params.record.tenantId,
		createdAt: new Date(nowMs).toISOString(),
		hostPort: params.record.hostPort,
		image: params.record.image,
		runtime: params.record.runtime
	};
	let fileCount = 0;
	let skippedSymlinks = 0;
	let skippedSpecial = 0;
	let totalBytes = 0;
	let totalEntries = 0;
	let exceeded = false;
	let tooManyEntries = false;
	let leaseLost = false;
	let unrestorablePath;
	let lastLeaseProbeMs = params.now();
	const maxBytes = params.maxBytes ?? DEFAULT_FLEET_BACKUP_MAX_BYTES;
	const maxEntries = params.maxEntries ?? FLEET_BACKUP_MAX_ENTRIES;
	try {
		await fs$1.writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, { mode: 384 });
		const filter = (entryPath, stat) => {
			if (exceeded || tooManyEntries || leaseLost) return false;
			if (params.now() - lastLeaseProbeMs >= BACKUP_LEASE_PROBE_INTERVAL_MS) {
				lastLeaseProbeMs = params.now();
				try {
					params.checkpoint();
				} catch {
					leaseLost = true;
					return false;
				}
			}
			const type = "type" in stat ? stat.type : void 0;
			if ("isSymbolicLink" in stat ? stat.isSymbolicLink() : type === "SymbolicLink") {
				skippedSymlinks += 1;
				return false;
			}
			const isFile = "isFile" in stat ? stat.isFile() : type === "File";
			const isDirectory = "isDirectory" in stat ? stat.isDirectory() : type === "Directory";
			if (!isFile && !isDirectory) {
				skippedSpecial += 1;
				return false;
			}
			const archivedPath = remapArchivePath(entryPath, manifestPath, dataTarget, authTarget);
			if (!isAllowedRestorePath(archivedPath)) {
				unrestorablePath = unrestorablePath ?? entryPath;
				return false;
			}
			totalEntries += archivedPath.split("/").filter(Boolean).length;
			if (totalEntries > maxEntries) {
				tooManyEntries = true;
				return false;
			}
			if (isFile) {
				totalBytes += stat.size;
				fileCount += 1;
				if (totalBytes > maxBytes) {
					exceeded = true;
					return false;
				}
			}
			return true;
		};
		await pipeline(tar.c({
			gzip: true,
			portable: true,
			preservePaths: true,
			linkCache: createBackupLinkCache(),
			filter,
			onWriteEntry: (entry) => {
				entry.path = remapArchivePath(entry.path, manifestPath, dataTarget, authTarget);
			}
		}, [
			manifestPath,
			dataTarget,
			authTarget
		]), createWriteStream(tempArchivePath, {
			flags: "wx",
			mode: 384
		}));
		try {
			params.checkpoint();
		} catch {
			leaseLost = true;
		}
		if (leaseLost) throw new Error(`Fleet backup for ${params.record.tenantId} lost its operation lease; the partial archive was discarded. Retry the backup.`);
		if (exceeded) throw new Error(`Fleet backup exceeds the ${maxBytes}-byte limit; raise --max-bytes or reduce the cell data.`);
		if (tooManyEntries) throw new Error(`Fleet backup exceeds the ${maxEntries}-entry limit; reduce the number of files in the cell data.`);
		if (unrestorablePath !== void 0) throw new Error(`Fleet backup refuses a file name its restore path rules would reject: ${unrestorablePath}. Rename the file inside the cell and retry.`);
		try {
			await publishFileNoClobber(tempArchivePath, archivePath, {
				strategy: "link-or-copy",
				durability: "degrade"
			});
		} catch (error) {
			if (error.code === "EEXIST") throw new Error(`Refusing to overwrite existing fleet backup archive: ${archivePath}`, { cause: error });
			if (getPublishFileExclusiveFailureDetails(error)?.cleanup === "unknown") throw new Error(`Fleet backup publication failed: ${formatErrorMessage(error)}. A partial archive may remain at ${archivePath}; inspect or remove it before retrying.`, { cause: error });
			throw error;
		}
		return {
			tenant: params.record.tenantId,
			archivePath,
			fileCount,
			skippedSymlinks,
			skippedSpecial,
			note: "Archive contains tenant state and auth secrets; store it like a credential."
		};
	} finally {
		await fs$1.rm(tempArchivePath, { force: true }).catch(() => void 0);
		await fs$1.rm(tempDir, {
			recursive: true,
			force: true
		}).catch(() => void 0);
	}
}
function isAllowedRestorePath(rawPath) {
	if (rawPath.includes("\\")) return false;
	const normalized = path.posix.normalize(rawPath);
	if (normalized !== rawPath || normalized.startsWith("/") || normalized.startsWith("../")) return false;
	return normalized === "manifest.json" || normalized === "data" || normalized.startsWith("data/") || normalized === "auth" || normalized.startsWith("auth/");
}
function resolveRestoreOwner(hostIdentity, containerUser) {
	if (hostIdentity?.uid !== 0) return;
	if (!containerUser) return {
		uid: 1e3,
		gid: 1e3
	};
	return containerUser.uid > 0 ? {
		uid: containerUser.uid,
		gid: containerUser.gid
	} : void 0;
}
async function chownTree(root, owner) {
	await fs$1.chown(root, owner.uid, owner.gid);
	for (const entry of await fs$1.readdir(root, { withFileTypes: true })) {
		const entryPath = path.join(root, entry.name);
		if (entry.isSymbolicLink()) throw new Error(`Refusing to chown symlink in restored fleet data: ${entryPath}`);
		if (entry.isDirectory()) await chownTree(entryPath, owner);
		else await fs$1.chown(entryPath, owner.uid, owner.gid);
	}
}
async function restoreFleetCell(params) {
	await params.containers.assertLocal(params.record.runtime);
	const archivePath = path.resolve(params.from);
	const archiveStat = await fs$1.lstat(archivePath);
	if (!archiveStat.isFile() || archiveStat.isSymbolicLink()) throw new Error(`Fleet restore archive must be a regular file: ${archivePath}`);
	const canonicalArchive = await fs$1.realpath(archivePath);
	if ((await Promise.all([canonicalizeForContainment(params.record.dataDir), canonicalizeForContainment(cellAuthSecretDir(params.stateDir, params.record.tenantId))])).some((root) => isPathInside(root, canonicalArchive))) throw new Error("Fleet restore archive must not be stored inside the cell data or auth directory.");
	const inspectionResult = await params.containers.inspect(params.record.runtime, params.record.containerName);
	if (inspectionResult.kind === "missing") throw new Error(`Fleet cell container is missing for ${params.record.tenantId}; remove the stale registration without purging data (openclaw fleet rm ${params.record.tenantId} --force), recreate a stopped cell with the intended image (openclaw fleet create ${params.record.tenantId} --no-start --image <image>), then retry fleet restore.`);
	const inspection = assertManagedInspection(params.record, inspectionResult);
	if (inspection.running && !params.force) throw new Error(`Fleet cell ${params.record.tenantId} is running; pass --force to stop it and replace its state.`);
	const wasRunning = inspection.running;
	requireInspectedGatewayToken(inspection, "restore");
	requireInspectedAttemptId(inspection, "restore");
	const tempRoot = path.join(params.stateDir, "fleet", "restore-tmp");
	await fs$1.mkdir(tempRoot, {
		recursive: true,
		mode: 448
	});
	const tempDir = await fs$1.mkdtemp(path.join(tempRoot, `${params.record.tenantId}-`));
	await fs$1.chmod(tempDir, 448);
	let preserveTemp = false;
	let stoppedForRestore = false;
	let containerRemoved = false;
	let previousDisplaced = false;
	let stateSwapped = false;
	let replacementAttemptId = "";
	try {
		let invalidArchive = false;
		let totalPathComponents = 0;
		let tooManyPathComponents = false;
		const maxBytes = params.maxBytes ?? DEFAULT_FLEET_BACKUP_MAX_BYTES;
		const maxEntries = params.maxEntries ?? FLEET_BACKUP_MAX_ENTRIES;
		try {
			await extractArchive({
				archivePath,
				destDir: tempDir,
				kind: "tar",
				tarGzip: true,
				timeoutMs: RESTORE_EXTRACT_TIMEOUT_MS,
				entryModes: "clamp",
				entryFilter: (entry) => {
					totalPathComponents += entry.path.split("/").filter(Boolean).length;
					if (totalPathComponents > maxEntries) {
						tooManyPathComponents = true;
						return "skip";
					}
					if (entry.kind !== "file" && entry.kind !== "directory" || !isAllowedRestorePath(entry.path)) {
						invalidArchive = true;
						return "skip";
					}
					return "extract";
				},
				onFiltered: "reject-archive",
				limits: {
					maxArchiveBytes: Math.max(1, archiveStat.size),
					maxEntries,
					maxExtractedBytes: maxBytes,
					maxEntryBytes: maxBytes,
					maxEntryPathComponents: maxEntries
				}
			});
		} catch (error) {
			if (tooManyPathComponents || error instanceof ArchiveLimitError && (error.code === ARCHIVE_LIMIT_ERROR_CODE.ENTRY_COUNT_EXCEEDS_LIMIT || error.code === ARCHIVE_LIMIT_ERROR_CODE.ENTRY_PATH_COMPONENTS_EXCEEDS_LIMIT)) throw new Error(`Fleet restore exceeds the ${maxEntries}-entry limit; the archive is not a usable fleet cell backup.`, { cause: error });
			if (error instanceof ArchiveLimitError && (error.code === ARCHIVE_LIMIT_ERROR_CODE.EXTRACTED_SIZE_EXCEEDS_LIMIT || error.code === ARCHIVE_LIMIT_ERROR_CODE.ENTRY_EXTRACTED_SIZE_EXCEEDS_LIMIT)) throw new Error(`Fleet restore exceeds the ${maxBytes}-byte limit; raise --max-bytes or use a smaller archive.`, { cause: error });
			if (invalidArchive || error instanceof ArchiveSecurityError || error instanceof ArchiveFormatError) throw new Error("Archive is not a fleet cell backup or was tampered with.", { cause: error });
			throw error;
		}
		const safeRoot = await root(tempDir, {
			symlinks: "reject",
			hardlinks: "reject",
			maxBytes: MANIFEST_MAX_BYTES,
			nonBlockingRead: true
		});
		let manifest;
		try {
			manifest = JSON.parse((await safeRoot.read("manifest.json")).buffer.toString("utf8"));
		} catch (error) {
			throw new Error("Archive is not a fleet cell backup or was tampered with.", { cause: error });
		}
		if (typeof manifest !== "object" || manifest === null || !("kind" in manifest) || manifest.kind !== "openclaw-fleet-cell-backup" || !("schemaVersion" in manifest) || manifest.schemaVersion !== 1 || !("tenant" in manifest) || typeof manifest.tenant !== "string") throw new Error("Archive is not a fleet cell backup or was tampered with.");
		if (manifest.tenant !== params.record.tenantId) throw new Error(`Backup archive belongs to tenant ${manifest.tenant}; refusing to restore it into ${params.record.tenantId}.`);
		const extractedData = path.join(tempDir, "data");
		const dataStat = await fs$1.lstat(extractedData);
		if (!dataStat.isDirectory() || dataStat.isSymbolicLink()) throw new Error("Archive is not a fleet cell backup or was tampered with.");
		const extractedAuth = path.join(tempDir, "auth");
		let authStat;
		try {
			authStat = await fs$1.lstat(extractedAuth);
		} catch (error) {
			if (error.code === "ENOENT") throw new Error("Archive is not a fleet cell backup or was tampered with.", { cause: error });
			throw error;
		}
		if (!authStat.isDirectory() || authStat.isSymbolicLink()) throw new Error("Archive is not a fleet cell backup or was tampered with.");
		const containerUser = await resolveContainerUser({
			runtime: params.record.runtime,
			containers: params.containers,
			hostIdentity: params.hostIdentity,
			user: inspection.user
		});
		const imageOwner = resolveRestoreOwner(params.hostIdentity, containerUser);
		const token = params.generateToken();
		const attemptId = params.generateAttemptId();
		replacementAttemptId = attemptId;
		const profile = {
			...buildProfileBaseFromInspection({
				record: params.record,
				stateDir: params.stateDir,
				inspection,
				containerUser,
				selinuxRelabel: params.selinuxRelabel,
				token,
				context: "restore"
			}),
			image: inspection.imageId,
			attemptId
		};
		validateCellContainerProfile(profile);
		const authSecretDir = cellAuthSecretDir(params.stateDir, params.record.tenantId);
		const dataTarget = await resolvePurgeTarget(path.join(params.stateDir, "fleet", "cells"), params.record.dataDir, params.record.tenantId);
		const authTarget = await resolvePurgeTarget(path.join(params.stateDir, "fleet", "auth-profile-secrets"), authSecretDir, params.record.tenantId);
		const replacedRoot = path.join(tempDir, "replaced");
		await fs$1.mkdir(replacedRoot, { mode: 448 });
		assertManagedNetwork(params.record, await params.containers.inspectNetwork(params.record.runtime, cellNetworkName(params.record.tenantId)));
		if (wasRunning) {
			params.checkpoint();
			assertManagedInspection(params.record, await params.containers.inspect(params.record.runtime, params.record.containerName));
			await params.containers.stop(params.record.runtime, params.record.containerName);
			stoppedForRestore = true;
		}
		params.checkpoint();
		assertManagedInspection(params.record, await params.containers.inspect(params.record.runtime, params.record.containerName));
		await params.containers.remove(params.record.runtime, params.record.containerName, false);
		containerRemoved = true;
		params.checkpoint();
		previousDisplaced = true;
		if (dataTarget) await fs$1.rename(dataTarget, path.join(replacedRoot, "data"));
		if (authTarget) await fs$1.rename(authTarget, path.join(replacedRoot, "auth"));
		await fs$1.rename(extractedData, params.record.dataDir);
		await fs$1.rename(extractedAuth, authSecretDir);
		stateSwapped = true;
		await prepareCellDirectories(params.record, authSecretDir, imageOwner);
		if (imageOwner) await Promise.all([chownTree(params.record.dataDir, imageOwner), chownTree(authSecretDir, imageOwner)]);
		await prepareCellConfig(params.record, imageOwner);
		params.checkpoint();
		await params.containers.run(profile, wasRunning);
		if (wasRunning) await verifyReplacementHealthy({
			containers: params.containers,
			record: params.record,
			attemptId,
			fetchImpl: params.fetchImpl,
			now: params.now,
			sleep: params.sleep,
			checkpoint: params.checkpoint,
			timeoutMs: RESTORE_VERIFY_TIMEOUT_MS,
			pollMs: RESTORE_VERIFY_POLL_MS,
			context: "restore"
		});
		return {
			tenant: params.record.tenantId,
			archivePath,
			token,
			tokenNote: "Shown once. The previous Gateway token was rotated by this restore.",
			started: wasRunning,
			url: `http://127.0.0.1:${params.record.hostPort}`
		};
	} catch (error) {
		if (containerRemoved) {
			preserveTemp = true;
			let replacementNote = "";
			try {
				const current = await params.containers.inspect(params.record.runtime, params.record.containerName);
				if (current.kind === "ok" && current.labels["openclaw.fleet.attempt"] === replacementAttemptId && current.running) {
					await params.containers.stop(params.record.runtime, params.record.containerName);
					replacementNote = " The interrupted replacement container was stopped; retry fleet restore to rotate a fresh Gateway token.";
				} else if (current.kind === "unavailable") replacementNote = " The replacement container state could not be verified; stop it manually before retrying fleet restore.";
			} catch {
				replacementNote = " The interrupted replacement container could not be stopped; stop it manually before retrying fleet restore.";
			}
			const recoveryNote = stateSwapped ? ` Restored data is already in place under the cell directories; displaced previous data is preserved at ${tempDir}/replaced.` : previousDisplaced ? ` Previous data is preserved at ${tempDir}/replaced and the extracted archive remains at ${tempDir}.` : ` Previous cell data remains in place; the extracted archive remains at ${tempDir}.`;
			throw new Error(`Fleet restore for ${params.record.tenantId} was interrupted after the cell container was removed: ${formatErrorMessage(error)}.${replacementNote}${recoveryNote}`, { cause: error });
		}
		if (stoppedForRestore) try {
			const current = assertManagedInspection(params.record, await params.containers.inspect(params.record.runtime, params.record.containerName));
			if (!current.running && current.labels["openclaw.fleet.attempt"] === inspection.labels["openclaw.fleet.attempt"]) await params.containers.start(params.record.runtime, params.record.containerName);
		} catch {}
		throw error;
	} finally {
		if (!preserveTemp) await fs$1.rm(tempDir, {
			recursive: true,
			force: true
		}).catch(() => void 0);
	}
}
//#endregion
//#region src/fleet/containers.redaction.ts
function secretPrefixSuffixLength(text, redactValues) {
	let longest = 0;
	for (const value of redactValues) {
		const max = Math.min(value.length - 1, text.length);
		for (let length = max; length > longest; length -= 1) if (value.startsWith(text.slice(text.length - length))) {
			longest = length;
			break;
		}
	}
	return longest;
}
function createRedactingStreamWriter(target, redactValues) {
	const decoder = new StringDecoder("utf8");
	let pending = "";
	const redact = (text) => {
		let redacted = text;
		for (const value of redactValues) if (value) redacted = redacted.replaceAll(value, "<redacted>");
		return redacted;
	};
	const emit = (text) => {
		if (!text) return true;
		return target.write(redact(text));
	};
	return {
		write: (chunk) => {
			pending += decoder.write(chunk);
			const keep = secretPrefixSuffixLength(pending, redactValues);
			const cut = pending.length - keep;
			if (cut <= 0) return true;
			const writable = emit(pending.slice(0, cut));
			pending = pending.slice(cut);
			return writable;
		},
		flush: () => {
			emit(pending + decoder.end());
			pending = "";
		}
	};
}
//#endregion
//#region src/fleet/containers.runtime.ts
const DELIBERATE_STREAM_STOP_SIGNALS = /* @__PURE__ */ new Set([
	"SIGINT",
	"SIGTERM",
	"SIGHUP",
	"SIGQUIT",
	"SIGBREAK",
	"SIGPIPE"
]);
const COMMAND_TIMEOUT_MS = 6e5;
const COMMAND_MAX_OUTPUT_BYTES = 4194304;
var InvalidInspectOutputError = class extends Error {
	constructor() {
		super("container inspect returned an invalid response");
	}
};
function requireRecord(value) {
	if (!isRecord(value)) throw new InvalidInspectOutputError();
	return value;
}
function requireString(value) {
	if (typeof value !== "string" || value.length === 0) throw new InvalidInspectOutputError();
	return value;
}
function requireBoolean(value) {
	if (typeof value !== "boolean") throw new InvalidInspectOutputError();
	return value;
}
function requireNonNegativeNumber(value) {
	if (typeof value !== "number" || !Number.isFinite(value) || value < 0) throw new InvalidInspectOutputError();
	return value;
}
function readOptionalInspectString(value) {
	if (value === void 0 || value === null || value === "") return;
	if (typeof value !== "string") throw new InvalidInspectOutputError();
	return value;
}
function readLabels(value) {
	if (value === void 0 || value === null) return {};
	const record = requireRecord(value);
	const labels = {};
	for (const [key, label] of Object.entries(record)) {
		if (typeof label !== "string") throw new InvalidInspectOutputError();
		labels[key] = label;
	}
	return labels;
}
function readStringRecord(value) {
	if (value === void 0 || value === null) return {};
	if (!isStringRecord(value)) throw new InvalidInspectOutputError();
	return value;
}
function readStringArray(value) {
	if (value === void 0 || value === null) return [];
	if (!Array.isArray(value) || !value.every((entry) => typeof entry === "string")) throw new InvalidInspectOutputError();
	return value;
}
function readOptionalBoolean(value) {
	if (value === void 0 || value === null) return;
	return requireBoolean(value);
}
function readRestartPolicy(value) {
	if (value === void 0 || value === null) return;
	return readOptionalInspectString(requireRecord(value).Name);
}
function readPortBindings(value) {
	if (value === void 0 || value === null) return [];
	const bindings = [];
	for (const [containerPort, rawEntries] of Object.entries(requireRecord(value))) {
		if (!containerPort || !Array.isArray(rawEntries)) throw new InvalidInspectOutputError();
		for (const rawEntry of rawEntries) {
			const entry = requireRecord(rawEntry);
			bindings.push({
				containerPort,
				hostIp: requireString(entry.HostIp),
				hostPort: requireString(entry.HostPort)
			});
		}
	}
	return bindings;
}
function readNetworkAttachments(value) {
	if (value === void 0 || value === null) return [];
	const record = requireRecord(value);
	return Object.entries(record).map(([id, rawAttachment]) => {
		if (!id) throw new InvalidInspectOutputError();
		const attachment = requireRecord(rawAttachment);
		const name = readOptionalInspectString(attachment.Name ?? attachment.name);
		const normalized = { id };
		if (name) normalized.name = name;
		return normalized;
	}).toSorted((left, right) => left.id.localeCompare(right.id));
}
function readEnvironment(value) {
	if (value === void 0 || value === null) return {};
	if (!Array.isArray(value) || !value.every((entry) => typeof entry === "string")) throw new InvalidInspectOutputError();
	const environment = {};
	for (const assignment of value) {
		const separator = assignment.indexOf("=");
		if (separator <= 0) throw new InvalidInspectOutputError();
		environment[assignment.slice(0, separator)] = assignment.slice(separator + 1);
	}
	return environment;
}
function readPidsLimit(value) {
	if (value === void 0 || value === null || value === 0) return;
	if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) throw new InvalidInspectOutputError();
	return value;
}
function parseInspectOutput(stdout) {
	let parsed;
	try {
		parsed = JSON.parse(stdout);
	} catch {
		throw new InvalidInspectOutputError();
	}
	if (!Array.isArray(parsed) || parsed.length !== 1) throw new InvalidInspectOutputError();
	const inspected = requireRecord(parsed[0]);
	const state = requireRecord(inspected.State);
	const config = requireRecord(inspected.Config);
	const hostConfig = requireRecord(inspected.HostConfig);
	const nanoCpus = requireNonNegativeNumber(hostConfig.NanoCpus);
	const user = readOptionalInspectString(config.User);
	const usernsMode = readOptionalInspectString(hostConfig.UsernsMode);
	return {
		kind: "ok",
		containerId: requireString(inspected.Id),
		state: requireString(state.Status),
		running: requireBoolean(state.Running),
		labels: readLabels(config.Labels),
		environment: readEnvironment(config.Env),
		imageId: requireString(inspected.Image),
		memory: String(requireNonNegativeNumber(hostConfig.Memory)),
		cpus: String(nanoCpus / 1e9),
		pidsLimit: readPidsLimit(hostConfig.PidsLimit),
		storageOpt: readStringRecord(hostConfig.StorageOpt),
		capDrop: readStringArray(hostConfig.CapDrop),
		effectiveCaps: inspected.EffectiveCaps === void 0 ? void 0 : readStringArray(inspected.EffectiveCaps),
		securityOpt: readStringArray(hostConfig.SecurityOpt),
		init: readOptionalBoolean(hostConfig.Init),
		restartPolicy: readRestartPolicy(hostConfig.RestartPolicy),
		portBindings: readPortBindings(hostConfig.PortBindings),
		...user ? { user } : {},
		...usernsMode ? { usernsMode } : {}
	};
}
function parseNetworkInspectOutput(stdout) {
	let parsed;
	try {
		parsed = JSON.parse(stdout);
	} catch {
		throw new InvalidInspectOutputError();
	}
	if (!Array.isArray(parsed) || parsed.length !== 1) throw new InvalidInspectOutputError();
	const inspected = requireRecord(parsed[0]);
	return {
		kind: "ok",
		labels: readLabels(inspected.Labels ?? inspected.labels),
		attachedContainers: readNetworkAttachments(inspected.Containers ?? inspected.containers),
		internal: readOptionalBoolean(inspected.Internal ?? inspected.internal) ?? false
	};
}
function parseDockerContextEndpoint(stdout) {
	let parsed;
	try {
		parsed = JSON.parse(stdout);
	} catch {
		throw new Error("docker context inspect returned an invalid response");
	}
	if (!Array.isArray(parsed) || parsed.length !== 1) throw new Error("docker context inspect returned an invalid response");
	try {
		return requireString(requireRecord(requireRecord(requireRecord(parsed[0]).Endpoints).docker).Host);
	} catch {
		throw new Error("docker context inspect returned an invalid response");
	}
}
function isLocalDockerEndpoint(endpoint) {
	const normalized = endpoint.toLowerCase();
	if (normalized.startsWith("unix:///")) return normalized.length > 8;
	return process.platform === "win32" && normalized.startsWith("npipe:////./pipe/") && normalized.length > 17;
}
function parsePodmanServiceIsRemote(stdout) {
	let parsed;
	try {
		parsed = JSON.parse(stdout);
	} catch {
		throw new Error("podman info returned an invalid response");
	}
	try {
		const serviceIsRemote = requireRecord(requireRecord(parsed).host).serviceIsRemote;
		if (typeof serviceIsRemote !== "boolean") throw new Error();
		return serviceIsRemote;
	} catch {
		throw new Error("podman info returned an invalid response");
	}
}
function parseDockerRootlessInfo(stdout) {
	let parsed;
	try {
		parsed = JSON.parse(stdout);
	} catch {
		throw new Error("docker info returned an invalid security-options response");
	}
	if (!Array.isArray(parsed) || !parsed.every((entry) => typeof entry === "string")) throw new Error("docker info returned an invalid security-options response");
	return parsed.some((entry) => entry.split(",").includes("name=rootless"));
}
function readEnvironmentValues(args, extraValues) {
	const values = [];
	for (let index = 0; index < args.length; index += 1) {
		const arg = args[index] ?? "";
		let assignment;
		if (arg === "--env" || arg === "-e") {
			assignment = args[index + 1];
			index += 1;
		} else if (arg.startsWith("--env=")) assignment = arg.slice(6);
		else if (arg.startsWith("-e=")) assignment = arg.slice(3);
		if (!assignment) continue;
		const separator = assignment.indexOf("=");
		if (separator >= 0 && separator < assignment.length - 1) values.push(assignment.slice(separator + 1));
	}
	for (const value of extraValues ?? []) if (value) values.push(value);
	return [...new Set(values)];
}
function redactEnvironmentValues(text, args, extraValues) {
	let redacted = text;
	const values = readEnvironmentValues(args, extraValues).toSorted((left, right) => right.length - left.length);
	for (const value of values) redacted = redacted.replaceAll(value, "<redacted>");
	return redacted;
}
function formatExecutorError(error, runtime, args, extraValues) {
	const detail = error instanceof Error ? redactEnvironmentValues(error.message, args, extraValues).trim() : "";
	return new Error(detail || `${runtime} container command failed`);
}
function commandFailureError(runtime, args, result, extraValues) {
	const detail = redactEnvironmentValues(result.stderr, args, extraValues).trim();
	return new Error(detail || `${runtime} container command failed with exit code ${result.code}`);
}
const defaultFleetContainerCommandExecutor = async (runtime, args, options) => {
	const result = await runCommandWithTimeout([runtime, ...args], {
		timeoutMs: COMMAND_TIMEOUT_MS,
		maxOutputBytes: COMMAND_MAX_OUTPUT_BYTES
	});
	const normalized = {
		stdout: result.stdout,
		stderr: redactEnvironmentValues(result.stderr, args, options.redactValues),
		code: result.code ?? 1
	};
	if (normalized.code !== 0 && !options.allowFailure) throw commandFailureError(runtime, args, normalized, options.redactValues);
	return normalized;
};
const defaultFleetContainerStreamExecutor = (runtime, args, options) => new Promise((resolve, reject) => {
	const child = spawn(runtime, args, { stdio: [
		"ignore",
		"pipe",
		"pipe"
	] });
	attachChildProcessBridge(child);
	const stdout = createRedactingStreamWriter(process.stdout, options.redactValues);
	const stderr = createRedactingStreamWriter(process.stderr, options.redactValues);
	const onTargetError = () => {
		child.kill("SIGTERM");
	};
	process.stdout.once("error", onTargetError);
	process.stderr.once("error", onTargetError);
	const pipeWithBackpressure = (source, targetStream, writer) => {
		source?.on("data", (chunk) => {
			if (!writer.write(chunk)) {
				source.pause();
				targetStream.once("drain", () => source.resume());
			}
		});
	};
	pipeWithBackpressure(child.stdout, process.stdout, stdout);
	pipeWithBackpressure(child.stderr, process.stderr, stderr);
	child.on("error", (error) => {
		if (child.pid === void 0) reject(error);
	});
	child.once("close", (code, signal) => {
		stdout.flush();
		stderr.flush();
		process.stdout.removeListener("error", onTargetError);
		process.stderr.removeListener("error", onTargetError);
		resolve({
			code,
			signal
		});
	});
});
function isMissingContainerError(stderr) {
	const normalized = stderr.toLowerCase();
	return normalized.includes("no such object") || normalized.includes("no such container") || normalized.includes("no container with name or id") || /container .+ does not exist/u.test(normalized);
}
function isMissingNetworkError(stderr) {
	const normalized = stderr.toLowerCase();
	return normalized.includes("no such network") || normalized.includes("network not found") || /network .+ not found/u.test(normalized) || /network .+ does not exist/u.test(normalized);
}
function validateNetworkName(networkName) {
	const normalized = networkName.trim();
	if (!normalized || normalized.startsWith("-")) throw new Error("Fleet network name is invalid.");
	return normalized;
}
function validateContainerName(containerName) {
	const normalized = containerName.trim();
	if (!normalized || normalized.startsWith("-")) throw new Error("Fleet container name is invalid.");
	return normalized;
}
function buildLogsArgs(containerName, options) {
	const args = ["logs"];
	if (options.follow) args.push("--follow");
	if (options.tail !== void 0) {
		if (!Number.isSafeInteger(options.tail) || options.tail < 1) throw new Error("Fleet logs --tail must be a positive integer.");
		args.push("--tail", String(options.tail));
	}
	if (options.since !== void 0) {
		if (!options.since || options.since.startsWith("-") || /[\s\p{Cc}]/u.test(options.since)) throw new Error("Fleet logs --since must be non-empty, must not start with '-', and must not contain whitespace or control characters.");
		args.push("--since", options.since);
	}
	args.push(validateContainerName(containerName));
	return args;
}
function createFleetContainerRuntime(executor = defaultFleetContainerCommandExecutor, streamExecutor = defaultFleetContainerStreamExecutor) {
	const execute = async (runtime, args, options = {}) => {
		try {
			const result = await executor(runtime, args, options);
			if (result.code !== 0 && !options.allowFailure) throw commandFailureError(runtime, args, result, options.redactValues);
			return {
				...result,
				stderr: redactEnvironmentValues(result.stderr, args, options.redactValues)
			};
		} catch (error) {
			throw formatExecutorError(error, runtime, args, options.redactValues);
		}
	};
	return {
		async assertLocal(runtime) {
			if (runtime === "podman") {
				if (parsePodmanServiceIsRemote((await execute("podman", [
					"info",
					"--format",
					"json"
				])).stdout)) throw new Error("Fleet requires local Podman; remote cells are not supported.");
				return;
			}
			if (!isLocalDockerEndpoint(parseDockerContextEndpoint((await execute("docker", ["context", "inspect"])).stdout))) throw new Error("Fleet requires a local Docker endpoint; remote cells are not supported.");
		},
		async inspect(runtime, containerName) {
			const args = [
				"container",
				"inspect",
				validateContainerName(containerName)
			];
			let result;
			try {
				result = await execute(runtime, args, { allowFailure: true });
			} catch (error) {
				return {
					kind: "unavailable",
					state: "unknown",
					error: formatExecutorError(error, runtime, args).message
				};
			}
			if (result.code !== 0) {
				if (isMissingContainerError(result.stderr)) return {
					kind: "missing",
					state: "missing"
				};
				return {
					kind: "unavailable",
					state: "unknown",
					error: result.stderr.trim() || `${runtime} container inspect failed`
				};
			}
			try {
				return parseInspectOutput(result.stdout);
			} catch {
				return {
					kind: "unavailable",
					state: "unknown",
					error: "container inspect returned an invalid response"
				};
			}
		},
		async inspectNetwork(runtime, networkName) {
			const args = [
				"network",
				"inspect",
				validateNetworkName(networkName)
			];
			let result;
			try {
				result = await execute(runtime, args, { allowFailure: true });
			} catch (error) {
				return {
					kind: "unavailable",
					error: formatExecutorError(error, runtime, args).message
				};
			}
			if (result.code !== 0) {
				if (isMissingNetworkError(result.stderr)) return { kind: "missing" };
				return {
					kind: "unavailable",
					error: result.stderr.trim() || `${runtime} network inspect failed`
				};
			}
			try {
				return parseNetworkInspectOutput(result.stdout);
			} catch {
				return {
					kind: "unavailable",
					error: "network inspect returned an invalid response"
				};
			}
		},
		async isDockerRootless() {
			return parseDockerRootlessInfo((await execute("docker", [
				"info",
				"--format",
				"{{json .SecurityOptions}}"
			])).stdout);
		},
		async run(profile, start) {
			validateCellContainerProfile(profile);
			await withContainerEnvFile(profile.environment, async (environmentFile) => {
				const args = start ? buildCellRunArgs(profile, { environmentFile }) : buildCellCreateArgs(profile, { environmentFile });
				await execute(profile.runtime, args, { redactValues: Object.values(profile.environment) });
			});
		},
		async pull(runtime, image) {
			await execute(runtime, ["pull", validateFleetImage(image)]);
		},
		async createNetwork(runtime, networkName, labels, options) {
			const labelArgs = Object.entries(labels).toSorted(([left], [right]) => left.localeCompare(right)).flatMap(([key, value]) => ["--label", `${key}=${value}`]);
			await execute(runtime, [
				"network",
				"create",
				"--driver",
				"bridge",
				...options.internal ? ["--internal"] : [],
				...labelArgs,
				validateNetworkName(networkName)
			]);
		},
		async removeNetwork(runtime, networkName) {
			await execute(runtime, [
				"network",
				"rm",
				validateNetworkName(networkName)
			]);
		},
		async start(runtime, containerName) {
			await execute(runtime, ["start", validateContainerName(containerName)]);
		},
		async stop(runtime, containerName) {
			await execute(runtime, ["stop", validateContainerName(containerName)]);
		},
		async restart(runtime, containerName) {
			await execute(runtime, ["restart", validateContainerName(containerName)]);
		},
		async logs(runtime, containerName, options) {
			const result = await streamExecutor(runtime, buildLogsArgs(containerName, options), { redactValues: options.redactValues });
			if (result.code === 0) return;
			const deliberateStop = result.signal !== null && DELIBERATE_STREAM_STOP_SIGNALS.has(result.signal);
			if (options.follow && (deliberateStop || result.code === 130)) return;
			throw new Error(`${runtime} logs failed with ${result.signal ? `signal ${result.signal}` : `exit code ${result.code ?? 1}`}.`);
		},
		async remove(runtime, containerName, force) {
			await execute(runtime, [
				"rm",
				...force ? ["--force"] : [],
				validateContainerName(containerName)
			]);
		}
	};
}
//#endregion
//#region src/fleet/doctor.runtime.ts
function finding(check, status, detail) {
	return {
		check,
		status,
		detail
	};
}
async function directoryFindings(params) {
	try {
		const stat = await fs$1.lstat(params.dir);
		if (!stat.isDirectory() || stat.isSymbolicLink()) return [finding(params.check, "fail", `${params.dir} is not a real directory.`)];
		const findings = [(stat.mode & 511) === 448 ? finding(params.check, "pass", `${params.dir} is a private 0700 directory.`) : finding(params.check, "fail", `${params.dir} has mode 0${(stat.mode & 511).toString(8)}; expected 0700.`)];
		if (params.expectedUid !== void 0 && params.expectedUid > 0 && stat.uid !== params.expectedUid) findings.push(finding(`${params.check}-owner`, "warn", `${params.dir} is owned by uid ${stat.uid}; container uid is ${params.expectedUid}.`));
		return findings;
	} catch (error) {
		return [finding(params.check, "fail", `${params.dir} cannot be inspected: ${error instanceof Error ? error.message : String(error)}.`)];
	}
}
function diskLimitFinding(runtime, diskLimit, applied) {
	try {
		validateDiskSize(diskLimit);
	} catch {
		return finding("disk-limit", "fail", `Container disk limit label is malformed and would break upgrade/restore replay: ${diskLimit}.`);
	}
	if (runtime === "docker" && applied !== diskLimit) return finding("disk-limit", "fail", `Container disk limit label is ${diskLimit} but the applied storage option is ${applied ?? "unset"}.`);
	return finding("disk-limit", "pass", runtime === "docker" ? `Container writable-layer disk limit is ${diskLimit}.` : `Container writable-layer disk limit was requested as ${diskLimit}; Podman does not expose the applied storage option for verification.`);
}
async function runFleetDoctor(params) {
	const records = params.tenant ? [requireCell(params.env, params.tenant)] : listFleetCells(params.env);
	const stateDir = resolveStateDir(params.env);
	return await Promise.all(records.map(async (record) => {
		const findings = [];
		try {
			await params.containers.assertLocal(record.runtime);
			findings.push(finding("runtime-local", "pass", `${record.runtime} uses a local container endpoint.`));
		} catch (error) {
			findings.push(finding("runtime-local", "fail", `${record.runtime} locality check failed: ${error instanceof Error ? error.message : String(error)}.`));
			return {
				tenant: record.tenantId,
				findings
			};
		}
		let inspection;
		try {
			inspection = await params.containers.inspect(record.runtime, record.containerName);
		} catch (error) {
			findings.push(finding("container-present", "fail", `Container inspection failed: ${error instanceof Error ? error.message : String(error)}.`));
			return {
				tenant: record.tenantId,
				findings
			};
		}
		if (inspection.kind !== "ok") {
			findings.push(finding("container-present", "fail", inspection.kind === "missing" ? `Container ${record.containerName} is missing.` : `Container inspection failed: ${inspection.error}.`));
			return {
				tenant: record.tenantId,
				findings
			};
		}
		findings.push(finding("container-present", "pass", `Container ${record.containerName} is present.`));
		const owned = inspection.labels["openclaw.fleet.tenant"] === record.tenantId && inspection.labels["openclaw.fleet.owner"] === cellOwnerId(record.dataDir);
		findings.push(owned ? finding("container-owned", "pass", `Container ownership labels match tenant ${record.tenantId}.`) : finding("container-owned", "fail", `Container ownership labels do not match tenant ${record.tenantId}.`));
		findings.push(inspection.running ? finding("container-running", "pass", `Container ${record.containerName} is running.`) : finding("container-running", "warn", `Container ${record.containerName} is stopped.`));
		if (owned) {
			if (inspection.running) {
				const health = await probeCellHealth({
					port: record.hostPort,
					fetchImpl: params.fetchImpl
				});
				findings.push(health.status === "ok" ? finding("gateway-health", "pass", `Gateway health check returned HTTP ${health.httpStatus}.`) : finding("gateway-health", "fail", `Gateway health check failed: ${health.status === "failed" ? health.error : health.reason}.`));
			} else findings.push(finding("gateway-health", "warn", "Gateway health check was skipped because the container is stopped."));
			const capsDropped = inspection.capDrop.includes("ALL") || inspection.effectiveCaps !== void 0 && inspection.effectiveCaps.length === 0;
			findings.push(capsDropped ? finding("cap-drop", "pass", "Container drops all Linux capabilities.") : finding("cap-drop", "fail", "Container does not drop all Linux capabilities."));
			findings.push(inspection.securityOpt.some((option) => option === "no-new-privileges" || option === "no-new-privileges:true") ? finding("security-opt", "pass", "Container enables no-new-privileges.") : finding("security-opt", "fail", "Container does not enable no-new-privileges."));
			findings.push(inspection.init === true ? finding("init", "pass", "Container init is enabled.") : finding("init", "fail", "Container init is not enabled."));
			for (const [check, value] of [
				["pids-limit", inspection.pidsLimit],
				["memory-limit", Number(inspection.memory)],
				["cpu-limit", Number(inspection.cpus)]
			]) findings.push(typeof value === "number" && Number.isFinite(value) && value > 0 ? finding(check, "pass", `${check} is positive.`) : finding(check, "fail", `${check} is missing or invalid.`));
			findings.push(inspection.restartPolicy === "unless-stopped" ? finding("restart-policy", "pass", "Restart policy is unless-stopped.") : finding("restart-policy", "fail", `Restart policy is ${inspection.restartPolicy ?? "unset"}; expected unless-stopped.`));
			const expectedPort = String(record.hostPort);
			const binding = inspection.portBindings[0];
			const validBinding = inspection.portBindings.length === 1 && binding?.containerPort === `18789/tcp` && binding.hostIp === "127.0.0.1" && binding.hostPort === expectedPort;
			findings.push(validBinding ? finding("port-binding", "pass", `Gateway port is bound to 127.0.0.1:${expectedPort}.`) : finding("port-binding", "fail", `Gateway port binding must be exactly ${FLEET_GATEWAY_PORT}/tcp to 127.0.0.1:${expectedPort}.`));
			const diskLimit = inspection.labels[FLEET_DISK_LIMIT_LABEL];
			if (diskLimit !== void 0) findings.push(diskLimitFinding(record.runtime, diskLimit, inspection.storageOpt.size));
			else if (inspection.storageOpt.size !== void 0) findings.push(finding("disk-limit", "pass", `Container writable-layer disk limit is ${inspection.storageOpt.size}.`));
			findings.push(inspection.environment.OPENCLAW_GATEWAY_TOKEN ? finding("gateway-token-env", "pass", "Gateway token environment is present.") : finding("gateway-token-env", "fail", "Gateway token environment is missing or empty."));
		}
		const networkName = cellNetworkName(record.tenantId);
		const network = await params.containers.inspectNetwork(record.runtime, networkName);
		if (network.kind !== "ok") findings.push(finding("network-present", "fail", network.kind === "missing" ? `Network ${networkName} is missing.` : `Network inspection failed: ${network.error}.`));
		else {
			findings.push(finding("network-present", "pass", `Network ${networkName} is present.`));
			const networkOwned = network.labels["openclaw.fleet.tenant"] === record.tenantId && network.labels["openclaw.fleet.owner"] === cellOwnerId(record.dataDir);
			findings.push(networkOwned ? finding("network-owned", "pass", `Network ownership labels match tenant ${record.tenantId}.`) : finding("network-owned", "fail", `Network ownership labels do not match tenant ${record.tenantId}.`));
			const foreignAttachments = network.attachedContainers.filter((attachment) => attachment.name !== record.containerName);
			findings.push(foreignAttachments.length > 0 ? finding("network-attachments", "fail", "Unexpected containers are attached to the cell network.") : record.runtime === "docker" && inspection.running && network.attachedContainers.length !== 1 ? finding("network-attachments", "fail", "The running cell container is not attached to its network.") : finding("network-attachments", "pass", "No unexpected containers are attached to the network."));
			findings.push(network.internal && record.runtime === "docker" ? finding("network-egress", "fail", "Docker internal networking breaks the published loopback Gateway port.") : network.internal ? finding("network-egress", "pass", "egress: internal") : finding("network-egress", "pass", "egress: bridge (unrestricted)"));
		}
		const userMatch = inspection.user?.match(/^(\d+):(\d+)$/u);
		const expectedUid = userMatch && Number(userMatch[1]) > 0 ? Number(userMatch[1]) : void 0;
		findings.push(...await directoryFindings({
			check: "data-dir",
			dir: record.dataDir,
			expectedUid
		}));
		findings.push(...await directoryFindings({
			check: "auth-dir",
			dir: cellAuthSecretDir(stateDir, record.tenantId),
			expectedUid
		}));
		return {
			tenant: record.tenantId,
			findings
		};
	}));
}
//#endregion
//#region src/fleet/service.runtime.ts
const OFFICIAL_IMAGE_UID = 1e3;
const OFFICIAL_IMAGE_GID = 1e3;
const CELL_VERIFY_TIMEOUT_MS = 6e4;
const CELL_VERIFY_POLL_MS = 1e3;
async function probeLoopbackPort(port) {
	return await new Promise((resolve) => {
		const server = createServer();
		server.once("error", (error) => {
			resolve(error.code !== "EADDRINUSE");
		});
		server.listen(port, "127.0.0.1", () => {
			server.close(() => resolve(true));
		});
	});
}
function createFleetService(options = {}) {
	const env = options.env ?? process.env;
	const containers = options.containers ?? createFleetContainerRuntime();
	const fetchImpl = options.fetch ?? fetch;
	const now = options.now ?? Date.now;
	const { generateToken = () => crypto.randomBytes(16).toString("hex") } = options;
	const generateAttemptId = options.generateAttemptId ?? (() => crypto.randomBytes(16).toString("hex"));
	const getuid = options.getuid ?? (() => process.getuid?.());
	const getgid = options.getgid ?? (() => process.getgid?.());
	const sleep = options.sleep ?? ((ms) => new Promise((resolve) => {
		setTimeout(resolve, ms);
	}));
	const selinuxEnabled = options.selinuxEnabled ?? detectHostSelinux;
	const updateImage = options.updateImage ?? updateFleetCellImage;
	const probePort = options.probePort ?? probeLoopbackPort;
	return {
		async create(createOptions) {
			const tenantId = validateTenantId(createOptions.tenant);
			const image = validateFleetImage(createOptions.image ?? "ghcr.io/openclaw/openclaw:latest");
			const runtime = createOptions.runtime ?? "docker";
			const network = createOptions.network ?? "bridge";
			const diskSize = createOptions.disk === void 0 ? void 0 : validateDiskSize(createOptions.disk);
			const { gatewayToken } = createOptions;
			if (gatewayToken !== void 0 && !gatewayToken.trim()) throw new Error("Gateway token must not be empty.");
			const token = gatewayToken ?? generateToken();
			const environment = buildCellEnvironment(token, parseEnvAssignments(createOptions.env ?? []));
			const attemptId = generateAttemptId();
			await containers.assertLocal(runtime);
			if (network === "internal" && runtime === "docker") throw new Error("Docker cannot publish loopback ports for containers on --internal networks, so the cell Gateway's 127.0.0.1 port would be unreachable and health-gated operations would always fail. Use --runtime podman for internal cells, or keep the default bridge network and enforce Docker egress policy with host firewall rules (DOCKER-USER chain).");
			return await withFleetCellOperation({
				env,
				tenantId,
				operationName: "create",
				operation: async (checkpoint) => {
					checkpoint();
					const stateDir = resolveStateDir(env);
					const usedPorts = new Set(listFleetCells(env).map((cell) => cell.hostPort));
					const reservation = {
						tenantId,
						createdAtMs: now(),
						image,
						runtime,
						containerName: cellContainerName(tenantId),
						dataDir: cellDataDir(stateDir, tenantId)
					};
					let record;
					if (createOptions.port !== void 0) {
						const candidatePort = allocateHostPort(usedPorts, createOptions.port);
						if (!await probePort(candidatePort)) throw new Error(`Host port ${candidatePort} is already in use on 127.0.0.1 by another process.`);
						record = reserveFleetCell(env, {
							...reservation,
							requestedPort: candidatePort
						});
					} else {
						const unavailablePorts = new Set(usedPorts);
						while (!record) {
							for (const cell of listFleetCells(env)) unavailablePorts.add(cell.hostPort);
							const candidate = allocateHostPort(unavailablePorts);
							if (!await probePort(candidate)) {
								unavailablePorts.add(candidate);
								continue;
							}
							try {
								record = reserveFleetCell(env, {
									...reservation,
									requestedPort: candidate
								});
							} catch (error) {
								if (getFleetCell(env, tenantId)) throw error;
								if (!listFleetCells(env).some((cell) => cell.hostPort === candidate)) throw error;
								unavailablePorts.add(candidate);
							}
						}
					}
					let result;
					let networkAttempted = false;
					let containerAttempted = false;
					try {
						const authSecretDir = cellAuthSecretDir(stateDir, tenantId);
						const hostIdentity = readHostIdentity(getuid, getgid);
						const containerUser = await resolveContainerUser({
							runtime,
							containers,
							hostIdentity
						});
						const imageOwner = hostIdentity?.uid === 0 && !containerUser ? {
							uid: OFFICIAL_IMAGE_UID,
							gid: OFFICIAL_IMAGE_GID
						} : void 0;
						const profile = {
							tenantId,
							containerName: record.containerName,
							networkName: cellNetworkName(tenantId),
							image,
							runtime,
							hostPort: record.hostPort,
							dataDir: record.dataDir,
							authSecretDir,
							ownerId: cellOwnerId(record.dataDir),
							attemptId,
							memory: createOptions.memory ?? "2g",
							cpus: createOptions.cpus ?? "2",
							...diskSize ? { diskSize } : {},
							pidsLimit: createOptions.pidsLimit ?? 512,
							environment,
							...containerUser ? { containerUser } : {},
							selinuxRelabel: await selinuxEnabled()
						};
						validateCellContainerProfile(profile);
						checkpoint();
						await prepareCellDirectories(record, authSecretDir, imageOwner);
						assertCurrentReservation(env, record);
						const started = createOptions.start !== false;
						networkAttempted = true;
						checkpoint();
						await containers.createNetwork(runtime, profile.networkName, {
							[FLEET_TENANT_LABEL]: tenantId,
							[FLEET_OWNER_LABEL]: profile.ownerId,
							[FLEET_ATTEMPT_LABEL]: attemptId
						}, { internal: network === "internal" });
						assertCurrentReservation(env, record);
						containerAttempted = true;
						checkpoint();
						await containers.run(profile, false);
						assertCurrentReservation(env, record);
						checkpoint();
						await prepareCellConfig(record, imageOwner);
						assertCurrentReservation(env, record);
						if (started) {
							checkpoint();
							await containers.start(runtime, record.containerName);
							assertCurrentReservation(env, record);
						}
						const url = `http://127.0.0.1:${record.hostPort}`;
						result = {
							tenant: tenantId,
							containerName: record.containerName,
							port: record.hostPort,
							image,
							runtime,
							started,
							token,
							tokenNote: "Shown once. Store this Gateway token securely.",
							url,
							nextStep: `Open ${url}, then configure per-tenant channel accounts inside the cell.`
						};
					} catch (error) {
						let releaseReservation = true;
						try {
							if (containerAttempted) releaseReservation = await cleanupFailedCreateContainer(record, containers, attemptId, checkpoint);
							if (releaseReservation && networkAttempted) releaseReservation = await cleanupFailedCreateNetwork(record, containers, attemptId, checkpoint);
						} catch {
							releaseReservation = false;
						}
						if (releaseReservation) try {
							checkpoint();
							deleteFleetCell(env, tenantId);
						} catch {}
						if (diskSize && error instanceof Error && /storage[ -]?opt|pquota|backingfs/iu.test(error.message)) throw new Error(`Fleet cannot enforce --disk on this container storage backend: ${error.message}. --disk requires Docker overlay2 on XFS with the pquota mount option (or btrfs/zfs storage drivers), or Podman overlay storage on XFS. Retry without --disk or move container storage to a supported filesystem.`, { cause: error });
						throw error;
					}
					if (result.started) try {
						await verifyReplacementHealthy({
							containers,
							record,
							attemptId,
							fetchImpl,
							now,
							sleep,
							checkpoint,
							timeoutMs: CELL_VERIFY_TIMEOUT_MS,
							pollMs: CELL_VERIFY_POLL_MS,
							context: "create"
						});
					} catch (error) {
						throw new Error(`Fleet cell ${tenantId} was created but did not become healthy within 60s; inspect it with \`openclaw fleet status ${tenantId}\` or \`openclaw fleet logs ${tenantId}\`, or remove it with \`openclaw fleet rm ${tenantId} --force\`.`, { cause: error });
					}
					return result;
				}
			});
		},
		async list() {
			const records = listFleetCells(env);
			const localityChecks = /* @__PURE__ */ new Map();
			const inspections = await Promise.all(records.map(async (record) => {
				try {
					let locality = localityChecks.get(record.runtime);
					if (!locality) {
						locality = containers.assertLocal(record.runtime);
						localityChecks.set(record.runtime, locality);
					}
					await locality;
					return await containers.inspect(record.runtime, record.containerName);
				} catch (error) {
					return {
						kind: "unavailable",
						state: "unknown",
						error: error instanceof Error ? error.message : String(error)
					};
				}
			}));
			return records.map((record, index) => ({
				tenant: record.tenantId,
				state: inspectionState(record, inspections[index] ?? {
					kind: "unavailable",
					state: "unknown",
					error: "inspect result missing"
				}),
				port: record.hostPort,
				image: record.image,
				created: new Date(record.createdAtMs).toISOString()
			}));
		},
		async status(tenant) {
			const record = requireCell(env, tenant);
			let inspection;
			try {
				await containers.assertLocal(record.runtime);
				inspection = await containers.inspect(record.runtime, record.containerName);
			} catch (error) {
				inspection = {
					kind: "unavailable",
					state: "unknown",
					error: error instanceof Error ? error.message : String(error)
				};
			}
			const url = `http://127.0.0.1:${record.hostPort}/healthz`;
			let container;
			let health;
			if (inspection.kind === "ok") {
				const managed = inspection.labels["openclaw.fleet.tenant"] === record.tenantId && inspection.labels["openclaw.fleet.owner"] === cellOwnerId(record.dataDir);
				container = {
					state: managed ? inspection.state : "unknown",
					running: inspection.running,
					managed,
					...managed ? { imageId: inspection.imageId } : {}
				};
				health = managed && inspection.running ? await probeCellHealth({
					port: record.hostPort,
					fetchImpl
				}) : {
					status: "skipped",
					url,
					reason: managed ? "container is not running" : "fleet ownership mismatch"
				};
			} else if (inspection.kind === "missing") {
				container = {
					state: "missing",
					running: false,
					managed: false
				};
				health = {
					status: "skipped",
					url,
					reason: "container is missing"
				};
			} else {
				container = {
					state: "unknown",
					running: false,
					managed: false,
					error: inspection.error
				};
				health = {
					status: "skipped",
					url,
					reason: "container runtime unavailable"
				};
			}
			return {
				tenant: record.tenantId,
				containerName: record.containerName,
				runtime: record.runtime,
				port: record.hostPort,
				image: record.image,
				created: new Date(record.createdAtMs).toISOString(),
				dataDir: record.dataDir,
				container,
				health
			};
		},
		async lifecycle(tenant, action) {
			const tenantId = validateTenantId(tenant);
			await containers.assertLocal(requireCell(env, tenantId).runtime);
			return await withFleetCellOperation({
				env,
				tenantId,
				operationName: action,
				operation: async (checkpoint) => {
					const record = requireCell(env, tenantId);
					await containers.assertLocal(record.runtime);
					assertManagedInspection(record, await containers.inspect(record.runtime, record.containerName));
					checkpoint();
					await containers[action](record.runtime, record.containerName);
					return {
						tenant: record.tenantId,
						action
					};
				}
			});
		},
		async logs(logOptions) {
			const record = requireCell(env, validateTenantId(logOptions.tenant));
			await containers.assertLocal(record.runtime);
			const inspection = assertManagedInspection(record, await containers.inspect(record.runtime, record.containerName));
			const gatewayCredential = inspection.environment.OPENCLAW_GATEWAY_TOKEN;
			await containers.logs(record.runtime, inspection.containerId, {
				follow: logOptions.follow,
				tail: logOptions.tail,
				since: logOptions.since,
				redactValues: gatewayCredential ? [gatewayCredential] : []
			});
		},
		async upgrade(tenant, requestedImage) {
			const tenantId = validateTenantId(tenant);
			const explicitImage = requestedImage === void 0 ? void 0 : validateFleetImage(requestedImage);
			await containers.assertLocal(requireCell(env, tenantId).runtime);
			return await withFleetCellOperation({
				env,
				tenantId,
				operationName: "upgrade",
				operation: async (checkpoint) => {
					const record = requireCell(env, tenantId);
					await containers.assertLocal(record.runtime);
					const inspection = assertManagedInspection(record, await containers.inspect(record.runtime, record.containerName));
					const image = explicitImage ?? validateFleetImage(record.image);
					const token = requireInspectedGatewayToken(inspection, "upgrade");
					const containerUser = await resolveContainerUser({
						runtime: record.runtime,
						containers,
						hostIdentity: readHostIdentity(getuid, getgid),
						user: inspection.user
					});
					const previousAttemptId = requireInspectedAttemptId(inspection, "upgrade");
					const nextAttemptId = generateAttemptId();
					const profileBase = buildProfileBaseFromInspection({
						record,
						stateDir: resolveStateDir(env),
						inspection,
						containerUser,
						selinuxRelabel: await selinuxEnabled(),
						token,
						context: "upgrade"
					});
					const oldProfile = {
						...profileBase,
						image: inspection.imageId,
						attemptId: previousAttemptId
					};
					const nextProfile = {
						...profileBase,
						image,
						attemptId: nextAttemptId
					};
					validateCellContainerProfile(oldProfile);
					validateCellContainerProfile(nextProfile);
					checkpoint();
					await containers.pull(record.runtime, image);
					checkpoint();
					assertManagedNetwork(record, await containers.inspectNetwork(record.runtime, cellNetworkName(record.tenantId)));
					try {
						if (inspection.running) {
							checkpoint();
							await containers.stop(record.runtime, record.containerName);
						}
						checkpoint();
						await containers.remove(record.runtime, record.containerName, false);
						checkpoint();
						await containers.run(nextProfile, true);
						await verifyReplacementHealthy({
							containers,
							record,
							attemptId: nextAttemptId,
							fetchImpl,
							now,
							sleep,
							checkpoint,
							timeoutMs: CELL_VERIFY_TIMEOUT_MS,
							pollMs: CELL_VERIFY_POLL_MS,
							context: "upgrade"
						});
						checkpoint();
						updateImage(env, record.tenantId, image);
					} catch (error) {
						try {
							await restorePreviousCell({
								record,
								containers,
								oldProfile,
								previousAttemptId,
								nextAttemptId,
								wasRunning: inspection.running,
								checkpoint
							});
						} catch {
							throw new Error(`Fleet upgrade failed for ${record.tenantId}; the previous container could not be restored.`, { cause: error });
						}
						throw new Error(`Fleet upgrade failed for ${record.tenantId}; the previous container was restored.`, { cause: error });
					}
					return {
						tenant: record.tenantId,
						action: "upgrade",
						image
					};
				}
			});
		},
		async backup(params) {
			const tenantId = validateTenantId(params.tenant);
			return await withFleetCellOperation({
				env,
				tenantId,
				operationName: "backup",
				operation: async (checkpoint) => {
					checkpoint();
					return await backupFleetCell({
						record: requireCell(env, tenantId),
						stateDir: resolveStateDir(env),
						containers,
						now,
						checkpoint,
						out: params.out,
						maxBytes: params.maxBytes
					});
				}
			});
		},
		async restore(params) {
			const tenantId = validateTenantId(params.tenant);
			return await withFleetCellOperation({
				env,
				tenantId,
				operationName: "restore",
				operation: async (checkpoint) => {
					return await restoreFleetCell({
						record: requireCell(env, tenantId),
						stateDir: resolveStateDir(env),
						containers,
						fetchImpl,
						now,
						sleep,
						checkpoint,
						generateToken,
						generateAttemptId,
						hostIdentity: readHostIdentity(getuid, getgid),
						selinuxRelabel: await selinuxEnabled(),
						from: params.from,
						force: params.force,
						maxBytes: params.maxBytes
					});
				}
			});
		},
		async doctor(tenant) {
			return await runFleetDoctor({
				env,
				containers,
				fetchImpl,
				tenant,
				getuid,
				getgid
			});
		},
		async remove(params) {
			if (params.purgeData && !params.force) throw new Error("--purge-data requires --force.");
			const tenantId = validateTenantId(params.tenant);
			await containers.assertLocal(requireCell(env, tenantId).runtime);
			return await withFleetCellOperation({
				env,
				tenantId,
				operationName: "rm",
				operation: async (checkpoint) => {
					const record = requireCell(env, tenantId);
					await containers.assertLocal(record.runtime);
					const stateDir = resolveStateDir(env);
					const authSecretDir = cellAuthSecretDir(stateDir, record.tenantId);
					const purgeTargets = [];
					if (params.purgeData) {
						const dataTarget = await resolvePurgeTarget(path.join(stateDir, "fleet", "cells"), record.dataDir, record.tenantId);
						if (dataTarget) purgeTargets.push(dataTarget);
						const authTarget = await resolvePurgeTarget(path.join(stateDir, "fleet", "auth-profile-secrets"), authSecretDir, record.tenantId);
						if (authTarget) purgeTargets.push(authTarget);
					}
					const inspection = await containers.inspect(record.runtime, record.containerName);
					if (inspection.kind === "unavailable") throw new Error(`Cannot inspect ${record.runtime} container for tenant ${record.tenantId}: ${inspection.error}`);
					const networkName = cellNetworkName(record.tenantId);
					const networkInspection = await containers.inspectNetwork(record.runtime, networkName);
					if (networkInspection.kind === "unavailable") throw new Error(`Cannot inspect ${record.runtime} network for tenant ${record.tenantId}: ${networkInspection.error}`);
					if (networkInspection.kind === "ok") assertManagedNetwork(record, networkInspection);
					if (inspection.kind === "ok") {
						assertManagedInspection(record, inspection);
						if (inspection.running && !params.force) throw new Error(`Fleet cell ${record.tenantId} is running; use --force to remove it.`);
						checkpoint();
						await containers.remove(record.runtime, record.containerName, params.force === true);
					}
					if (networkInspection.kind === "ok") {
						checkpoint();
						await containers.removeNetwork(record.runtime, networkName);
					}
					if (purgeTargets.length > 0) {
						checkpoint();
						await Promise.all(purgeTargets.map((target) => fs$1.rm(target, {
							recursive: true,
							force: true
						})));
					}
					checkpoint();
					deleteFleetCell(env, record.tenantId);
					return {
						tenant: record.tenantId,
						action: "rm",
						dataPurged: params.purgeData === true
					};
				}
			});
		}
	};
}
//#endregion
//#region src/cli/fleet-cli/commands.runtime.ts
const fleetService = createFleetService();
async function runFleetCreateCommand(options) {
	const result = await fleetService.create(options);
	if (options.json) {
		defaultRuntime.writeJson(result);
		return;
	}
	defaultRuntime.log(`Tenant: ${result.tenant}`);
	defaultRuntime.log(`Container: ${result.containerName}`);
	defaultRuntime.log(`Port: ${result.port}`);
	defaultRuntime.log(`Token: ${result.token}`);
	defaultRuntime.log(result.tokenNote);
	defaultRuntime.log(`Next: ${result.nextStep}`);
}
async function runFleetBackupCommand(options) {
	const result = await fleetService.backup(options);
	if (options.json) {
		defaultRuntime.writeJson(result);
		return;
	}
	defaultRuntime.log(`Archive: ${result.archivePath}`);
	if (result.skippedSymlinks > 0 || result.skippedSpecial > 0) defaultRuntime.log(`Skipped ${result.skippedSymlinks} symlink(s) and ${result.skippedSpecial} special file(s); Fleet archives regular files and directories only.`);
	defaultRuntime.log(result.note);
}
async function runFleetRestoreCommand(options) {
	const result = await fleetService.restore(options);
	if (options.json) {
		defaultRuntime.writeJson(result);
		return;
	}
	defaultRuntime.log(`Tenant: ${result.tenant}`);
	defaultRuntime.log(`Token: ${result.token}`);
	defaultRuntime.log(result.tokenNote);
}
async function runFleetDoctorCommand(options) {
	const reports = await fleetService.doctor(options.tenant);
	if (options.json) defaultRuntime.writeJson(reports);
	else {
		for (const report of reports) {
			defaultRuntime.log(`${report.tenant}:`);
			const nonPass = report.findings.filter((entry) => entry.status !== "pass");
			if (nonPass.length === 0) defaultRuntime.log("  ok");
			else for (const entry of nonPass) defaultRuntime.log(`  [${entry.status}] ${entry.check}: ${entry.detail}`);
		}
		const failures = reports.flatMap((report) => report.findings).filter((entry) => entry.status === "fail").length;
		const warnings = reports.flatMap((report) => report.findings).filter((entry) => entry.status === "warn").length;
		defaultRuntime.log(`Summary: ${reports.length} cell(s), ${failures} failure(s), ${warnings} warning(s).`);
	}
	if (reports.some((report) => report.findings.some((entry) => entry.status === "fail"))) process.exitCode = 1;
}
async function runFleetListCommand(options) {
	const cells = await fleetService.list();
	if (options.json) {
		defaultRuntime.writeJson({ cells });
		return;
	}
	if (cells.length === 0) {
		defaultRuntime.log("No fleet cells.");
		return;
	}
	defaultRuntime.log(renderTable({
		width: getTerminalTableWidth(),
		columns: [
			{
				key: "tenant",
				header: "Tenant",
				minWidth: 10,
				flex: true
			},
			{
				key: "state",
				header: "State",
				minWidth: 10
			},
			{
				key: "port",
				header: "Port",
				minWidth: 7
			},
			{
				key: "image",
				header: "Image",
				minWidth: 24,
				flex: true
			},
			{
				key: "created",
				header: "Created",
				minWidth: 24
			}
		],
		rows: cells.map((cell) => ({
			tenant: cell.tenant,
			state: cell.state,
			port: String(cell.port),
			image: cell.image,
			created: cell.created
		}))
	}).trimEnd());
}
function formatHealth(health) {
	if (health.status === "ok") return `ok (HTTP ${health.httpStatus})`;
	if (health.status === "failed") return `failed (${health.error})`;
	return `skipped (${health.reason})`;
}
async function runFleetStatusCommand(options) {
	const result = await fleetService.status(options.tenant);
	if (options.json) {
		defaultRuntime.writeJson(result);
		return;
	}
	defaultRuntime.log(`Tenant: ${result.tenant}`);
	defaultRuntime.log(`Container: ${result.containerName}`);
	defaultRuntime.log(`State: ${result.container.state}`);
	defaultRuntime.log(`Port: ${result.port}`);
	defaultRuntime.log(`Image: ${result.image}`);
	defaultRuntime.log(`Created: ${result.created}`);
	defaultRuntime.log(`Data: ${result.dataDir}`);
	defaultRuntime.log(`Health: ${formatHealth(result.health)}`);
}
async function runFleetLogsCommand(options) {
	await fleetService.logs(options);
}
async function runFleetLifecycleCommand(options) {
	const result = await fleetService.lifecycle(options.tenant, options.action);
	defaultRuntime.log(`${result.action} complete for fleet cell ${result.tenant}.`);
}
async function runFleetUpgradeCommand(options) {
	const result = await fleetService.upgrade(options.tenant, options.image);
	defaultRuntime.log(`Upgraded fleet cell ${result.tenant} to ${result.image}.`);
}
async function runFleetRemoveCommand(options) {
	const result = await fleetService.remove(options);
	defaultRuntime.log(result.dataPurged ? `Removed fleet cell ${result.tenant} and purged its data.` : `Removed fleet cell ${result.tenant}; data retained.`);
}
//#endregion
export { runFleetBackupCommand, runFleetCreateCommand, runFleetDoctorCommand, runFleetLifecycleCommand, runFleetListCommand, runFleetLogsCommand, runFleetRemoveCommand, runFleetRestoreCommand, runFleetStatusCommand, runFleetUpgradeCommand };