openclaw/dist/placement-dispatch-failure-CgBxWl6y.mjs

Multi-channel AI gateway with extensible messaging integrations

import { r as truncateUtf16Safe } from "./utf16-slice-D_ngcYKd.mjs";
import { t as createSubsystemLogger } from "./subsystem-4NzR43uz.mjs";
import { n as availableWorkerSlots } from "./worker-capacity-ChjgV5BU.mjs";
import { r as getRuntimeConfig } from "./io.runtime-B134MOMz.mjs";
import "./config-VnHbY_QV.mjs";
import { i as matchesWorkerPlacementTarget, r as isFailedWorkerPlacementEnvironmentGone } from "./placement-target-BZL1jHxv.mjs";
import { f as placementTurnOwner } from "./placement-record-BZ5ExJ2S.mjs";
import { f as isTerminalWorkerEnvironmentState } from "./store-validation-BCmnp3uW.mjs";
import { t as boundedWorkerError } from "./worker-error-Db8qkNbb.mjs";
import { d as resolveRequiredNodeCommandAuthority, l as resolveNodeCommandAllowlist, o as isNodeCommandAllowed } from "./node-command-policy-I3ly1VaT.mjs";
import { c as resolveNodeWorkerExecutionIssue } from "./node-runner-inventory-7Le5X7EN.mjs";
import { n as readNodeSessionWithheldCommands } from "./node-registry-Bgl9wido.mjs";
import { i as deviceUnavailableText, o as resolveDeviceWorkerAvailability } from "./device-provider-BDuNCezV.mjs";
import { t as STALE_WORKER_BUILD_REASON } from "./admission-B5dBx3Fa.mjs";
//#region src/gateway/worker-environments/device-placement-eligibility.ts
/** Revalidates the admitted connection and execution mode immediately before a placement write. */
function createDevicePlacementAuthority(getTransport) {
	return (node, requirement, executionMode) => {
		if (getTransport()?.isCurrent(node, requirement.consumesWorkerSlot, requirement.requiredNodeCommands, executionMode === "worker-turn") !== true) return false;
		const declaredCommands = [...node.commands];
		const allowlist = resolveNodeCommandAllowlist(getRuntimeConfig(), {
			commands: declaredCommands,
			approvedCommands: declaredCommands
		});
		return requirement.requiredNodeCommands.every((command) => isNodeCommandAllowed({
			command,
			declaredCommands,
			allowlist
		}).ok);
	};
}
/** Raised only before a dispatch begins workspace preparation. */
var DevicePlacementUnavailableError = class extends Error {
	constructor(deviceId, message) {
		super(message);
		this.deviceId = deviceId;
	}
};
async function resolveDevicePlacementEligibility(params) {
	const { deviceId, requirement } = params;
	if (!requirement) return {
		ok: false,
		error: `runtime ${params.runtimeId ?? "selection"} does not support paired-device placement; select a compatible runtime or cloud worker provider`
	};
	const availability = await resolveDeviceWorkerAvailability(params.environmentService, deviceId);
	if (!availability.available || !availability.node) return {
		ok: false,
		error: deviceUnavailableText(deviceId, availability)
	};
	const node = availability.node;
	const issue = params.executionMode === "worker-turn" ? resolveNodeWorkerExecutionIssue(node.workerHost) : void 0;
	if (issue) return {
		ok: false,
		error: deviceUnavailableText(deviceId, {
			available: false,
			issue
		}),
		issue
	};
	if (node.nodeId !== deviceId || params.currentNode && (params.currentNode.nodeId !== node.nodeId || params.currentNode.connId && params.currentNode.connId !== node.connId || params.currentNode.pairingGeneration && params.currentNode.pairingGeneration !== node.pairingGeneration)) return {
		ok: false,
		error: deviceUnavailableText(deviceId, {
			available: false,
			unavailableReason: "disconnected"
		})
	};
	const declaredCommands = [...node.commands];
	const allowlist = resolveNodeCommandAllowlist(params.config, {
		...params.currentNode?.platform ? { platform: params.currentNode.platform } : {},
		...params.currentNode?.deviceFamily ? { deviceFamily: params.currentNode.deviceFamily } : {},
		commands: declaredCommands,
		approvedCommands: declaredCommands
	});
	const requiredNodeCommand = resolveRequiredNodeCommandAuthority({
		nodeId: deviceId,
		requiredCommands: requirement.requiredNodeCommands,
		declaredCommands: params.currentNode?.declaredCommands ?? declaredCommands,
		effectiveCommands: params.currentNode?.commands ?? declaredCommands,
		withheldCommands: params.currentNode ? readNodeSessionWithheldCommands(params.currentNode) : [],
		allowlist
	});
	if (requiredNodeCommand && requiredNodeCommand.state !== "invocable") return {
		ok: false,
		error: requiredNodeCommand.message
	};
	const availableSlots = availableWorkerSlots(node.workerHost.capacity);
	if (requirement.consumesWorkerSlot && availableSlots <= 0) return {
		ok: false,
		error: deviceUnavailableText(deviceId, {
			available: false,
			unavailableReason: "at-capacity"
		})
	};
	return {
		ok: true,
		availableSlots,
		node
	};
}
//#endregion
//#region src/gateway/worker-environments/placement-dispatch-failure.ts
const RECOVERY_ERROR_LIMIT = 1024;
const log = createSubsystemLogger("gateway/worker-placement");
function canRetryDeviceDispatch(params) {
	const { error, attempted, current } = params;
	return error instanceof DevicePlacementUnavailableError && error.deviceId === params.deviceId && current?.state === "failed" && attempted?.state === "failed" && current.generation === attempted.generation && current.environmentId === attempted.environmentId && current.sessionId === params.sessionId && current.sessionKey === params.sessionKey && current.agentId === params.agentId && isFailedWorkerPlacementEnvironmentGone({
		environmentService: params.environments,
		placement: current
	});
}
function workerDisappearanceError(environment) {
	if (!environment) return /* @__PURE__ */ new Error("cloud worker disappeared: environment record missing");
	if (!isTerminalWorkerEnvironmentState(environment.state)) return;
	return /* @__PURE__ */ new Error(`cloud worker disappeared: ${environment.error ?? `environment state ${environment.state}`}`);
}
function isUnavailableEnvironment(environment) {
	return environment.state === "draining" || environment.state === "destroying" || isTerminalWorkerEnvironmentState(environment.state);
}
function createPlacementFailureActions(deps) {
	const { environments, placements } = deps;
	const updateFailure = (placement, error, teardownErrors, assertCurrent) => {
		let recoveryError = boundedWorkerError(error);
		if (teardownErrors) recoveryError = boundedWorkerError(truncateUtf16Safe([recoveryError, ...teardownErrors].join("; "), RECOVERY_ERROR_LIMIT));
		return placements.fail({
			sessionId: placement.sessionId,
			expectedGeneration: placement.generation,
			recoveryError
		}, assertCurrent);
	};
	const cleanupEnvironment = async (params) => {
		const teardownErrors = [];
		params.authorize?.();
		try {
			await environments.stopTunnel(params.environmentId, params.ownerEpoch ?? void 0);
		} catch (error) {
			teardownErrors.push(`tunnel stop: ${boundedWorkerError(error)}`);
		}
		params.authorize?.();
		try {
			await environments.destroy(params.environmentId);
		} catch (error) {
			teardownErrors.push(`environment destroy: ${boundedWorkerError(error)}`);
		}
		return teardownErrors;
	};
	const teardownEnvironment = async (params) => {
		const environmentId = params.environmentId;
		const teardownErrors = environmentId ? await cleanupEnvironment({
			environmentId,
			ownerEpoch: params.ownerEpoch
		}) : [];
		return updateFailure(params.placement, params.primaryError, teardownErrors);
	};
	const cancelProvisioning = (placement, expected, assertCurrent) => {
		if (expected?.state !== "provisioning" || !matchesWorkerPlacementTarget(placement, expected)) throw new Error("Provisioning cloud worker placement changed during reclaim");
		return updateFailure(expected, /* @__PURE__ */ new Error("Cloud worker provisioning canceled"), void 0, assertCurrent);
	};
	const retryFailedTeardown = async (placement, authorize) => {
		if (!placement.environmentId) return;
		const environment = environments.get(placement.environmentId);
		if (!environment || isTerminalWorkerEnvironmentState(environment.state)) return;
		const teardownErrors = await cleanupEnvironment({
			environmentId: placement.environmentId,
			ownerEpoch: placement.activeOwnerEpoch,
			...authorize ? { authorize } : {}
		});
		if (teardownErrors.length > 0 && placement.recoveryError !== "Worker result abandoned by forced operator teardown") {
			const originalError = placement.terminalReason ?? placement.recoveryError;
			const recoveryError = boundedWorkerError([originalError, ...teardownErrors.filter((detail) => !originalError.includes(detail))].filter(Boolean).join("; "), RECOVERY_ERROR_LIMIT);
			if (recoveryError !== placement.recoveryError) await placements.fail({
				sessionId: placement.sessionId,
				expectedGeneration: placement.generation,
				recoveryError
			});
		}
		return teardownErrors.length > 0 ? boundedWorkerError(teardownErrors.join("; ")) : void 0;
	};
	const startDrain = async (placement) => {
		const draining = await placements.startDrain({
			sessionId: placement.sessionId,
			environmentId: placement.environmentId,
			ownerEpoch: placement.activeOwnerEpoch,
			expectedGeneration: placement.generation
		});
		if (draining.state !== "draining") throw new Error("Worker placement drain did not produce a draining placement");
		return draining;
	};
	const startReconcile = async (placement) => {
		const reconciling = await placements.startReconcile({
			sessionId: placement.sessionId,
			environmentId: placement.environmentId,
			ownerEpoch: placement.activeOwnerEpoch,
			expectedGeneration: placement.generation
		});
		if (reconciling.state !== "reconciling") throw new Error("Worker placement reconcile did not produce a reconciling placement");
		return reconciling;
	};
	const failDraining = async (placement, error, options = {}) => {
		if (placement.turnClaim && !options.forceClaimFence) return;
		const current = await placements.getAsync(placement.sessionId);
		if (current?.state !== "draining") return;
		if (current.turnClaim) await placements.closeWorkerTurnToolState({
			sessionId: current.sessionId,
			claimId: current.turnClaim.claimId,
			runId: current.turnClaim.runId,
			placementGeneration: current.turnClaim.generation,
			owner: placementTurnOwner(current)
		});
		const reconciling = await startReconcile(current);
		const teardownErrors = await cleanupEnvironment({
			environmentId: current.environmentId,
			ownerEpoch: current.activeOwnerEpoch
		});
		await updateFailure(reconciling, error, teardownErrors);
	};
	const reclaimActive = async (placement, environment, claimedTurnError) => {
		const draining = await startDrain(placement);
		if (draining.turnClaim) {
			await failDraining(draining, claimedTurnError, { forceClaimFence: true });
			return;
		}
		const reconciling = await startReconcile(draining);
		if (environment?.state === "failed" && environment.error === "Worker build does not match the current Gateway build; redispatch the session so its worker can bootstrap the current build before retrying." && environment.leaseId === null && !(await placements.listPendingWorkspaceResultsAsync(placement.sessionId)).some((result) => result.sessionId === placement.sessionId)) {
			await placements.transition({
				sessionId: reconciling.sessionId,
				from: "reconciling",
				to: "reclaimed",
				expectedGeneration: reconciling.generation
			});
			log.info(`Reclaimed idle cloud worker sessionId=${boundedWorkerError(placement.sessionId, 128)} environmentId=${boundedWorkerError(placement.environmentId, 128)}: ${STALE_WORKER_BUILD_REASON}`);
			return;
		}
		if (!environment || isTerminalWorkerEnvironmentState(environment.state)) {
			await updateFailure(reconciling, claimedTurnError, []);
			return;
		}
		const teardownErrors = await cleanupEnvironment({
			environmentId: placement.environmentId,
			ownerEpoch: placement.activeOwnerEpoch
		});
		if (teardownErrors.length > 0) {
			await updateFailure(reconciling, /* @__PURE__ */ new Error(`Worker reclaim teardown failed: ${teardownErrors.join("; ")}`), []);
			return;
		}
		await placements.transition({
			sessionId: reconciling.sessionId,
			from: "reconciling",
			to: "reclaimed",
			expectedGeneration: reconciling.generation
		});
	};
	const failActive = async (placement, error, options = {}) => {
		const draining = await startDrain(placement);
		await failDraining(draining, error, options);
	};
	return {
		cancelProvisioning,
		failActive,
		failDraining,
		reclaimActive,
		retryFailedTeardown,
		teardownEnvironment
	};
}
//#endregion
export { DevicePlacementUnavailableError as a, workerDisappearanceError as i, createPlacementFailureActions as n, createDevicePlacementAuthority as o, isUnavailableEnvironment as r, resolveDevicePlacementEligibility as s, canRetryDeviceDispatch as t };