openclaw
Version:
Multi-channel AI gateway with extensible messaging integrations
2,929 lines • 125 kB
JavaScript
import { n as MAX_TIMER_TIMEOUT_MS } from "./number-coercion-CLj0HTDM.js";
import "./src-vebZIeLe.js";
import { l as toErrorObject } from "./error-coercion-D_-xJ90S.js";
import { c as isRecord } from "./record-coerce-DItp3I4t.js";
import { t as stableStringify } from "./stable-stringify-C8X7niaI.js";
import { r as truncateUtf16Safe } from "./utf16-slice-D_ngcYKd.js";
import { p as redactSensitiveText } from "./redact-BtvPPfTi.js";
import { k as withTimeout } from "./fs-safe-B6pvPGnf.js";
import { r as racePromiseWithAbortSignal } from "./abort-signal-D2k14JsD.js";
import { c as isSqliteLockError } from "./node-sqlite-BpQX3W0e.js";
import { a as getNodeSqliteKysely, i as executeSqliteQueryTakeFirstSync, r as executeSqliteQuerySync } from "./kysely-sync-COmh4HWh.js";
import { i as openOpenClawStateDatabase, s as runOpenClawStateWriteTransaction } from "./openclaw-state-db-BRTnL-D8.js";
import { t as createDeferredCore } from "./deferred-D0La5CRk.js";
import { i as validateCloudWorkerProfileSettings } from "./zod-schema.cloud-workers-By8wz7-f.js";
import { $ as WorkerComputerResultSchema, B as WorkerSessionsSpawnParamsSchema, N as WorkerPortalParamsSchema, R as WorkerSessionsSendParamsSchema } from "./worker-admission-C1l9t3yY.js";
import "./worker-protocol-primitives-Dh40okcI.js";
import { t as KeyedAsyncQueue } from "./keyed-async-queue-CTreGrmR.js";
import { t as runTasksWithConcurrency } from "./run-with-concurrency-Dtu208ef.js";
import { t as safeEqualSecret } from "./secret-equal-DRsL8lKD.js";
import { x as runWithGatewayIndependentRootWorkContinuation } from "./gateway-work-admission-R1IpuDim.js";
import { Dr as WorkerMachineOptionsSchema } from "./users-CPWrgxrZ.js";
import { _ as validateWorkerInferenceTerminalOutcome, a as WORKER_PROTOCOL_MAX_INFERENCE_PAYLOAD_BYTES, g as validateWorkerInferenceTerminalFrame, m as validateWorkerInferenceEventFrame } from "./worker-inference-CNQgy0yo.js";
import { t as boundedJsonUtf8Bytes } from "./json-utf8-bytes-fm9i4b7G.js";
import { r as normalizeCapabilityProviderId } from "./provider-registry-shared-DVSyKRD5.js";
import { n as recordRuntimeActionDecision } from "./runtime-action-decision-C4JNkXkP.js";
import { t as WorkerProviderError } from "./capability-provider.types-cizOzEy5.js";
import "./session-accessor-YsytfDtG.js";
import { p as onSessionIdentityMutation } from "./session-history-eviction-C4srftLJ.js";
import { l as ScreenSnapshotParamsSchema, o as ComputerActParamsSchema } from "./computer-use-contract-CisZRwLE.js";
import { i as WorkerSkillWorkshopParamsSchema } from "./worker-skill-workshop-4AWiTHUY.js";
import { n as workerBootstrapOperationTimeoutMs } from "./bootstrap-CysxhrWL.js";
import { b as sameWorkerSessionTurnClaim, t as FORCED_WORKER_ABANDONMENT_ERROR, x as serializeWorkerSessionTurnClaim } from "./placement-record-BOfTzlFO.js";
import "./device-provider-identity-v6nXqNq_.js";
import { t as NodeWorkerComputerCloseParamsSchema } from "./node-computer-protocol-BAxblUsb.js";
import { t as boundedWorkerError } from "./worker-error-BQ2GkQ1M.js";
import { i as joinWorkerTunnelStops, r as WorkerTunnelOwnerDisconnectedError } from "./tunnel-contract-CB5bcNwX.js";
import { n as registerWorkerInferenceSessionDrain } from "./inference-control-internal-CQh2JfZC.js";
import { t as prepareWorkerProjectSnapshot } from "./workspace-git-base-qAMhycVO.js";
import { i as deriveEnvironmentIntent, n as readWorkerProjectSnapshot, t as createWorkerProjectPreparation } from "./project-preparation-B7R7LNs2.js";
import { a as validateWorkerConnectionIdentity, c as createWorkerCredentialMaterial, l as hashWorkerCredential, n as StaleWorkerBuildError, o as verifyWorkerAdmissionHandshake, r as admitWorkerConnection, t as STALE_WORKER_BUILD_REASON } from "./admission-CKq3puIP.js";
import { i as normalizeWorkerSshEndpoint, r as normalizeWorkerDesktopEndpoint, t as WorkerSessionAlreadyAttachedError } from "./store-DRhMWW4v.js";
import { n as serializeWorkerSessionToolResult, r as workerSessionToolErrorResult } from "./worker-session-tool-result-CtqO208P.js";
import { isDeepStrictEqual } from "node:util";
import fs from "node:fs/promises";
import { createHash, randomUUID } from "node:crypto";
import { Value } from "typebox/value";
//#region src/gateway/worker-environments/credential-broker.ts
function createWorkerCredentialBroker(options) {
const { store } = options;
const tunnels = options.tunnelManager;
const now = options.now;
const inference = { cancelEnvironment: options.cancelInferenceEnvironment };
const inState = options.inState;
const move = options.move;
const serviceError = options.serviceError;
const withLock = options.withLock;
const pendingCredentials = /* @__PURE__ */ new Map();
const credentialExpiry = () => {
const ttlMs = options.workerCredentialTtlMs ?? 6e5;
if (!Number.isSafeInteger(ttlMs) || ttlMs < 1) throw serviceError("invalid_state", "Worker credential lifetime is invalid");
const expiresAtMs = now() + ttlMs;
if (!Number.isSafeInteger(expiresAtMs)) throw serviceError("invalid_state", "Worker credential expiry is out of range");
return expiresAtMs;
};
const credentialMaterial = (claim) => createWorkerCredentialMaterial(options.generateWorkerCredential, claim);
const grantFrom = (params) => {
const record = params.record;
if (!record) throw serviceError("invalid_state", "Worker credential persistence failed");
return {
credential: params.credential,
deliveryId: record.credentialHash,
environmentId: record.environmentId,
bundleHash: record.bundleHash,
sessionId: record.sessionId,
rpcSetVersion: record.rpcSetVersion,
ownerEpoch: record.ownerEpoch,
expiresAtMs: record.expiresAtMs,
...params.claim ? { turnClaim: params.claim } : {}
};
};
const mintCredentialLocked = (request, claim) => {
if (store.getCredential(request.environmentId)) inference.cancelEnvironment(request.environmentId);
const material = credentialMaterial(claim);
const credential = {
environmentId: request.environmentId,
expectedOwnerEpoch: request.ownerEpoch,
credentialHash: material.credentialHash,
sessionId: request.sessionId,
rpcSetVersion: 1,
expiresAtMs: credentialExpiry()
};
const record = store.renewCredential(credential);
return {
credentialHash: material.credentialHash,
grant: grantFrom({
credential: material.credential,
record,
claim
})
};
};
const stageCredential = (grant) => {
pendingCredentials.set(grant.environmentId, grant);
return grant;
};
const commitReady = (record, receipt, patch = {}) => {
const material = credentialMaterial();
const ready = move(record, "ready", {
...patch,
bootstrapReceipt: receipt,
credential: {
credentialHash: material.credentialHash,
sessionId: null,
rpcSetVersion: 1,
expiresAtMs: credentialExpiry()
}
});
stageCredential(grantFrom({
credential: material.credential,
record: store.getCredential(record.environmentId)
}));
return ready;
};
const ensurePendingCredential = (record, sessionId) => {
const credential = store.getCredential(record.environmentId);
const pending = pendingCredentials.get(record.environmentId);
const turnClaim = sessionId === null ? void 0 : options.placementStore?.readWorkerTurnClaim({
sessionId,
environmentId: record.environmentId,
ownerEpoch: record.ownerEpoch
});
const credentialHasDurableTurn = credential?.deliveredAtMs !== null && credential?.ownerEpoch === record.ownerEpoch && credential.sessionId === sessionId && sessionId !== null && turnClaim !== void 0 && options.placementStore?.validateWorkerTurn(turnClaim) === true;
const credentialIsCurrent = credential?.ownerEpoch === record.ownerEpoch && credential.sessionId === sessionId && (credential.expiresAtMs > now() || credentialHasDurableTurn);
const pendingIsCurrent = credentialIsCurrent && pending?.deliveryId === credential.credentialHash && pending.ownerEpoch === record.ownerEpoch && pending.sessionId === sessionId;
if (credentialIsCurrent && credential.deliveredAtMs !== null) {
pendingCredentials.delete(record.environmentId);
return;
}
if (pendingIsCurrent) return;
pendingCredentials.delete(record.environmentId);
const minted = mintCredentialLocked({
environmentId: record.environmentId,
ownerEpoch: record.ownerEpoch,
sessionId
});
stageCredential(minted.grant);
if (sessionId && credential?.ownerEpoch === record.ownerEpoch) options.liveEvents?.rotateCredential({
credentialHash: minted.credentialHash,
environmentId: record.environmentId,
previousCredentialHash: credential.credentialHash,
runEpoch: record.ownerEpoch,
sessionId
});
};
const attachSession = async (request) => {
let stopping = options.isStopping();
if (stopping) throw serviceError("invalid_state", "Worker environment service is stopping");
return withLock(request.environmentId, async () => {
stopping = options.isStopping();
if (stopping) throw serviceError("invalid_state", "Worker environment service is stopping");
const current = store.get(request.environmentId);
if (!current) throw serviceError("environment_not_found", `Unknown worker environment: ${request.environmentId}`);
if (current.state !== "ready" && current.state !== "idle") throw serviceError("invalid_state", `Cannot attach worker in state: ${current.state}`);
let currentBuild;
try {
currentBuild = await options.prepareInstallation("bundle");
} catch {
throw serviceError("invalid_state", "Current worker build identity is unavailable");
}
if (!current.bootstrapReceipt || !verifyWorkerAdmissionHandshake(current.bootstrapReceipt, currentBuild)) throw new StaleWorkerBuildError();
const material = credentialMaterial();
let attached;
try {
attached = store.transition({
environmentId: request.environmentId,
from: current.state,
to: "attached",
expectedOwnerEpoch: request.ownerEpoch,
patch: {
attachedSessionIds: [request.sessionId],
credential: {
credentialHash: material.credentialHash,
sessionId: request.sessionId,
rpcSetVersion: 1,
expiresAtMs: credentialExpiry()
}
}
});
} catch (error) {
if (error instanceof WorkerSessionAlreadyAttachedError) throw serviceError("invalid_state", error.message);
throw error;
}
if (options.liveEvents) {
let liveSessionBound;
try {
liveSessionBound = options.liveEvents.bindSession({
environmentId: attached.environmentId,
runEpoch: attached.ownerEpoch,
sessionId: request.sessionId
});
} catch {
liveSessionBound = false;
}
if (!liveSessionBound) {
move(attached, "idle");
await tunnels?.stop(request.environmentId, current.ownerEpoch).catch(() => void 0);
throw serviceError("invalid_state", "Attached session target is unavailable");
}
}
pendingCredentials.delete(request.environmentId);
await tunnels?.stop(request.environmentId, current.ownerEpoch);
return stageCredential(grantFrom({
credential: material.credential,
record: store.getCredential(request.environmentId)
}));
});
};
const readPendingCredential = (binding, claim) => {
if (options.isStopping()) return;
const grant = pendingCredentials.get(binding.environmentId);
if (!grant || grant.ownerEpoch !== binding.ownerEpoch || grant.sessionId !== binding.sessionId) return;
const environment = store.get(binding.environmentId);
const credential = store.getCredential(binding.environmentId);
const credentialHash = grant.deliveryId;
const checkedAtMs = now();
if (!environment || !inState(environment, "ready", "idle", "attached") || environment.destroyRequestedAtMs !== null || environment.ownerEpoch !== binding.ownerEpoch || !credential || credential.credentialHash !== credentialHash || credential.ownerEpoch !== binding.ownerEpoch || credential.sessionId !== binding.sessionId || credential.deliveredAtMs !== null || credential.expiresAtMs <= checkedAtMs || grant.turnClaim === void 0 !== (claim === void 0) || claim !== void 0 && hashWorkerCredential(grant.credential, claim) !== credentialHash) return;
return {
checkedAtMs,
credentialHash,
grant
};
};
const bindingForClaim = (claim) => {
if (claim.owner.kind !== "worker") throw serviceError("invalid_state", "Worker turn credential claim is not worker-owned");
return {
environmentId: claim.owner.environmentId,
ownerEpoch: claim.owner.ownerEpoch,
sessionId: claim.sessionId
};
};
const validateTurnClaim = (claim) => claim.owner.kind === "worker" && options.placementStore?.validateWorkerTurn(claim) === true;
const acquireTurnCredential = (claim) => {
const binding = bindingForClaim(claim);
return withLock(binding.environmentId, async () => {
const placementStore = options.placementStore;
if (!placementStore || !validateTurnClaim(claim)) throw serviceError("invalid_state", "Worker turn credential claim is not authoritative");
const pending = readPendingCredential(binding, claim)?.grant;
if (pending) return pending;
const environment = store.get(binding.environmentId);
if (!environment || environment.state !== "attached" || environment.ownerEpoch !== binding.ownerEpoch || environment.attachedSessionIds.length !== 1 || environment.attachedSessionIds[0] !== binding.sessionId) throw serviceError("invalid_state", "Worker session credential owner is not attached");
const previous = store.getCredential(binding.environmentId);
const ackedSeq = previous?.sessionId === binding.sessionId ? placementStore.readWorkerTurnLiveAckCursor(claim) : void 0;
const minted = mintCredentialLocked(binding, claim);
const grant = stageCredential(minted.grant);
if (previous && ackedSeq !== void 0) options.liveEvents?.rotateCredential({
ackedSeq,
credentialHash: minted.credentialHash,
environmentId: binding.environmentId,
newProcessTurn: true,
previousCredentialHash: previous.credentialHash,
runEpoch: binding.ownerEpoch,
sessionId: binding.sessionId
});
return grant;
});
};
const acknowledgeCredentialDelivery = (grant) => {
if (grant.turnClaim && !validateTurnClaim(grant.turnClaim)) return false;
const pending = readPendingCredential(grant, grant.turnClaim);
if (!pending || pending.grant.deliveryId !== grant.deliveryId) return false;
store.markCredentialDelivered({
environmentId: grant.environmentId,
ownerEpoch: grant.ownerEpoch,
sessionId: grant.sessionId,
credentialHash: pending.credentialHash,
deliveredAtMs: pending.checkedAtMs
});
pendingCredentials.delete(grant.environmentId);
return true;
};
return {
acknowledgeCredentialDelivery,
acquireTurnCredential,
attachSession,
clear: () => pendingCredentials.clear(),
clearEnvironment: (environmentId) => pendingCredentials.delete(environmentId),
commitReady,
ensurePendingCredential,
takeMintedCredential: (binding) => readPendingCredential(binding)?.grant
};
}
//#endregion
//#region src/gateway/worker-environments/environment-access.ts
const TUNNEL_START_TIMEOUT_MS = 18e4;
function createWorkerEnvironmentAccess(options) {
const { store } = options;
const tunnels = options.tunnelManager;
const nodeTunnels = options.nodeTunnelManager;
const nodeDesktop = options.nodeDesktopCarrier;
const now = options.now;
const inState = options.inState;
const providerFor = options.providerFor;
const identityResolverFor = options.identityResolverFor;
const serviceError = options.serviceError;
const withLock = options.withLock;
const project = (record) => {
const desktopAvailable = inState(record, "ready", "idle", "attached") && record.desktop !== null;
const nodeTunnelStatus = nodeTunnels?.status(record.environmentId);
return {
...record,
...(record.state === "failed" || record.state === "orphaned") && record.lastError ? { error: boundedWorkerError(record.lastError) } : {},
desktopAvailable,
desktopApps: desktopAvailable ? record.desktop?.apps?.map((app) => app.id).toSorted() ?? [] : [],
tunnelStatus: nodeTunnelStatus && nodeTunnelStatus !== "stopped" ? nodeTunnelStatus : tunnels?.status(record.environmentId) ?? nodeTunnelStatus ?? "stopped"
};
};
const startTunnel = async (request) => {
let stopping = options.isStopping();
if (stopping) throw serviceError("invalid_state", "Worker environment service is stopping");
if (!tunnels && !nodeTunnels) throw serviceError("invalid_state", "Worker tunnel runtime is unavailable");
let currentBundle;
try {
currentBundle = await options.prepareCurrentBundle();
} catch {
throw serviceError("invalid_state", "Current worker build identity is unavailable");
}
let startup;
let stopStartup;
await withLock(request.environmentId, async () => {
stopping = options.isStopping();
if (stopping) throw serviceError("invalid_state", "Worker environment service is stopping");
const record = store.get(request.environmentId);
if (!record) throw serviceError("environment_not_found", `Unknown worker environment: ${request.environmentId}`);
if (!inState(record, "ready", "idle", "attached") || record.destroyRequestedAtMs !== null || !record.leaseId) throw serviceError("invalid_state", `Cannot start tunnel in state: ${record.state}`);
if (!record.bootstrapReceipt) throw serviceError("invalid_state", `Cannot start tunnel in state: ${record.state}`);
if (record.sharedHost === null) throw serviceError("provider_failure", "Worker lease isolation is not reconciled; retry after provider inspection");
const credential = store.getCredential(request.environmentId);
if (record.ownerEpoch !== request.ownerEpoch || !credential || credential.ownerEpoch !== request.ownerEpoch) throw serviceError("invalid_state", "Worker tunnel owner credential is not current");
if (!verifyWorkerAdmissionHandshake(record.bootstrapReceipt, currentBundle)) throw new StaleWorkerBuildError();
const nodeDeviceId = record.nodeDeviceId;
if (typeof nodeDeviceId === "string" && !record.sshEndpoint && record.bootstrapReceipt.installKind === "bundle") {
const sessionId = record.attachedSessionIds[0];
if (!nodeTunnels || !sessionId || record.attachedSessionIds.length !== 1 || credential.sessionId !== sessionId) throw serviceError("invalid_state", "Node worker tunnel runtime is unavailable");
startup = nodeTunnels.start({
executionMode: record.profileSnapshot.executionMode === "remote-exec" ? "remote-exec" : "worker-turn",
environmentId: record.environmentId,
ownerEpoch: record.ownerEpoch,
deviceId: nodeDeviceId,
sessionId,
expectedBuild: {
bundleHash: currentBundle.bundleHash,
openclawVersion: currentBundle.openclawVersion,
protocolFeatures: [...currentBundle.protocolFeatures]
}
});
stopStartup = async () => await nodeTunnels.stop(record.environmentId, record.ownerEpoch);
return;
}
if (!record.sshEndpoint) throw serviceError("invalid_state", "Worker environment has no supported tunnel transport");
if (!tunnels) throw serviceError("invalid_state", "Worker SSH tunnel runtime is unavailable");
const provider = providerFor(record.providerId);
startup = tunnels.start({
...request,
bundleHash: currentBundle.bundleHash,
ssh: record.sshEndpoint,
sharedHost: record.sharedHost,
resolveIdentity: identityResolverFor(record, provider, record.leaseId)
});
stopStartup = async () => await tunnels.stop(record.environmentId, record.ownerEpoch);
});
if (!startup) throw serviceError("invalid_state", "Worker tunnel failed to start");
const timeoutError = serviceError("provider_failure", "Worker tunnel did not connect within 3 minutes; check that the worker is online and reachable, then retry");
try {
return await withTimeout(startup, TUNNEL_START_TIMEOUT_MS, { createError: () => timeoutError });
} catch (error) {
if (error !== timeoutError) throw error;
stopStartup?.().catch(() => void 0);
throw timeoutError;
}
};
const observeDesktop = async (request) => {
let stopping = options.isStopping();
if (options.getConfig().cloudWorkers?.desktop !== true) throw serviceError("invalid_state", "worker desktop observe is disabled; enable the Desktop lab in Control UI Settings -> Labs (config: cloudWorkers.desktop)");
if (stopping) throw serviceError("invalid_state", "Worker environment service is stopping");
let startup;
let nodeStartup;
let ownerEpoch;
await withLock(request.environmentId, async () => {
stopping = options.isStopping();
if (stopping) throw serviceError("invalid_state", "Worker environment service is stopping");
const record = store.get(request.environmentId);
if (!record) throw serviceError("environment_not_found", `Unknown worker environment: ${request.environmentId}`);
if (!inState(record, "ready", "idle", "attached") || record.destroyRequestedAtMs !== null || !record.leaseId || !record.desktop) throw serviceError("invalid_state", "environment has no desktop; desktop is a warm-time capability of the profile");
ownerEpoch = record.ownerEpoch;
if (record.sshEndpoint) {
if (!tunnels) throw serviceError("invalid_state", "Worker SSH desktop runtime is unavailable");
const provider = providerFor(record.providerId);
startup = tunnels.desktop.acquire({
environmentId: record.environmentId,
ownerEpoch: record.ownerEpoch,
ssh: record.sshEndpoint,
desktop: record.desktop,
resolveIdentity: identityResolverFor(record, provider, record.leaseId)
});
return;
}
if (record.nodeDeviceId) {
if (!nodeDesktop) throw serviceError("invalid_state", "Worker node desktop runtime is unavailable");
nodeStartup = nodeDesktop.observe({
record,
control: request.control
});
return;
}
throw serviceError("invalid_state", "Worker environment has no desktop transport");
});
if (nodeStartup) return await nodeStartup;
if (!startup || ownerEpoch === void 0) throw serviceError("invalid_state", "Worker desktop tunnel failed to start");
const acquired = await startup;
const { DESKTOP_OBSERVE_PATH, mintDesktopObserverToken } = await import("./observe-bridge-BoVfcTPP.js");
const minted = mintDesktopObserverToken({
sourceKey: request.environmentId,
ownerEpoch,
control: request.control,
attachment: acquired.attachment,
nowMs: now()
});
return {
transport: "rfb",
wsPath: `${DESKTOP_OBSERVE_PATH}?token=${minted.token}`,
expiresAtMs: minted.expiresAtMs,
control: request.control,
...acquired.vncPassword ? { vncPassword: acquired.vncPassword } : {}
};
};
const launchDesktopApp = async (request) => {
let stopping = options.isStopping();
if (options.getConfig().cloudWorkers?.desktop !== true) throw serviceError("invalid_state", "worker desktop launch is disabled; enable the Desktop lab in Control UI Settings -> Labs (config: cloudWorkers.desktop)");
if (stopping) throw serviceError("invalid_state", "Worker environment service is stopping");
const requireLaunchable = () => {
stopping = options.isStopping();
if (stopping) throw serviceError("invalid_state", "Worker environment service is stopping");
const record = store.get(request.environmentId);
if (!record) throw serviceError("environment_not_found", `Unknown worker environment: ${request.environmentId}`);
if (!inState(record, "ready", "idle", "attached") || record.destroyRequestedAtMs !== null || !record.leaseId || !record.desktop) throw serviceError("invalid_state", "environment has no desktop; desktop is a warm-time capability of the profile");
const app = record.desktop.apps?.find((candidate) => candidate.id === request.app);
if (!app) throw serviceError("desktop_app_not_found", `environment does not advertise desktop app: ${request.app}`);
return {
app,
record
};
};
let startup;
let launchEpoch;
await withLock(request.environmentId, async () => {
const { app, record } = requireLaunchable();
launchEpoch = record.ownerEpoch;
if (record.sshEndpoint) {
if (!tunnels) throw serviceError("invalid_state", "Worker SSH desktop runtime is unavailable");
const provider = providerFor(record.providerId);
startup = tunnels.desktop.launchApp({
environmentId: record.environmentId,
ownerEpoch: record.ownerEpoch,
ssh: record.sshEndpoint,
app,
resolveIdentity: identityResolverFor(record, provider, record.leaseId)
});
return;
}
if (record.nodeDeviceId) {
if (!nodeDesktop) throw serviceError("invalid_state", "Worker node desktop runtime is unavailable");
startup = nodeDesktop.launchApp({
record,
app
});
return;
}
throw serviceError("invalid_state", "Worker environment has no desktop transport");
});
if (!startup || launchEpoch === void 0) throw serviceError("launcher_failure", "Worker desktop app launcher failed to start");
try {
await startup;
} catch (error) {
if (error && typeof error === "object" && "code" in error && error.code === "unsupported_platform") throw serviceError("unsupported_platform", "desktop app launch is not supported on Windows gateway hosts");
await withLock(request.environmentId, async () => {
const { record } = requireLaunchable();
if (record.ownerEpoch !== launchEpoch) throw serviceError("invalid_state", "Worker desktop app launch owner changed");
});
throw serviceError("launcher_failure", `worker desktop ${request.app} launcher failed; verify the app is installed and retry`);
}
await withLock(request.environmentId, async () => {
const { record } = requireLaunchable();
if (record.ownerEpoch !== launchEpoch) throw serviceError("invalid_state", "Worker desktop app launch owner changed");
});
return {
app: request.app,
status: "ready"
};
};
const stopTunnelOwners = async (stops) => {
const failure = (await Promise.allSettled(stops.filter((stop) => stop !== void 0))).find((result) => result.status === "rejected");
if (failure) throw failure.reason;
};
const stopTunnel = async (environmentId, ownerEpoch) => {
await withLock(environmentId, async () => stopTunnelOwners([
tunnels?.stop(environmentId, ownerEpoch),
nodeTunnels?.stop(environmentId, ownerEpoch),
nodeDesktop?.stop(environmentId, ownerEpoch)
]));
};
return {
get: (environmentId) => {
const record = store.get(environmentId);
return record ? project(record) : void 0;
},
launchDesktopApp,
list: () => store.list().map(project),
observeDesktop,
project,
startTunnel,
stopAllTunnels: () => stopTunnelOwners([
tunnels?.stopAll(),
nodeTunnels?.stopAll(),
nodeDesktop?.stopAll()
]),
stopTunnel
};
}
//#endregion
//#region src/gateway/worker-environments/inference-store.ts
const REQUEST_HASH_PATTERN = /^[a-f0-9]{64}$/u;
const DEFAULT_RETENTION = {
maxAgeMs: 864e5,
maxRows: 256,
maxBytes: 67108864
};
function required(value, field) {
if (typeof value !== "string" || !value.trim()) throw new Error(`Worker inference turn ${field} must be a non-empty string`);
return value.trim();
}
function nonNegativeInteger(value, field) {
if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) throw new Error(`Worker inference turn ${field} must be a non-negative integer`);
return value;
}
function normalizeRequestHash(value) {
if (typeof value !== "string" || !REQUEST_HASH_PATTERN.test(value)) throw new Error("Worker inference turn request hash must be lowercase SHA-256 hex");
return value;
}
function normalizeInput(input, nowMs) {
return {
environmentId: required(input.environmentId, "environment id"),
sessionId: required(input.sessionId, "session id"),
runEpoch: nonNegativeInteger(input.runEpoch, "run epoch"),
runId: required(input.runId, "run id"),
turnId: required(input.turnId, "turn id"),
requestHash: normalizeRequestHash(input.requestHash),
nowMs: nonNegativeInteger(nowMs, "timestamp")
};
}
function parseTerminalJson(value) {
let parsed;
try {
parsed = JSON.parse(value);
} catch (error) {
throw new Error("Worker inference cached terminal outcome is invalid", { cause: error });
}
if (!validateWorkerInferenceTerminalOutcome(parsed)) throw new Error("Worker inference cached terminal outcome is invalid");
return parsed;
}
function serializeTerminalOutcome(outcome) {
if (!validateWorkerInferenceTerminalOutcome(outcome)) throw new Error("Worker inference terminal outcome is invalid");
const serialized = JSON.stringify(outcome);
if (!serialized) throw new Error("Worker inference terminal outcome is not serializable");
return serialized;
}
const query = (db) => getNodeSqliteKysely(db);
function findTurn(db, input) {
return executeSqliteQueryTakeFirstSync(db, query(db).selectFrom("worker_inference_turns").selectAll().where("session_id", "=", input.sessionId).where("run_epoch", "=", input.runEpoch).where("run_id", "=", input.runId).where("turn_id", "=", input.turnId));
}
function findPendingTurn(db, input) {
return executeSqliteQueryTakeFirstSync(db, query(db).selectFrom("worker_inference_turns").selectAll().where("session_id", "=", input.sessionId).where("run_epoch", "=", input.runEpoch).where("run_id", "=", input.runId).where("state", "=", "pending"));
}
function classifyExistingTurn(row, input) {
if (!row) return;
if (row.environment_id !== input.environmentId || row.request_hash !== input.requestHash) return {
kind: "rejected",
reason: "conflict"
};
if (row.state === "pending" && row.terminal_json === null) return { kind: "recover" };
if (row.state === "terminal" && row.terminal_json !== null) return {
kind: "replay",
outcome: parseTerminalJson(row.terminal_json)
};
throw new Error("Worker inference turn row has invalid terminal state");
}
function insertPendingTurn(db, input) {
const turn = {
session_id: input.sessionId,
run_epoch: input.runEpoch,
run_id: input.runId,
turn_id: input.turnId,
environment_id: input.environmentId,
request_hash: input.requestHash,
state: "pending",
terminal_json: null,
created_at_ms: input.nowMs,
updated_at_ms: input.nowMs
};
executeSqliteQuerySync(db, query(db).insertInto("worker_inference_turns").values(turn));
}
function deleteTurn(db, row) {
executeSqliteQuerySync(db, query(db).deleteFrom("worker_inference_turns").where("session_id", "=", row.session_id).where("run_epoch", "=", row.run_epoch).where("run_id", "=", row.run_id).where("turn_id", "=", row.turn_id).where("state", "=", "terminal"));
}
function pruneTerminalTurns(params) {
const rows = executeSqliteQuerySync(params.db, query(params.db).selectFrom("worker_inference_turns").selectAll().where("state", "=", "terminal").orderBy("updated_at_ms", "desc").orderBy("session_id", "asc").orderBy("run_epoch", "desc").orderBy("run_id", "asc").orderBy("turn_id", "asc")).rows;
const isPreserved = (row) => params.preserve !== void 0 && row.session_id === params.preserve.sessionId && row.run_epoch === params.preserve.runEpoch && row.run_id === params.preserve.runId && row.turn_id === params.preserve.turnId;
rows.sort((left, right) => Number(isPreserved(right)) - Number(isPreserved(left)));
const cutoffMs = Math.max(0, params.nowMs - params.policy.maxAgeMs);
let retainedRows = 0;
let retainedBytes = 0;
for (const row of rows) {
const terminalBytes = Buffer.byteLength(row.terminal_json ?? "", "utf8");
const preserve = isPreserved(row);
const expired = row.updated_at_ms < cutoffMs;
const exceedsRows = retainedRows >= params.policy.maxRows;
const exceedsBytes = retainedRows > 0 && retainedBytes + terminalBytes > params.policy.maxBytes;
if (!preserve && (expired || exceedsRows || exceedsBytes)) {
deleteTurn(params.db, row);
continue;
}
retainedRows += 1;
retainedBytes += terminalBytes;
}
}
function createWorkerInferenceStore(options = {}) {
const path = (options.database ?? openOpenClawStateDatabase()).path;
const now = options.now ?? Date.now;
const retention = {
...DEFAULT_RETENTION,
...options.retention
};
const write = (operation) => runOpenClawStateWriteTransaction(({ db }) => operation(db), { path });
const begin = (rawInput) => {
const input = normalizeInput(rawInput, now());
return write((db) => {
pruneTerminalTurns({
db,
nowMs: input.nowMs,
policy: retention
});
const existing = classifyExistingTurn(findTurn(db, input), input);
if (existing) return existing;
if (findPendingTurn(db, input)) return {
kind: "rejected",
reason: "conflict"
};
insertPendingTurn(db, input);
return { kind: "claimed" };
});
};
const complete = (rawInput) => {
const input = normalizeInput(rawInput, now());
const terminalJson = serializeTerminalOutcome(rawInput.outcome);
return write((db) => {
const existing = classifyExistingTurn(findTurn(db, input), input);
if (!existing) throw new Error("Worker inference turn must begin before terminal completion");
if (existing.kind === "rejected") throw new Error(`Worker inference terminal completion rejected: ${existing.reason}`);
if (existing.kind === "replay") return existing.outcome;
if (executeSqliteQuerySync(db, query(db).updateTable("worker_inference_turns").set({
state: "terminal",
terminal_json: terminalJson,
updated_at_ms: input.nowMs
}).where("session_id", "=", input.sessionId).where("run_epoch", "=", input.runEpoch).where("run_id", "=", input.runId).where("turn_id", "=", input.turnId).where("environment_id", "=", input.environmentId).where("request_hash", "=", input.requestHash).where("state", "=", "pending")).numAffectedRows !== 1n) throw new Error("Worker inference turn changed during terminal completion");
pruneTerminalTurns({
db,
nowMs: input.nowMs,
policy: retention,
preserve: input
});
return rawInput.outcome;
});
};
const cancelPending = (params) => {
const nowMs = nonNegativeInteger(now(), "timestamp");
const terminalJson = serializeTerminalOutcome(params.outcome);
const identity = {
environmentId: required(params.environmentId, "environment id"),
sessionId: required(params.sessionId, "session id"),
runEpoch: nonNegativeInteger(params.runEpoch, "run epoch"),
runId: required(params.runId, "run id"),
turnId: required(params.turnId, "turn id")
};
write((db) => {
executeSqliteQuerySync(db, query(db).updateTable("worker_inference_turns").set({
state: "terminal",
terminal_json: terminalJson,
updated_at_ms: nowMs
}).where("session_id", "=", identity.sessionId).where("run_epoch", "=", identity.runEpoch).where("run_id", "=", identity.runId).where("turn_id", "=", identity.turnId).where("environment_id", "=", identity.environmentId).where("state", "=", "pending"));
pruneTerminalTurns({
db,
nowMs,
policy: retention,
preserve: identity
});
});
};
const recoverPending = (outcome) => {
const nowMs = nonNegativeInteger(now(), "timestamp");
const terminalJson = serializeTerminalOutcome(outcome);
write((db) => {
executeSqliteQuerySync(db, query(db).updateTable("worker_inference_turns").set({
state: "terminal",
terminal_json: terminalJson,
updated_at_ms: nowMs
}).where("state", "=", "pending"));
pruneTerminalTurns({
db,
nowMs,
policy: retention
});
});
};
return {
begin,
cancelPending,
complete,
recoverPending
};
}
//#endregion
//#region src/gateway/worker-environments/inference.ts
const DEFAULT_REQUEST_MAX_BYTES = WORKER_PROTOCOL_MAX_INFERENCE_PAYLOAD_BYTES;
const MAX_PROVIDER_OPERATIONS_PER_SESSION = 2;
function safeRevalidate(revalidate) {
try {
return revalidate?.() ?? null;
} catch {
return "provider-error";
}
}
function trySend(sink, frame) {
try {
sink.send(frame);
return true;
} catch {
return false;
}
}
function terminalError(reason, outcome) {
const usage = outcome?.type === "done" ? outcome.message.usage : outcome?.type === "error" ? outcome.usage : void 0;
return {
type: "error",
reason,
message: (() => {
switch (reason) {
case "model-not-approved": return "Model is not approved";
case "invalid-context": return "Inference context is invalid";
case "epoch-mismatch": return "Inference ownership changed";
case "session-not-attached": return "Session is not attached";
case "provider-error": return "Provider request failed";
case "cancelled": return "Inference cancelled";
}
return "Provider request failed";
})(),
...usage ? { usage } : {}
};
}
function validFrameBytes(frame, validate) {
const measured = boundedJsonUtf8Bytes(frame, WORKER_PROTOCOL_MAX_INFERENCE_PAYLOAD_BYTES);
if (measured.complete && measured.bytes <= WORKER_PROTOCOL_MAX_INFERENCE_PAYLOAD_BYTES && validate(frame)) return measured.bytes;
return null;
}
function terminalFrame(entry, outcome, seq = entry.seq + 1) {
return {
type: "event",
event: "worker.inference.terminal",
payload: {
runEpoch: entry.request.runEpoch,
sessionId: entry.request.sessionId,
runId: entry.request.runId,
turnId: entry.request.turnId,
seq,
outcome
}
};
}
function normalizeTerminalOutcome(entry, outcome) {
if (!validateWorkerInferenceTerminalOutcome(outcome) || validFrameBytes(terminalFrame(entry, outcome), validateWorkerInferenceTerminalFrame) === null) return terminalError("provider-error");
return outcome;
}
function matchesIdentity(identity, request) {
const claim = identity.turnClaim;
if (!claim || identity.sessionId !== request.sessionId || identity.runId !== request.runId || claim.sessionId !== request.sessionId || claim.runId !== request.runId) return "session-not-attached";
if (identity.ownerEpoch !== request.runEpoch) return "epoch-mismatch";
return null;
}
function createWorkerInferenceManager(options) {
const store = options.store ?? createWorkerInferenceStore();
const requestMaxBytes = options.requestMaxBytes ?? DEFAULT_REQUEST_MAX_BYTES;
const streamMaxBytes = options.streamMaxBytes ?? WORKER_PROTOCOL_MAX_INFERENCE_PAYLOAD_BYTES;
const active = /* @__PURE__ */ new Map();
const operations = /* @__PURE__ */ new Map();
const drainingSessionIds = /* @__PURE__ */ new Set();
let stopping = false;
store.recoverPending(terminalError("provider-error"));
const processFence = (entry) => {
if (entry.abortReason) return entry.abortReason;
const bindingError = matchesIdentity(entry.identity, entry.request);
if (bindingError) return bindingError;
if (active.get(entry.claimKey) !== entry) return "cancelled";
return null;
};
const durableFence = (entry) => {
const currentError = processFence(entry);
if (currentError) return currentError;
const revalidationError = safeRevalidate(entry.revalidate);
if (revalidationError) {
entry.abortReason = revalidationError;
entry.controller.abort();
return revalidationError;
}
return null;
};
const abortEntry = (entry, reason) => {
if (!entry.abortReason) entry.abortReason = reason;
if (!entry.controller.signal.aborted) entry.controller.abort();
};
const sendTerminal = (entry, outcome) => {
entry.seq += 1;
trySend(entry.sink, terminalFrame(entry, outcome, entry.seq));
};
const settleAbort = (entry, reason) => {
if (entry.settled) return true;
abortEntry(entry, reason);
let outcome;
try {
outcome = store.complete({
...entry.storeInput,
outcome: terminalError(entry.abortReason ?? reason)
});
} catch {
return false;
}
entry.settled = true;
if (active.get(entry.claimKey) === entry) {
active.delete(entry.claimKey);
sendTerminal(entry, outcome);
}
return true;
};
const finish = (entry, rawOutcome) => {
if (entry.settled) return;
const fence = durableFence(entry);
const outcome = normalizeTerminalOutcome(entry, fence ? terminalError(fence, rawOutcome) : rawOutcome);
let storedOutcome;
try {
storedOutcome = store.complete({
...entry.storeInput,
outcome
});
} catch {
entry.settled = true;
if (active.get(entry.claimKey) === entry) active.delete(entry.claimKey);
return;
}
entry.settled = true;
if (active.get(entry.claimKey) === entry) {
active.delete(entry.claimKey);
sendTerminal(entry, storedOutcome);
}
};
const executeEntry = async (entry) => {
const initialFence = durableFence(entry);
if (initialFence) {
finish(entry, terminalError(initialFence));
return;
}
let outcome;
try {
const config = options.getConfig?.();
outcome = await options.execute({
identity: entry.identity,
request: entry.request,
signal: entry.controller.signal,
emit: (event) => {
const fence = durableFence(entry);
if (fence) {
abortEntry(entry, fence);
return;
}
const nextSeq = entry.seq + 1;
const frame = {
type: "event",
event: "worker.inference.event",
payload: {
runEpoch: entry.request.runEpoch,
sessionId: entry.request.sessionId,
runId: entry.request.runId,
turnId: entry.request.turnId,
seq: nextSeq,
event
}
};
const frameBytes = validFrameBytes(frame, validateWorkerInferenceEventFrame);
if (frameBytes === null || entry.streamedBytes + frameBytes > streamMaxBytes) {
settleAbort(entry, "provider-error");
return;
}
if (!trySend(entry.sink, frame)) {
settleAbort(entry, "provider-error");
return;
}
entry.streamedBytes += frameBytes;
entry.seq = nextSeq;
},
isCurrent: () => durableFence(entry) === null,
...config ? { config } : {}
});
} catch {
outcome = terminalError(entry.abortReason ?? "provider-error");
}
finish(entry, outcome);
};
const launchEntry = (entry) => {
if (entry.launched || entry.settled) return;
entry.launched = true;
const operation = runWithGatewayIndependentRootWorkContinuation(() => executeEntry(entry), "worker:dispatch").catch(() => {
finish(entry, terminalError(entry.abortReason ?? "provider-error"));
});
operations.set(operation, entry.request.sessionId);
operation.then(() => operations.delete(operation), () => operations.delete(operation));
};
const start = (params) => {
if (stopping || drainingSessionIds.has(params.request.sessionId)) return {
ok: false,
reason: "cancelled"
};
const identityError = matchesIdentity(params.identity, params.request);
if (identityError) return {
ok: false,
reason: identityError
};
const revalidationError = safeRevalidate(params.revalidate);
if (revalidationError) return {
ok: false,
reason: revalidationError
};
const measured = boundedJsonUtf8Bytes(params.request, requestMaxBytes);
if (!measured.complete || measured.bytes > requestMaxBytes) return {
ok: false,
reason: "invalid-context"
};
const serialized = stableStringify(params.request);
const claim = params.identity.turnClaim;
const claimKey = serializeWorkerSessionTurnClaim(claim);
const hash = createHash("sha256").update(`${claimKey}\0${serialized}`).digest("hex");
const existing = active.get(claimKey);
if (existing) {
if (existing.request.turnId === params.request.turnId && existing.requestHash === hash && !existing.settled) {
const retryEntry = existing;
retryEntry.identity = params.identity;
retryEntry.sink = params.sink;
if (params.revalidate) retryEntry.revalidate = params.revalidate;
else delete retryEntry.revalidate;
return {
ok: true,
result: { status: "accepted" },
launch: () => launchEntry(retryEntry)
};
}
const staleFence = durableFence(existing);
if (!staleFence) return {
ok: false,
reason: "invalid-context"
};
settleAbort(existing, staleFence);
return {
ok: false,
reason: "invalid-context"
};
}
for (const concurrent of active.values()) {
if (concurrent.request.sessionId !== params.request.sessionId) continue;
const staleFence = durableFence(concurrent);
if (!staleFence) return {
ok: false,
reason: "invalid-context"
};
settleAbort(concurrent, staleFence);
return {
ok: false,
reason: "invalid-context"
};
}
const storeInput = {
environmentId: params.identity.environmentId,
sessionId: params.request.sessionId,
runEpoch: params.request.runEpoch,
runId: params.request.runId,
turnId: params.request.turnId,
requestHash: hash
};
let begin;
try {
begin = store.begin(storeInput);
} catch {
return {
ok: false,
reason: "provider-error"
};
}
if (begin.kind === "rejected") return {
ok: false,
reason: "invalid-context"
};
const replayResult = (cachedOutcome) => {
let launched = false;
return {
ok: true,
result: { status: "replayed" },
launch: () => {
if (launched) return;
launched = true;
const fence = safeRevalidate(params.revalidate);
const frame = {
type: "event",
event: "worker.inference.terminal",
payload: {
runEpoch: params.request.runEpoch,
sessionId: params.request.sessionId,
runId: params.request.runId,
turnId: params.request.turnId,
seq: 1,
outcome: fence ? terminalError(fence) : cachedOutcome
}
};
trySend(params.sink, frame);
}
};
};
if (begin.kind === "replay") return replayResult(begin.outcome);
if (begin.kind === "recover") {
const outcome = terminalError("provider-error");
let storedOutcome;
try {
storedOutcome = store.complete({
...storeInput,
outcome
});
} catch {
return {
ok: false,
reason: "provider-error"
};
}
return replayResult(storedOutcome);
}
let runningForSession = 0;
for (const sessionId of operations.values()) if (sessionId === params.request.sessionId) runningForSession += 1;
if (runningForSession >= MAX_PROVIDER_OPERATIONS_PER_SESSION) try {
return replayResult(store.complete({
...storeInput,
outcome: terminalError("provider-error")
}));
} catch {
return {
ok: false,
reason: "provider-error"
};
}
const entry = {
claimKey,
identity: params.identity,
request: params.request,
requestHash: hash,
storeInput,
sink: params.sink,
...params.revalidate ? { revalidate: params.revalidate } : {},
controller: new AbortController(),
seq: 0,
streamedBytes: 0,
launched: false,
settled: false
};
active.set(claimKey, entry);
return {
ok: true,
result: { status: "accepted" },
launch: () => launchEntry(entry)
};
};
const cancel = (params) => {
const identityError = matchesIdentity(params.identity, params.request);
if (identityError) return {
ok: false,
reason: identityError
};
const revalidationError = safeRevalidate(params.revalidate);
if (revalidationError) return {
ok: false,
reason: revalidationError
};
const claimKey = serializeWorkerSessionTurnClaim(params.identity.turnClaim);
const entry = active.get(claimKey);
if (entry?.request.turnId === params.request.turnId) {
if (!settleAbort(entry, "cancelled")) return {
ok: false,
reason: "provider-error"
};
} else try {
store.cancelPending({
environmentId: params.identity.environmentId,
sessionId: params.request.sessionId,
runEpoch: params.request.runEpoch,
runId: params.request.runId,
turnId: params.request.turnId,
outcome: terminalError("cancelled")
});
} catch {
return {
ok: false,
reason: "provider-error"
};
}
return {
ok: true,
result: { status: "cancelled" }
};
};
const cancelWhere = (predicate, reason, onCancel) => {
let terminalPersistenceFailed = false;
for (const entry of active.values()) if (predicate(entry)) {
onCancel?.(entry);
terminalPersistenceFailed = !settleAbort(entry, reason) || terminalPersistenceFailed;
}
return terminalPersistenceFailed;
};
const cancelEnvironment = (environmentId, reason = "session-not-attached") => {
cancelWhere((entry) => entry.identity.environmentId === environmentId, reason);
};
const cancelClaim = (claim) => {
const claimKey = serializeWorkerSessionTurnClaim(claim);
cancelWhere((entry) => entry.claimKey === claimKey, "session-not-attached");
};
const cancelSession = (sessionId, runId) => {
const cancelledRunIds = /* @__PURE__ */ new Set();
cancelWhere((entry) => entry.request.sessionId === sessionId && (runId === void 0 || entry.request.runId === runId), "cancelled", (entry) => cancelledRunIds.add(entry.request.runId));
return [...cancelledRunIds].toSorted();
};
const hasSession = (sessionId, runId) => {
for (const entry of active.values()) if (entry.request.sessionId === sessionId && (runId === void 0 || entry.request.runId === runId)) return true;
return false;
};
const hasSessionOperation = (sessionId) => {
for (const operationSessionId of operations.values()) if (operationSessionId === sessionId) return true;
return false;
};
const beginSessionDrain = (sessionId) => {
if (drainingSessionIds.has(sessionId)) throw new Error(`Worker inference drain already owns session ${sessionId}`);
drainingSessionIds.add(sessionId);
const terminalPersistenceFailed = cancelWhere((entry) => entry.request.sessionId === sessionId, "cancelled");
const providerOperations = [];
for (const [operation, operationSessionId] of operations) if (operationSessionId === sessionId) providerOperations.push(operation);
let released = false;
return {
drained: Promise.allSettled(providerOperations).then(() => {
if (terminalPersistenceFailed) throw new Error(`Worker inference terminal persistence failed for session ${sessionId}`);
}),
hasWork: () => hasSession(sessionId) || hasSessionOperation(sessionId),
release: () => {
if (released) return;
released = true;
drainingSessionIds.delete(sessionId);
}
};
};
const resolveSessionIdForRunId = (runId) => {
const sessionIds = /* @__PURE__ */ new Set();
for (const entry of active.values()) if (entry.request.runId === runId) sessionIds.add(entry.request.sessionId);
return sessionIds.size === 1 ? sessionIds.values().next().value : void 0;
};
const stop = async () => {
stopping = true;
cancelWhere(() => true, "provider-error");
await withTimeout(Promise.allSettled(operations.keys()), options.stopDrainMs ?? 5e3, "Worker inference shutdown").catch(() => void 0);
};
const manager = {
start,
cancel,
cancelEnvironment,
cancelClaim,
cancelSession,
hasSession,
resolveSessionIdForRunId,
stop
};
Object.defineProperty(manager, "beginSessionDrain", { value: beginSessionDrain });
return manager;
}
//#endregion
//#region src/gateway/worker-environments/service-validation.ts
function requireInheritedWorkerProfileAuthorization(profileId, providerId, settings, configuredProviderId, serviceError) {
if (providerId === "device" && isRecord(settings) && typeof settings.device === "string" && profileId === `device:${settings.device}`) return;
if (!configuredProviderId) throw serviceError("profile_not_found", `Unknown worker profile: ${profileId}`);
if (normalizeCapabilityProviderId(configuredProviderId) !== providerId) throw serviceError("invalid_profile", "Inherited worker provider identity changed");
}
function requireProviderOperationTimeoutMs(operation, timeoutMs) {
if (timeoutMs === void 0) return;
if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 2147e6) throw new Error(`Worker provider ${operation} timeout must be an integer from 1 through ${MAX_TIMER_TIMEOUT_MS}ms`);
return timeoutMs;
}
function isWorkerMachineOptions(value) {
return Value.Check(WorkerMachineOptionsSchema, value);
}
function normalizeWorkerMachineOptions(value) {
if (!isWorkerMachineOptions(value)) return;
const ids = /* @__PURE__ */ new Set();
let hasDefault = false;
for (const option of value) {
if (option.id.trim() !== option.id || option.label.trim() !== option.label || ids.has(option.id) || option.default === true && hasDefault) return;
ids.add(option.id);
hasDefault ||= option.default === true;
}
return value.map((option) => ({
id: option.id,
label: option.label,
...option.cpu === void 0 ? {} : { cpu: option.cpu },
...option.memoryGb === void 0 ? {} : { memoryGb: option.memoryGb },
...option.default === void 0 ? {} : { default: option.default }
}));
}
function requireWorkerLeaseStatus(value) {
if (!isRecord(value)) throw new Error("Worker provider returned an invalid inspection result");
const status = value.status;
if (status !== "active" && status !== "dormant" && status !== "destroyed" && status !== "unknown") throw new Error("Worker provider returned an invalid inspection status");
if (status === "active") {
if (value.sharedHost !== void 0 && typeof value.sharedHost !== "boolean") throw new Error("Worker provider returned an invalid inspection result");
return {
status,
sharedHost: value.sharedHost === true
};
}
if (value.sharedHost !== void 0) throw new Error("Worker provider returned an invalid inspection result");
return { status };
}
function resolveWorkerLeaseTransportError(provider, transport, executionMode) {
const modes = provider.supportedExecutionModes;
if (executionMode !== void 0 && executionMode !== "worker-turn" && executionMode !== "remote-exec") return new WorkerProviderError("Worker environment has an invalid placement execution mode");
if (transport === "ssh" && (executionMode === "worker-turn" || modes !== void 0 && !modes.includes("remote-exec"))) return new WorkerProviderError("worker-turn providers must return a node lease");
if (executionMode !== void 0 && !modes?.includes(executionMode)) return new WorkerProviderError(`Worker provider ${provider.id} does not advertise ${executionMode} for its ${transport} lease`);
}
function requireWorkerAllocation(value) {
if (!isRecord(value) || typeof value.leaseId !== "string" || !value.leaseId.trim() || typeof value.sharedHost !== "boolean") throw new Error("Worker provider returned an invalid allocation identity");
return {
leaseId: value.leaseId.trim(),
sharedHost: value.sharedHost
};
}
function requireWorkerLease(value) {
const hasSsh = isRecord(value) && Object.hasOwn(value, "ssh");
const hasNode = isRecord(value) && Object.hasOwn(value, "node");
if (!isRecord(value) || typeof value.leaseId !== "string" || !value.leaseId.trim() || hasSsh === hasNode || hasSsh && !isRecord(value.ssh) || hasNode && !isRecord(value.node) || value.sharedHost !== void 0 && typeof value.sharedHost !== "boolean") throw new Error("Worker provider returned an invalid provision result");
const common = {
leaseId: value.leaseId.trim(),
...value.sharedHost === true ? { sharedHost: true } : {},
...value.desktop === void 0 ? {} : { desktop: normalizeWorkerDesktopEndpoint(value.desktop) }
};
if (hasSsh) return {
...common,
ssh: normalizeWorkerSshEndpoint(value.ssh)
};
const deviceId = value.node.deviceId;
if (typeof deviceId !== "string" || !deviceId.trim()) throw new Error("Worker provider returned an invalid node device id");
return {
...common,
node: { deviceId: deviceId.trim() }
};
}
//#endregion
//#region src/gateway/worker-environments/provider-intent.ts
/** Admits one immutable allocation intent before the provider lifecycle can allocate a lease. */
function createWorkerProviderIntent(options) {
const { store, inState, serviceError, withLock, providerFor, requireWorkerProfile, resumeProvision } = options;
return async (profileId, idempotencyKey, createOptions = {}) => {
const { inherited: requestedInherited, machineClass, executionMode, projectPath, signal } = createOptions;
signal?.throwIfAborted();
const inherited = requestedInherited ? {
...requestedInherited,
profileSnapshot: { ...requestedInherited.profileSnapshot }
} : void 0;
if (inherited) delete inherited.profileSnapshot.project;
const provisionSnapshot = {
...machineClass === void 0 ? {} : { machineClass },
...executionMode === void 0 ? {} : { executionMode }
};
if (options.isStopping()) throw serviceError("invalid_state", "Worker environment service is stopping");
const normalizedProfileId = profileId.trim();
if (!normalizedProfileId || normalizedProfileId !== profileId) throw serviceError("invalid_profile", "Worker profile id must be non-empty and trimmed");
const { environmentId, provisionOperationId } = deriveEnvironmentIntent(idempotencyKey);
return withLock(environmentId, async () => {
signal?.throwIfAborted();
if (options.isStopping()) throw serviceError("invalid_state", "Worker environment service is stopping");
const existing = store.get(environmentId);
if (existing) {
const existingProject = readWorkerProjectSnapshot(existing.profileSnapshot.project);
if (existingProject && projectPath) {
const root = await fs.realpath(projectPath);
signal?.throwIfAborted();
if (existingProject.root !== root) throw serviceError("invalid_profile", "Idempotency key belongs to another project");
}
if (existing.profileId !== normalizedProfileId || inherited !== void 0 && (existing.providerId !== inherited.providerId || !isDeepStrictEqual(existing.profileSnapshot, {
...inherited.profileSnapshot,
...provisionSnapshot,
...existingProject ? { project: existingProject } : {}
})) || inherited === void 0 && (existing.profileSnapshot.machineClass !== machineClass || existing.profileSnapshot.executionMode !== executionMode)) throw serviceError("invalid_profile", "Idempotency key belongs to another profile");
if (existing.destroyRequestedAtMs !== null) return existing;
if (!existing.leaseId && inState(existing, "requested", "provisioning")) return resumeProvision(existing, void 0, signal);
return existing;
}
let provider;
let providerId;
let profileSnapshot;
const profiles = options.getConfig().cloudWorkers?.profiles;
const configuredProfile = profiles && Object.hasOwn(profiles, normalizedProfileId) ? profiles[normalizedProfileId] : void 0;
if (inherited) {
providerId = normalizeCapabilityProviderId(inherited.providerId) ?? inherited.providerId;
if (providerId !== inherited.providerId) throw serviceError("invalid_profile", "Inherited worker provider id is not canonical");
requireInheritedWorkerProfileAuthorization(normalizedProfileId, providerId, inherited.profileSnapshot.settings, configuredProfile?.provider, serviceError);
provider = providerFor(providerId);
if ((normalizeCapabilityProviderId(provider.id) ?? provider.id) !== providerId) throw serviceError("invalid_profile", "Inherited worker provider identity changed");
profileSnapshot = requireWorkerProfile({
...inherited.profileSnapshot,
...provisionSnapshot
});
} else {
if (!configuredProfile) throw serviceError("profile_not_found", `Unknown worker profile: ${normalizedProfileId}`);
provider = providerFor(configuredProfile.provider);
providerId = normalizeCapabilityProviderId(provider.id) ?? provider.id;
const settings = requireWorkerProfile(configuredProfile.settings ?? {});
profileSnapshot = requireWorkerProfile({
install: configuredProfile.install ?? "bundle",
settings,
...provisionSnapshot
});
}
if (projectPath && provider.supportsProjectPreparation?.(requireWorkerProfile(profileSnapshot.settings), machineClass)) {
if (!options.projectNamespace) throw serviceError("invalid_state", "Worker project preparation namespace is unavailable");
const project = await prepareWorkerProjectSnapshot({
localPath: projectPath,
namespace: options.projectNamespace,
signal
});
signal?.throwIfAborted();
if (options.isStopping()) throw serviceError("invalid_state", "Worker environment service is stopping");
if (project) profileSnapshot = {
...profileSnapshot,
project
};
}
const intent = store.createIntent({
environmentId,
providerId,
profileId: normalizedProfileId,
profileSnapshot,
provisionOperationId
});
return resumeProvision(intent, provider, signal);
});
};
}
//#endregion
//#region src/gateway/worker-environments/provider-node-provisioning.ts
function createWorkerNodeProvisioning(options) {
const prepareBundle = async (preparedInstallation, signal) => {
const artifact = preparedInstallation?.install === "bundle" ? preparedInstallation : await options.prepareInstallation("bundle", signal);
signal?.throwIfAborted();
if (artifact.install !== "bundle") throw new Error("Worker bundle preparation returned the wrong install channel");
return artifact;
};
const prepare = async (record, provider, signal) => {
if (record.state !== "requested" || !provider.requiresNodeEnrollment || !options.prepareNodeBootstrap) return;
try {
await options.prepareNodeBootstrap(record, signal);
} catch (error) {
signal?.throwIfAborted();
const current = options.store.get(record.environmentId);
if (current?.state === "requested" && current.provisionOperationId === record.provisionOperationId) options.move(current, "failed", { lastError: boundedWorkerError(error) });
throw options.serviceError("bootstrap_failure", `Worker node bootstrap preparation failed: ${boundedWorkerError(error)}`);
}
const current = options.store.get(record.environmentId);
if (options.isStopping() || !current || current.state !== record.state || current.provisionOperationId !== record.provisionOperationId || current.destroyRequestedAtMs !== null) throw options.serviceError("invalid_state", "Worker provisioning changed during bootstrap preparation");
};
const createEnrollmentOperation = (record, provider, signal, preparedInstallation) => {
if (provider.requiresNodeEnrollment !== true) return;
const prepareNodeEnrollment = options.prepareNodeEnrollment;
const prepareNodeRuntime = options.prepareNodeRuntime;
if (!prepareNodeEnrollment) throw new Error("Worker node enrollment runtime is unavailable");
let open = true;
const controller = new AbortController();
let runtime;
let pendingRuntime;
let enrollment;
let pending;
const close = () => {
if (!open) return;
open = false;
signal?.removeEventListener("abort", close);
controller.abort();
if (runtime) {
options.closeNodeRuntime?.(runtime);
runtime = void 0;
}
if (enrollment) {
options.closeNodeEnrollment?.(enrollment);
enrollment = void 0;
}
};
signal?.addEventListener("abort", close, { once: true });
if (signal?.aborted) close();
const assertCurrent = () => {
const current = options.store.get(record.environmentId);
if (!open || options.isStopping() || current?.state !== "provisioning" || current.destroyRequestedAtMs !== null || current.provisionOperationId !== record.provisionOperationId || current.ownerEpoch !== record.ownerEpoch) {
controller.abort();
throw new DOMException("Worker provisioning operation is closed", "AbortError");
}
};
const assertRuntimeCurrent = () => {
assertCurrent();
if (pending) throw new Error("Worker node enrollment has already begun");
};
return {
prepareRuntime: prepareNodeRuntime ? async () => {
assertRuntimeCurrent();
pendingRuntime ??= (async () => {
const artifact = await prepareBundle(preparedInstallation, controller.signal);
assertRuntimeCurrent();
const prepared = await prepareNodeRuntime(record, artifact, controller.signal);
try {
assertRuntimeCurrent();
} catch (error) {
options.closeNodeRuntime?.(prepared);
throw error;
}
runtime = prepared;
return prepared;
})();
return await pendingRuntime;
} : void 0,
begin: async () => {
assertCurrent();
if (runtime) {
options.closeNodeRuntime?.(runtime);
runtime = void 0;
}
pending ??= prepareNodeEnrollment(record, controller.signal).then((prepared) => {
try {
assertCurrent();
} catch (error) {
options.closeNodeEnrollment?.(prepared);
throw error;
}
enrollment = prepared;
return prepared;
});
return await pending;
},
close
};
};
const finish = async (record, lease, provider, patch, preparedInstallation, cancellation) => {
const nodePatch = {
...patch,
nodeDeviceId: lease.node.deviceId,
sshEndpoint: null
};
let nodeBuild;
try {
if (!options.ensureNodeWorkerBundle) throw new Error("Device worker bundle installer is unavailable");
const artifact = await prepareBundle(preparedInstallation, cancellation?.signal);
cancellation?.assertActive();
nodeBuild = await options.ensureNodeWorkerBundle({
deviceId: lease.node.deviceId,
artifact,
prewarm: record.profileSnapshot.executionMode !== "remote-exec",
signal: cancellation?.signal
});
cancellation?.assertActive();
} catch (error) {
return await options.failBootstrap(record, lease.leaseId, provider, error, nodePatch);
}
return options.commitReady(record, {
...nodeBuild,
installKind: "bundle"
}, nodePatch);
};
return {
prepare,
createEnrollmentOperation,
finish
};
}
//#endregion
//#region src/gateway/worker-environments/provider-owner-lifecycle.ts
function createWorkerProviderOwnerLifecycle(options) {
const { store, serviceError, move, inState, callProvider, saveError, withLock, providerFor, requireWorkerProfile } = options;
const tunnels = options.tunnelManager;
const lifecycleLease = (record, leaseId) => ({
leaseId,
profile: requireWorkerProfile(record.profileSnapshot.settings)
});
const requireCurrentOwner = (record) => {
const current = store.get(record.environmentId);
if (!current || current.ownerEpoch !== record.ownerEpoch || current.state !== record.state || current.leaseId !== record.leaseId || current.nodeDeviceId !== record.nodeDeviceId || current.sharedHost !== record.sharedHost || !isDeepStrictEqual(current.attachedSessionIds, record.attachedSessionIds)) throw serviceError("invalid_state", "Worker environment owner changed during teardown");
return current;
};
const stopOwner = async (record, reason) => {
requireCurrentOwner(record);
const sessionId = record.attachedSessionIds.length === 1 ? record.attachedSessionIds[0] : null;
if (sessionId) options.placementStore?.prepareWorkspaceResultOwnerRevocation({
sessionId,
environmentId: record.environmentId,
ownerEpoch: record.ownerEpoch
}, new Error(record.lastError ?? "Cloud worker owner revoked before workspace recovery"));
store.revokeEnvironmentCredential(record.environmentId);
await tunnels?.stop(record.environmentId, record.ownerEpoch, record.nodeDeviceId !== null && record.sharedHost === false ? reason : void 0);
return requireCurrentOwner(record);
};
const destroyLease = async (record, provider, lease) => {
requireCurrentOwner(record);
const timeoutMs = options.providerCallTimeoutMs === void 0 ? requireProviderOperationTimeoutMs("destroy", provider.resolveDestroyTimeoutMs?.(lease.profile)) : void 0;
await options.callProvider(record.environmentId, () => {
requireCurrentOwner(record);
return provider.destroy(lease);
}, timeoutMs);
};
const beginDrain = (record) => {
const failurePatch = record.teardownTerminalState === "failed" ? { lastError: record.lastError } : void 0;
return inState(record, "bootstrapping", "ready", "attached", "idle") ? move(record, "draining", failurePatch) : record;
};
const beginDestroy = (record) => {
const failurePatch = record.teardownTerminalState === "failed" ? { lastError: record.lastError } : void 0;
const draining = beginDrain(record);
if (draining.state === "draining") return move(draining, "destroying", failurePatch);
if (draining.state === "destroying") return draining;
throw serviceError("invalid_state", `Cannot destroy worker in state: ${record.state}`);
};
const finishProvenDestroy = async (record) => {
const destroying = beginDestroy(requireCurrentOwner(record));
if (destroying.nodeSetupId) await options.retireNodeEnrollment?.(destroying);
requireCurrentOwner(destroying);
if (destroying.teardownTerminalState !== "failed") return move(destroying, "destroyed");
return move(destroying, "failed", {
leaseId: null,
nodeDeviceId: null,
sshEndpoint: null,
sharedHost: false,
lastError: destroying.lastError ?? "Worker bootstrap failed after provider teardown"
});
};
const cancelRequested = (record) => move(record, "failed", { lastError: "Provisioning canceled before provider allocation" });
const finishDestroy = async (record, provider) => {
let r = record;
if (r.state === "requested") return cancelRequested(requireCurrentOwner(r));
r = await stopOwner(r, "provider-destroying");
r = r.nodeDeviceId !== null && r.sharedHost === false ? r : beginDrain(r);
const owningProvider = provider ?? providerFor(r.providerId);
let leaseId = r.leaseId;
if (!leaseId) {
let allocation;
try {
allocation = requireWorkerAllocation(await callProvider(r.environmentId, () => {
requireCurrentOwner(r);
return owningProvider.resolveAllocation(requireWorkerProfile(r.profileSnapshot.settings), r.provisionOperationId);
}));
} catch (error) {
saveError(requireCurrentOwner(r), error);
throw serviceError("provider_failure", boundedWorkerError(error));
}
r = move(requireCurrentOwner(r), "draining", {
...allocation,
lastError: r.lastError
});
leaseId = allocation.leaseId;
}
const providerOwnsMachine = r.nodeDeviceId !== null && r.sharedHost === false;
const destroying = providerOwnsMachine ? r : beginDestroy(r);
try {
await destroyLease(destroying, owningProvider, lifecycleLease(destroying, leaseId));
} catch (error) {
saveError(requireCurrentOwner(destroying), error);
throw serviceError("provider_failure", boundedWorkerError(error));
}
return await finishProvenDestroy(providerOwnsMachine ? await stopOwner(destroying, "provider-destroyed") : destroying);
};
const destroy = async (environmentId, destroyOptions = {}) => {
if (options.isStopping()) throw serviceError("invalid_state", "Worker environment service is stopping");
return withLock(environmentId, async () => {
const abandonment = destroyOptions.abandonment;
abandonment?.authorize?.();
let record = store.get(environmentId);
if (!record) throw serviceError("environment_not_found", `Unknown worker environment: ${environmentId}`);
if (inState(record, "destroyed", "failed", "orphaned") && (!abandonment || record.state === "destroyed" || record.state === "failed" && !record.leaseId)) return record;
if (abandonment && (record.providerId !== "device" || record.ownerEpoch !== abandonment.ownerEpoch || !record.nodeDeviceId || record.sharedHost === false || record.attachedSessionIds.length !== 1 || record.attachedSessionIds[0] !== abandonment.sessionId)) throw serviceError("invalid_state", "Abandoned device worker owner changed before retirement");
if (destroyOptions.requireUnattached && record.attachedSessionIds.length > 0) throw serviceError("invalid_state", "Attached cloud workers must be stopped through sessions.reclaim");
record = store.requestDestroy({
environmentId,
state: record.state,
...abandonment ? {
terminalState: "failed",
lastError: FORCED_WORKER_ABANDONMENT_ERROR
} : {}
});
try {
const destroyed = await finishDestroy(record);
abandonment?.authorize?.();
return destroyed;
} catch (error) {
if (!abandonment || !(error instanceof WorkerTunnelOwnerDisconnectedError)) throw error;
abandonment.authorize?.();
const current = requireCurrentOwner(record);
if (current.destroyRequestedAtMs === null || store.getCredential(environmentId)) throw serviceError("invalid_state", "Abandoned device worker authority is not fenced");
return saveError(current, error);
}
});
};
return {
requireCurrentOwner,
stopOwner,
destroyLease,
beginDrain,
finishProvenDestroy,
lifecycleLease,
finishDestroy,
destroy
};
}
//#endregion
//#region src/gateway/worker-environments/provider-persisted-lease.ts
function requestStaleWorkerDestroy(record, store) {
return record.state === "attached" ? store.requestDestroy({
environmentId: record.environmentId,
state: record.state,
terminalState: "failed",
lastError: STALE_WORKER_BUILD_REASON
}) : record;
}
async function retireMismatchedWorkerLease(record, provider, store, finishDestroy) {
const transport = record.nodeDeviceId ? "node" : record.sshEndpoint ? "ssh" : void 0;
const modeError = transport ? resolveWorkerLeaseTransportError(provider, transport, record.profileSnapshot.executionMode) : void 0;
if (!modeError || record.destroyRequestedAtMs !== null) return false;
await finishDestroy(store.requestDestroy({
environmentId: record.environmentId,
state: record.state,
terminalState: "failed",
lastError: modeError.message
}), provider).catch(() => void 0);
return true;
}
//#endregion
//#region src/gateway/worker-environments/provider-provisioning-cancellation.ts
function createWorkerProvisionCancellation(store, record, signal) {
let owners = 1;
const settled = createDeferredCore();
let intentError;
const requestStop = () => {
try {
const current = store.get(record.environmentId);
if (current?.provisionOperationId === record.provisionOperationId && current.ownerEpoch === record.ownerEpoch && (current.state === "requested" || current.state === "provisioning" || current.state === "bootstrapping")) store.requestDestroy({
environmentId: current.environmentId,
state: current.state
});
} catch (error) {
intentError = toErrorObject(error, "Worker cancellation intent failed");
}
};
signal.addEventListener("abort", requestStop, { once: true });
if (signal.aborted) requestStop();
const close = () => {
if (--owners === 0) {
signal.removeEventListener("abort", requestStop);
settled.resolve();
}
};
return {
signal,
settled: settled.promise,
close,
assertActive: () => {
if (intentError !== void 0) throw intentError;
signal.throwIfAborted();
},
retainProvider: (run) => {
owners += 1;
return async () => {
try {
signal.throwIfAborted();
return await run();
} finally {
close();
}
};
}
};
}
//#endregion
//#region src/gateway/worker-environments/provider-lifecycle.ts
const ORPHANED_LEASE_ERROR = "Worker provider no longer recognizes the lease";
function createWorkerProviderLifecycle(options) {
const { store, callBootstrap, callProvider, inState, move, saveError, serviceError } = options;
const { commitReady, ensurePendingCredential } = options.credentialBroker;
function requireWorkerProfile(value) {
const error = validateCloudWorkerProfileSettings(value);
if (error) throw serviceError("invalid_profile", error);
return value;
}
const identityResolverFor = (record, provider, leaseId) => {
const profile = requireWorkerProfile(record.profileSnapshot.settings);
const resolveSshIdentity = options.resolveSshIdentity;
return async (keyRef) => {
if (!resolveSshIdentity) throw new Error("Worker SSH identity resolution is unavailable");
return await callProvider(record.environmentId, () => resolveSshIdentity({
provider,
leaseId,
profile,
keyRef
}));
};
};
const providerFor = (providerId) => {
const provider = options.resolveProvider(providerId);
if (provider) return provider;
throw serviceError("provider_not_found", `Worker provider is unavailable: ${providerId}`);
};
const { requireCurrentOwner, stopOwner, destroyLease, beginDrain, finishProvenDestroy, lifecycleLease, finishDestroy, destroy } = createWorkerProviderOwnerLifecycle({
...options,
providerFor,
requireWorkerProfile
});
const listMachineOptions = async (profileId) => {
const profile = options.getConfig().cloudWorkers?.profiles?.[profileId];
if (!profile) return;
return normalizeWorkerMachineOptions(await options.resolveProvider(profile.provider)?.listMachineOptions?.(requireWorkerProfile(profile.settings ?? {})));
};
const installFor = (record) => {
const install = record.profileSnapshot.install;
if (install === void 0 || install === "bundle") return "bundle";
if (install === "npm") return "npm";
throw serviceError("invalid_profile", "Worker profile has an invalid install method");
};
const failBootstrap = async (record, leaseId, provider, error, failureCode = "bootstrap_failure", leasePatch) => {
const detail = boundedWorkerError(error);
const failureLabel = failureCode === "invalid_profile" ? "Worker provider returned an incompatible lease" : leasePatch?.nodeDeviceId ? "Worker node bootstrap failed" : "Worker bootstrap failed";
const requested = store.requestDestroy({
environmentId: record.environmentId,
state: record.state,
terminalState: "failed",
lastError: detail
});
const stopped = await stopOwner(requested);
const draining = move(stopped, "draining", {
...leasePatch,
lastError: detail
});
const destroying = move(draining, "destroying", { lastError: detail });
try {
await destroyLease(destroying, provider, lifecycleLease(destroying, leaseId));
} catch (cleanupError) {
saveError(destroying, /* @__PURE__ */ new Error(`${detail}; provider teardown pending: ${boundedWorkerError(cleanupError)}`));
throw serviceError(failureCode, `${failureLabel}; teardown is pending: ${detail}`);
}
await finishProvenDestroy(destroying);
throw serviceError(failureCode, `${failureLabel}: ${detail}`);
};
const preserveIndeterminateProvisionCleanup = (record, error) => {
const detail = `${boundedWorkerError(error.provisionError, 480)}; provider teardown pending: ${boundedWorkerError(error.cleanupError, 480)}`;
store.adoptProvisionCleanupFailure({
environmentId: record.environmentId,
leaseId: error.leaseId,
lastError: detail
});
throw serviceError("provider_failure", `Worker provider operation failed; teardown is pending: ${detail}`);
};
const nodeProvisioning = createWorkerNodeProvisioning({
...options,
commitReady,
failBootstrap: async (record, leaseId, provider, error, patch) => await failBootstrap(record, leaseId, provider, error, "bootstrap_failure", patch)
});
const finishBootstrap = async (record, provider, installation, cancellation) => {
if (record.state !== "bootstrapping" || !record.leaseId || !record.sshEndpoint) throw serviceError("invalid_state", "Worker bootstrap requires a provisioned SSH lease");
const leaseId = record.leaseId;
const sshEndpoint = record.sshEndpoint;
let receipt;
try {
receipt = await callBootstrap(installation, (signal) => options.bootstrapWorker({
operationId: record.provisionOperationId,
sshEndpoint,
installation,
resolveIdentity: identityResolverFor(record, provider, leaseId),
signal: cancellation ? AbortSignal.any([signal, cancellation.signal]) : signal
}));
cancellation?.assertActive();
if (!verifyWorkerAdmissionHandshake(receipt, installation)) throw new Error("Worker bootstrap receipt does not match the expected build identity");
} catch (error) {
return await failBootstrap(record, leaseId, provider, error);
}
return commitReady(record, {
...receipt,
installKind: "bundle"
});
};
const finishProvision = async (record, provider, preparedInstallation, cancellation) => {
let lease;
let executionMode;
let enrollmentOperation;
let projectOperation;
try {
const profile = requireWorkerProfile(record.profileSnapshot.settings);
const requestedExecutionMode = record.profileSnapshot.executionMode;
if (requestedExecutionMode !== void 0 && requestedExecutionMode !== "worker-turn" && requestedExecutionMode !== "remote-exec") throw new WorkerProviderError("Worker environment has an invalid placement execution mode");
executionMode = requestedExecutionMode;
if (executionMode && !provider.supportedExecutionModes?.includes(executionMode)) throw new Error(`Worker provider ${provider.id} does not support ${executionMode} placement`);
const providerTimeoutMs = options.providerCallTimeoutMs === void 0 ? requireProviderOperationTimeoutMs("provision", provider.resolveProvisionTimeoutMs?.(profile)) : void 0;
const machineClass = typeof record.profileSnapshot.machineClass === "string" ? record.profileSnapshot.machineClass : void 0;
enrollmentOperation = nodeProvisioning.createEnrollmentOperation(record, provider, cancellation?.signal, preparedInstallation);
const project = readWorkerProjectSnapshot(record.profileSnapshot.project);
if (project) {
if (!provider.supportsProjectPreparation?.(profile, machineClass) || !options.projectNamespace) throw new Error("Worker provider cannot resume its prepared project contract");
projectOperation = createWorkerProjectPreparation({
project,
namespace: options.projectNamespace,
signal: cancellation?.signal,
requireCurrent: () => {
const current = requireCurrentOwner(record);
if (options.isStopping() || current.destroyRequestedAtMs !== null || current.provisionOperationId !== record.provisionOperationId || !isDeepStrictEqual(current.profileSnapshot.project, project)) throw new Error("Worker project preparation owner is no longer current");
}
});
}
const provisionOptions = machineClass || executionMode || enrollmentOperation || projectOperation || cancellation ? {
...machineClass ? { machineClass } : {},
...executionMode ? { executionMode } : {},
...enrollmentOperation ? {
beginNodeEnrollment: enrollmentOperation.begin,
prepareNodeRuntime: enrollmentOperation.prepareRuntime
} : {},
...cancellation ? { signal: cancellation.signal } : {},
...projectOperation ? { project: projectOperation.project } : {}
} : void 0;
cancellation?.assertActive();
const provision = () => {
const current = requireCurrentOwner(record);
if (options.isStopping() || current.destroyRequestedAtMs !== null) throw new Error("Worker provisioning operation is closed");
return provider.provision(profile, record.provisionOperationId, provisionOptions);
};
lease = requireWorkerLease(await callProvider(record.environmentId, cancellation ? cancellation.retainProvider(provision) : provision, providerTimeoutMs));
} catch (error) {
if (WorkerProviderError.isCleanupIndeterminate(error)) return preserveIndeterminateProvisionCleanup(record, error);
cancellation?.assertActive();
const detail = boundedWorkerError(error);
if (error instanceof WorkerProviderError || options.isServiceError(error, "invalid_profile")) {
move(record, "failed", { lastError: detail });
throw serviceError("invalid_profile", `Worker provider rejected profile: ${detail}`);
}
saveError(record, error);
throw serviceError("provider_failure", `Worker provider operation failed: ${detail}`);
} finally {
projectOperation?.close();
enrollmentOperation?.close();
}
const patch = {
leaseId: lease.leaseId,
sharedHost: lease.sharedHost === true,
desktop: lease.desktop ?? null,
...lease.node ? {
nodeDeviceId: lease.node.deviceId,
sshEndpoint: null
} : {
nodeDeviceId: null,
sshEndpoint: lease.ssh
}
};
if (cancellation?.signal.aborted) {
move(requireCurrentOwner(record), "draining", patch);
cancellation.assertActive();
}
const leaseModeError = resolveWorkerLeaseTransportError(provider, lease.node ? "node" : "ssh", executionMode);
if (leaseModeError) return await failBootstrap(record, lease.leaseId, provider, leaseModeError, "invalid_profile", patch);
if (lease.node) return await nodeProvisioning.finish(record, lease, provider, patch, preparedInstallation, cancellation);
const bootstrapping = move(record, "bootstrapping", patch);
let installation = preparedInstallation;
if (!installation) try {
installation = await options.prepareInstallation(installFor(bootstrapping), cancellation?.signal);
cancellation?.assertActive();
} catch (error) {
return await failBootstrap(bootstrapping, lease.leaseId, provider, error);
}
return finishBootstrap(bootstrapping, provider, installation, cancellation);
};
const resumeProvision = async (record, provider = providerFor(record.providerId), signal, retainProviderSettlement) => {
const cancellation = signal ? createWorkerProvisionCancellation(store, record, signal) : void 0;
if (cancellation) retainProviderSettlement?.(cancellation.settled);
try {
let installation;
await nodeProvisioning.prepare(record, provider, signal);
cancellation?.assertActive();
if (record.state === "requested" && record.destroyRequestedAtMs === null && provider.provisionBeforeInstallation !== true) {
try {
installation = await options.prepareInstallation(installFor(record), signal);
} catch (error) {
cancellation?.assertActive();
const detail = boundedWorkerError(error);
move(record, "failed", { lastError: detail });
throw serviceError("bootstrap_failure", `Worker installation preparation failed: ${detail}`);
}
cancellation?.assertActive();
}
const provisioning = record.state === "requested" ? move(record, "provisioning") : record;
return await finishProvision(provisioning, provider, installation, cancellation);
} finally {
cancellation?.close();
}
};
const reconcileRecord = async (initialRecord, signal, retainProviderSettlement) => {
let record = initialRecord;
if (record.state === "requested" && record.destroyRequestedAtMs !== null) {
await finishDestroy(record);
return;
}
let currentBundle;
if (record.destroyRequestedAtMs === null && inState(record, "ready", "idle", "attached")) try {
currentBundle = await options.prepareInstallation("bundle", signal);
if (record.bootstrapReceipt) {
if (verifyWorkerAdmissionHandshake(record.bootstrapReceipt, currentBundle)) {
const sessionId = record.state === "attached" ? record.attachedSessionIds[0] : null;
if (record.state !== "attached" || sessionId) {
ensurePendingCredential(record, sessionId ?? null);
record = store.get(record.environmentId) ?? record;
}
}
}
} catch {
signal?.throwIfAborted();
}
let provider;
try {
provider = providerFor(record.providerId);
} catch (error) {
saveError(record, error);
return;
}
const leaseId = record.leaseId;
if (!leaseId) {
await (record.destroyRequestedAtMs !== null ? finishDestroy(record, provider) : resumeProvision(record, provider, signal, retainProviderSettlement)).catch(() => void 0);
return;
}
if (await retireMismatchedWorkerLease(record, provider, store, finishDestroy)) return;
const inspection = await callProvider(record.environmentId, () => provider.inspect(lifecycleLease(record, leaseId))).then(requireWorkerLeaseStatus).catch((error) => {
saveError(record, error);
});
if (!inspection) return;
const { status } = inspection;
const teardownExpected = record.destroyRequestedAtMs !== null || record.state === "destroying";
if (status === "destroyed") {
requireCurrentOwner(record);
const requested = record.destroyRequestedAtMs === null ? store.requestDestroy({
environmentId: record.environmentId,
state: record.state,
...!teardownExpected ? {
terminalState: "failed",
lastError: "Worker environment disappeared before teardown was requested"
} : {}
}) : record;
const stopped = await stopOwner(requested, "provider-destroyed");
const draining = beginDrain(stopped);
await finishProvenDestroy(draining).catch((error) => {
saveError(draining, error);
});
return;
}
if (status === "unknown") {
requireCurrentOwner(record);
const requested = teardownExpected ? record : store.requestDestroy({
environmentId: record.environmentId,
state: record.state,
terminalState: "failed",
lastError: ORPHANED_LEASE_ERROR
});
await finishDestroy(requested, provider).catch(() => void 0);
return;
}
if (status === "dormant") {
if (teardownExpected) await finishDestroy(record, provider).catch(() => void 0);
return;
}
const inspectedSharedHost = inspection.sharedHost === true;
if (record.sharedHost !== null && record.sharedHost !== inspectedSharedHost) record = await stopOwner(record);
record = store.reconcileSharedHost({
environmentId: record.environmentId,
state: record.state,
leaseId,
sharedHost: inspectedSharedHost
});
if (record.destroyRequestedAtMs !== null) {
await finishDestroy(record, provider).catch(() => void 0);
return;
}
if (!record.sshEndpoint || record.state === "attached") {
if (currentBundle && (!record.bootstrapReceipt || !verifyWorkerAdmissionHandshake(record.bootstrapReceipt, currentBundle))) await finishDestroy(requestStaleWorkerDestroy(record, store), provider).catch(() => void 0);
return;
}
if (record.state === "draining" && record.destroyRequestedAtMs === null) {
record = await stopOwner(record);
move(record, "orphaned", { lastError: record.lastError ?? ORPHANED_LEASE_ERROR });
return;
}
if (inState(record, "bootstrapping", "ready", "idle")) {
let cancellation = signal ? createWorkerProvisionCancellation(store, record, signal) : void 0;
if (cancellation) retainProviderSettlement?.(cancellation.settled);
try {
cancellation?.assertActive();
let installation = currentBundle;
try {
installation ??= await options.prepareInstallation("bundle", signal);
} catch (error) {
if (record.bootstrapReceipt && inState(record, "ready", "idle")) {
saveError(record, error);
return;
}
await failBootstrap(record, leaseId, provider, error).catch(() => void 0);
return;
}
if (record.bootstrapReceipt && verifyWorkerAdmissionHandshake(record.bootstrapReceipt, installation)) {
ensurePendingCredential(record, null);
return;
}
if (installFor(record) === "npm") try {
installation = await options.prepareInstallation("npm", signal);
} catch (error) {
await failBootstrap(record, leaseId, provider, error).catch(() => void 0);
return;
}
record = await stopOwner(record);
cancellation?.assertActive();
const bootstrapping = record.state === "bootstrapping" ? record : move(record, "bootstrapping");
if (cancellation && bootstrapping.ownerEpoch !== record.ownerEpoch) {
cancellation.close();
cancellation = createWorkerProvisionCancellation(store, bootstrapping, cancellation.signal);
retainProviderSettlement?.(cancellation.settled);
cancellation.assertActive();
}
await finishBootstrap(bootstrapping, provider, installation, cancellation).catch(() => void 0);
return;
} finally {
cancellation?.close();
}
}
if (inState(record, "draining", "destroying")) await finishDestroy(record, provider).catch(() => void 0);
};
return {
createWithProfile: createWorkerProviderIntent({
...options,
providerFor,
requireWorkerProfile,
resumeProvision
}),
destroy,
identityResolverFor,
listMachineOptions,
providerFor,
reconcileRecord
};
}
//#endregion
//#region src/gateway/worker-environments/worker-turn-computer-rpc.ts
function createWorkerComputerRpc(params) {
return async (identity, request, signal) => {
const admitted = params.validate(identity);
if (!admitted.ok) return admitted;
if (!params.execute) return {
ok: false,
reason: "gateway-unavailable"
};
const assertCurrent = () => {
signal?.throwIfAborted();
const current = params.validate(identity);
if (!current.ok) throw new Error(`Worker computer authority closed: ${current.closeReason}`);
};
let commandParams;
try {
commandParams = JSON.parse(request.paramsJson);
} catch {
return {
ok: false,
closeReason: "invalid-frame"
};
}
try {
const schema = request.command === "screen.snapshot" ? ScreenSnapshotParamsSchema : ComputerActParamsSchema;
if (!(request.command === "computer.act" && Value.Check(NodeWorkerComputerCloseParamsSchema, commandParams)) && !Value.Check(schema, commandParams)) return {
ok: false,
closeReason: "invalid-frame"
};
assertCurrent();
const result = await params.execute({
identity,
request,
signal,
assertCurrent
});
const current = params.validate(identity);
if (!current.ok) return current;
signal?.throwIfAborted();
const response = {
type: "res",
id: "x".repeat(128),
ok: true,
payload: result
};
if (!Value.Check(WorkerComputerResultSchema, result) || Buffer.byteLength(JSON.stringify(response), "utf8") > 26214400) throw new Error("Computer result exceeds the worker image transport limit.");
return {
ok: true,
result
};
} catch (error) {
const current = params.validate(identity);
if (!current.ok) return current;
const message = error instanceof Error ? error.message : "Worker computer operation failed";
return {
ok: false,
reason: "gateway-unavailable",
message: truncateUtf16Safe(redactSensitiveText(message, { mode: "tools" }), 256)
};
}
};
}
//#endregion
//#region src/gateway/worker-environments/worker-turn-rpc.ts
var WorkerTranscriptAuthorityError = class extends Error {
constructor(outcome) {
super("Worker transcript authority closed");
this.outcome = outcome;
}
};
function createWorkerTurnRpc(options) {
const { store } = options;
const inference = options.inference;
const now = options.now;
const withLock = options.withLock;
const observedAckCursors = /* @__PURE__ */ new Map();
const pendingTerminalTurnFences = /* @__PURE__ */ new Map();
const terminalTurnFences = /* @__PURE__ */ new Map();
const workerAdmissionReceiptScope = randomUUID();
let workerAdmissionOrdinal = 0;
const placementClaim = (identity) => identity.turnClaim ?? void 0;
const processTurnBinding = (identity) => {
const turnClaim = placementClaim(identity);
return turnClaim ? {
turnClaim,
credentialHash: identity.credentialHash
} : void 0;
};
const admitWorkerAt = (admission, expectedBuild, nowMs) => {
const claim = admission.sessionId !== null ? options.placementStore?.readWorkerTurnClaim({
sessionId: admission.sessionId,
environmentId: admission.environmentId,
ownerEpoch: admission.ownerEpoch
}) : void 0;
return admitWorkerConnection({
store,
admission,
expectedBuild,
nowMs,
...claim ? { turnClaim: claim } : {},
allowExpiredCredential: true
});
};
const finishWorkerAdmission = (admission, result, capability) => {
if (!capability) return result;
const reasonCode = result.ok ? "worker_admission_gate_allowed" : `worker_admission_${result.reason ?? "failed"}`.replaceAll("-", "_");
workerAdmissionOrdinal += 1;
capability.run((identity) => recordRuntimeActionDecision({
token: identity.executionIdentityToken,
family: "worker",
operation: "admit",
outcome: result.ok ? "allowed" : "denied",
coverageState: "enforced",
reasonCode,
owner: "worker-runtime",
decisionBoundary: "gateway.worker-admission",
policyRefs: [
"worker:credential",
"worker:build",
"worker:owner-epoch",
"worker:turn-claim"
],
summary: result.ok ? "The current worker credential, build, owner epoch, and turn claim passed admission." : "Worker admission was denied by the current credential, build, owner, or claim gate.",
remediation: result.ok ? [] : [{
code: "reprovision_worker",
text: "Redispatch the session so the worker receives the current build and credential binding."
}],
discriminator: JSON.stringify([
admission.sessionId,
admission.runId,
admission.environmentId,
admission.ownerEpoch,
workerAdmissionReceiptScope,
workerAdmissionOrdinal
])
})).catch(() => void 0);
return result;
};
const matchesTurnBinding = (left, right) => sameWorkerSessionTurnClaim(left.turnClaim, right.turnClaim) && safeEqualSecret(left.credentialHash, right.credentialHash);
const recordAckCursor = (binding, cursor) => {
const current = observedAckCursors.get(binding.turnClaim.sessionId);
const currentTurn = current && matchesTurnBinding(current, binding) ? current : void 0;
const next = {
...binding,
transcriptSeq: "transcriptSeq" in cursor ? Math.max(currentTurn?.transcriptSeq ?? 0, cursor.transcriptSeq) : currentTurn?.transcriptSeq ?? 0,
liveSeq: "liveSeq" in cursor ? Math.max(currentTurn?.liveSeq ?? 0, cursor.liveSeq) : currentTurn?.liveSeq ?? 0
};
observedAckCursors.set(binding.turnClaim.sessionId, next);
return next;
};
const observedAckCursorFor = (binding) => {
const observed = observedAckCursors.get(binding.turnClaim.sessionId);
return observed && matchesTurnBinding(observed, binding) ? observed : void 0;
};
const validateWorkerPlacement = (identity) => {
if (identity.sessionId === null && identity.runId === null) return "sessionless";
if (!options.placementStore) return "invalid";
const claim = placementClaim(identity);
return claim && options.placementStore.validateWorkerTurn(claim) ? "durable" : "invalid";
};
const isTerminalLiveEvent = (request) => request.event.kind === "lifecycle" && (request.event.payload.phase === "finishing" || request.event.payload.phase === "end" || request.event.payload.phase === "error" && (request.event.payload.aborted === true || request.event.payload.fallbackExhaustedFailure === true));
const validateAttachedWorkerRequest = (identity, runEpoch, request) => {
if (options.isStopping()) return {
ok: false,
closeReason: "environment-unavailable"
};
const placement = validateWorkerPlacement(identity);
if (placement === "invalid") return {
ok: false,
closeReason: "placement-mismatch"
};
const turnBinding = processTurnBinding(identity);
const terminalFence = identity.sessionId ? terminalTurnFences.get(identity.sessionId) : void 0;
if (turnBinding && terminalFence && matchesTurnBinding(terminalFence, turnBinding)) {
if (!(request.kind === "transcript" && request.seq <= terminalFence.transcriptSeq || request.kind === "live" && request.seq <= terminalFence.liveSeq)) return {
ok: false,
closeReason: "placement-mismatch"
};
}
const credential = store.getCredential(identity.environmentId);
if (!credential || !safeEqualSecret(credential.credentialHash, identity.credentialHash)) return {
ok: false,
closeReason: "credential-replaced"
};
if (now() >= credential.expiresAtMs && placement !== "durable") return {
ok: false,
closeReason: "credential-expired"
};
const environment = store.get(identity.environmentId);
if (!environment || environment.destroyRequestedAtMs !== null) return {
ok: false,
closeReason: "environment-unavailable"
};
if (runEpoch !== identity.ownerEpoch || runEpoch !== credential.ownerEpoch || runEpoch !== environment.ownerEpoch) return {
ok: false,
reason: "epoch-mismatch"
};
if (environment.state !== "attached" || !identity.sessionId || credential.sessionId !== identity.sessionId || environment.attachedSessionIds.length !== 1 || environment.attachedSessionIds[0] !== identity.sessionId) return {
ok: false,
reason: "session-not-attached"
};
if (turnBinding && terminalFence && !matchesTurnBinding(terminalFence, turnBinding)) terminalTurnFences.delete(turnBinding.turnClaim.sessionId);
return { ok: true };
};
const commitTranscript = (identity, request) => withLock(identity.environmentId, async () => {
const assertCurrent = () => {
const binding = validateAttachedWorkerRequest(identity, request.runEpoch, {
kind: "transcript",
seq: request.seq
});
if (!binding.ok) throw new WorkerTranscriptAuthorityError(binding);
};
try {
assertCurrent();
if (!options.applyTranscriptCommit) return {
ok: false,
closeReason: "gateway-unavailable"
};
const result = await options.applyTranscriptCommit({
identity,
request,
assertCurrent
});
assertCurrent();
if (result.ok || result.reason === "stale-base-leaf") {
const placement = placementClaim(identity);
const processTurn = processTurnBinding(identity);
if (!placement || !processTurn) return {
ok: false,
closeReason: "placement-mismatch"
};
options.placementStore?.updateAckCursors({
claim: placement,
transcriptSeq: request.seq
});
recordAckCursor(processTurn, { transcriptSeq: request.seq });
}
return result;
} catch (error) {
if (error instanceof WorkerTranscriptAuthorityError) return error.outcome;
throw error;
}
});
const validateTool = (identity, toolName) => {
const requestAdmission = validateAttachedWorkerRequest(identity, identity.ownerEpoch, { kind: "session-tool" });
if (!requestAdmission.ok) return "closeReason" in requestAdmission ? requestAdmission : {
ok: false,
closeReason: "placement-mismatch"
};
const binding = placementClaim(identity);
if (!binding || !options.placementStore?.isWorkerTurnToolAuthorized(binding, toolName)) return {
ok: false,
closeReason: "method-not-allowed"
};
return { ok: true };
};
const executeComputer = createWorkerComputerRpc({
execute: options.executeComputer,
validate: (identity) => validateTool(identity, "computer")
});
const executeSessionTool = async (identity, toolName, request, signal) => {
const validate = () => validateTool(identity, toolName);
const admitted = validate();
if (!admitted.ok) return admitted;
if (!options.executeSessionTool) return {
ok: false,
reason: "gateway-unavailable"
};
const operation = toolName === "skill_workshop" && Value.Check(WorkerSkillWorkshopParamsSchema, request) ? {
toolName,
request
} : toolName === "sessions_spawn" && Value.Check(WorkerSessionsSpawnParamsSchema, request) ? {
toolName,
request
} : toolName === "sessions_send" && Value.Check(WorkerSessionsSendParamsSchema, request) ? {
toolName,
request
} : toolName === "portal" && Value.Check(WorkerPortalParamsSchema, request) ? {
toolName,
request
} : void 0;
if (!operation) return {
ok: false,
closeReason: "invalid-frame"
};
let result;
try {
result = await options.executeSessionTool({
identity,
...operation,
...signal ? { signal } : {}
});
} catch (error) {
result = { resultJson: serializeWorkerSessionToolResult(workerSessionToolErrorResult(error)) };
}
const current = validate();
return current.ok ? {
ok: true,
result
} : current;
};
const applyLiveEvent = (identity, request) => {
const binding = validateAttachedWorkerRequest(identity, request.runEpoch, {
kind: "live",
seq: request.seq
});
if (!binding.ok) {
if ("closeReason" in binding) return binding;
return {
ok: false,
details: { reason: binding.reason }
};
}
if (request.runId !== identity.runId) return {
ok: false,
closeReason: "placement-mismatch"
};
if (!options.liveEvents) return {
ok: false,
closeReason: "gateway-unavailable"
};
const result = options.liveEvents.apply({
identity,
request
});
if (result.ok) {
const processTurn = processTurnBinding(identity);
if (!processTurn) return {
ok: false,
closeReason: "placement-mismatch"
};
recordAckCursor(processTurn, { liveSeq: result.result.ackedSeq });
}
return result;
};
const pushLiveEvent = async (identity, request) => {
return await withLock(identity.environmentId, async () => {
const placement = placementClaim(identity);
const processTurn = processTurnBinding(identity);
const observed = processTurn ? observedAckCursorFor(processTurn) : void 0;
const wasNewSequence = request.seq > (observed?.liveSeq ?? 0);
const result = applyLiveEvent(identity, request);
if (!result.ok || !placement || !processTurn) return result;
const pending = pendingTerminalTurnFences.get(placement.sessionId);
if (pending && !matchesTurnBinding(pending, processTurn)) pendingTerminalTurnFences.delete(placement.sessionId);
if (isTerminalLiveEvent(request) && wasNewSequence) pendingTerminalTurnFences.set(placement.sessionId, {
...processTurn,
terminalLiveSeq: request.seq
});
const terminal = pendingTerminalTurnFences.get(placement.sessionId);
if (terminal && matchesTurnBinding(terminal, processTurn) && result.result.ackedSeq >= terminal.terminalLiveSeq) {
options.placementStore?.updateAckCursors({
claim: placement,
liveSeq: result.result.ackedSeq
});
terminalTurnFences.set(placement.sessionId, observedAckCursorFor(processTurn) ?? recordAckCursor(processTurn, { liveSeq: result.result.ackedSeq }));
pendingTerminalTurnFences.delete(placement.sessionId);
}
return result;
});
};
const revalidateInference = (identity, request) => {
if (request.sessionId !== identity.sessionId) return "session-not-attached";
const binding = validateAttachedWorkerRequest(identity, request.runEpoch, { kind: "inference" });
return binding.ok ? null : "reason" in binding ? binding.reason : "session-not-attached";
};
const startInference = (identity, request, sink) => {
if (request.sessionId !== identity.sessionId || request.runId !== identity.runId) return {
ok: false,
reason: "session-not-attached"
};
const binding = validateAttachedWorkerRequest(identity, request.runEpoch, { kind: "inference" });
if (!binding.ok) return binding;
return inference.start({
identity,
request,
sink,
revalidate: () => revalidateInference(identity, request)
});
};
const cancelInference = (identity, request) => {
if (request.sessionId !== identity.sessionId || request.runId !== identity.runId) return {
ok: false,
reason: "session-not-attached"
};
const binding = validateAttachedWorkerRequest(identity, request.runEpoch, { kind: "inference" });
if (!binding.ok) return binding;
return inference.cancel({
identity,
request,
revalidate: () => revalidateInference(identity, request)
});
};
return {
admitWorker: async (admission) => {
const claim = admission.sessionId === null || admission.runId === null ? void 0 : options.placementStore?.readWorkerTurnClaim({
sessionId: admission.sessionId,
environmentId: admission.environmentId,
ownerEpoch: admission.ownerEpoch
});
const capability = claim?.runId === admission.runId ? options.placementStore?.getExecutionIdentityCapability?.(claim) : void 0;
const finish = (result) => finishWorkerAdmission(admission, result, capability);
if (options.isStopping()) return finish({
ok: false,
reason: "environment-unavailable"
});
const preflightAtMs = now();
const preflight = admitWorkerAt(admission, admission.handshake, preflightAtMs);
if (!preflight.ok) return finish(preflight);
if (preflightAtMs >= preflight.identity.credentialExpiresAtMs) {
const placement = placementClaim(preflight.identity);
if (!placement || !options.placementStore?.validateWorkerTurn(placement)) return finish({
ok: false,
reason: "credential-expired"
});
}
let expectedBuild;
try {
expectedBuild = await options.prepareInstallation("bundle");
} catch {
return finish({
ok: false,
reason: "environment-unavailable"
});
}
if (options.isStopping()) return finish({
ok: false,
reason: "environment-unavailable"
});
const admittedAtMs = now();
const admitted = admitWorkerAt(admission, expectedBuild, admittedAtMs);
if (!admitted.ok) return finish(admitted);
const expired = admittedAtMs >= admitted.identity.credentialExpiresAtMs;
if (!options.placementStore || admitted.identity.sessionId === null && admitted.identity.runId === null) return finish(expired ? {
ok: false,
reason: "credential-expired"
} : admitted);
const placement = placementClaim(admitted.identity);
if (!placement || !options.placementStore.validateWorkerTurn(placement)) return finish({
ok: false,
reason: expired ? "credential-expired" : "placement-mismatch"
});
return finish(admitted);
},
validateWorkerConnection: (identity) => {
if (options.isStopping()) return "environment-unavailable";
const placement = validateWorkerPlacement(identity);
if (placement === "invalid") return "placement-mismatch";
const environmentFailure = validateWorkerConnectionIdentity({
store,
identity,
nowMs: now()
});
if (environmentFailure && !(environmentFailure === "credential-expired" && placement === "durable")) return environmentFailure;
return null;
},
commitTranscript,
pushLiveEvent,
executeSessionTool,
executeComputer,
startInference,
cancelInference,
cancelInferenceForSession: (params) => inference.cancelSession(params.sessionId, params.runId),
hasInferenceForSession: (sessionId, runId) => inference.hasSession(sessionId, runId),
resolveInferenceSessionForRunId: (runId) => inference.resolveSessionIdForRunId(runId),
clear: () => {
observedAckCursors.clear();
pendingTerminalTurnFences.clear();
terminalTurnFences.clear();
}
};
}
//#endregion
//#region src/gateway/worker-environments/service.ts
var WorkerEnvironmentServiceError = class extends Error {
constructor(code, message) {
super(message);
this.code = code;
}
};
const serviceError = (code, message) => new WorkerEnvironmentServiceError(code, message);
function createWorkerEnvironmentService(options) {
const { store } = options;
const warn = (message) => options.logger?.warn(message);
const operations = new KeyedAsyncQueue();
const providerOperations = new KeyedAsyncQueue();
const activeOperations = /* @__PURE__ */ new Set();
const now = options.now ?? Date.now;
const tunnelLifecycle = options.tunnelManager || options.nodeTunnelManager || options.nodeDesktopCarrier || options.nodePortalCarrier ? { stop: async (environmentId, ownerEpoch, reason) => {
await joinWorkerTunnelStops([
options.tunnelManager?.stop(environmentId, ownerEpoch),
options.nodeTunnelManager?.stop(environmentId, ownerEpoch, reason),
options.nodeDesktopCarrier?.stop(environmentId, ownerEpoch),
options.nodePortalCarrier?.stop(environmentId, ownerEpoch),
options.closeWorkerPortals?.(environmentId, ownerEpoch)
]);
} } : void 0;
const inference = createWorkerInferenceManager({
execute: options.executeInference,
getConfig: options.getConfig,
...options.inferenceStore ? { store: options.inferenceStore } : {}
});
const inferenceWithDrain = inference;
let reconcileInFlight;
let interval;
let unsubscribeSessionIdentityMutation;
let unsubscribeTurnClaimClosed = options.placementStore?.registerTurnClaimClosedHandler((claim) => inference.cancelClaim(claim));
let reconcileEnvironmentGuard;
let reconcileEnvironmentGuardClosing = false;
const guardedReconcileInFlight = /* @__PURE__ */ new Map();
let stopping = false;
const maintenanceAbort = new AbortController();
let maintenanceInFlight;
const inState = (record, ...states) => states.includes(record.state);
const trackOperation = (operation) => {
activeOperations.add(operation);
const release = () => activeOperations.delete(operation);
operation.then(release, release);
return operation;
};
const withLock = (environmentId, task) => trackOperation(operations.enqueue(environmentId, task));
const prepareInstallation = (install, signal) => {
signal?.throwIfAborted();
const preparation = trackOperation(Promise.resolve().then(() => options.prepareInstallation(install)));
return racePromiseWithAbortSignal(preparation, signal);
};
const callProvider = async (environmentId, run, timeoutMs) => {
let signalStarted;
const started = new Promise((resolve) => {
signalStarted = resolve;
});
const operation = trackOperation(providerOperations.enqueue(environmentId, async () => {
signalStarted();
return await run();
}));
await started;
return await withTimeout(operation, options.providerCallTimeoutMs ?? timeoutMs ?? 3e5, "Worker provider operation");
};
const callBootstrap = async (installation, run) => {
const controller = new AbortController();
const operation = Promise.resolve().then(() => run(controller.signal));
try {
return await withTimeout(operation, options.bootstrapCallTimeoutMs ?? workerBootstrapOperationTimeoutMs(installation), "Worker bootstrap operation");
} catch (error) {
controller.abort();
await operation.catch(() => void 0);
throw error;
}
};
const move = (record, to, patch) => {
const next = store.transition({
environmentId: record.environmentId,
from: record.state,
expectedOwnerEpoch: record.ownerEpoch,
to,
patch
});
if (to !== "ready" && to !== "idle" && to !== "attached") credentialBroker.clearEnvironment(record.environmentId);
if (to !== "attached") {
inference.cancelEnvironment(record.environmentId);
options.liveEvents?.clearEnvironment(record.environmentId);
}
return next;
};
const saveError = (record, error) => {
if (record.teardownTerminalState === "failed" && record.lastError) return record;
return store.recordError({
environmentId: record.environmentId,
state: record.state,
error: boundedWorkerError(error)
});
};
const credentialBroker = createWorkerCredentialBroker({
store,
prepareInstallation,
tunnelManager: tunnelLifecycle,
workerCredentialTtlMs: options.workerCredentialTtlMs,
generateWorkerCredential: options.generateWorkerCredential,
liveEvents: options.liveEvents,
placementStore: options.placementStore,
now,
isStopping: () => stopping,
cancelInferenceEnvironment: (environmentId) => inference.cancelEnvironment(environmentId),
inState,
move,
serviceError,
withLock
});
const providerLifecycle = createWorkerProviderLifecycle({
store,
getConfig: options.getConfig,
resolveProvider: options.resolveProvider,
prepareInstallation,
bootstrapWorker: options.bootstrapWorker,
resolveSshIdentity: options.resolveSshIdentity,
ensureNodeWorkerBundle: options.ensureNodeWorkerBundle,
prepareNodeBootstrap: options.prepareNodeBootstrap,
projectNamespace: options.projectNamespace,
prepareNodeRuntime: options.prepareNodeRuntime,
closeNodeRuntime: options.closeNodeRuntime,
prepareNodeEnrollment: options.prepareNodeEnrollment,
closeNodeEnrollment: options.closeNodeEnrollment,
retireNodeEnrollment: options.retireNodeEnrollment,
placementStore: options.placementStore,
providerCallTimeoutMs: options.providerCallTimeoutMs,
tunnelManager: tunnelLifecycle,
credentialBroker,
callBootstrap,
callProvider,
inState,
isServiceError: (error, code) => error instanceof WorkerEnvironmentServiceError && error.code === code,
isStopping: () => stopping,
move,
saveError,
serviceError,
withLock
});
const environmentAccess = createWorkerEnvironmentAccess({
store,
getConfig: options.getConfig,
prepareCurrentBundle: async () => await prepareInstallation("bundle"),
tunnelManager: options.tunnelManager,
nodeTunnelManager: options.nodeTunnelManager,
nodeDesktopCarrier: options.nodeDesktopCarrier,
now,
identityResolverFor: providerLifecycle.identityResolverFor,
inState,
isStopping: () => stopping,
providerFor: providerLifecycle.providerFor,
serviceError,
withLock
});
const turnRpc = createWorkerTurnRpc({
store,
prepareInstallation,
applyTranscriptCommit: options.applyTranscriptCommit,
liveEvents: options.liveEvents,
placementStore: options.placementStore,
executeSessionTool: options.executeSessionTool,
executeComputer: options.executeComputer,
inference,
isStopping: () => stopping,
now,
withLock
});
const reconcileEnvironmentCore = async (environmentId, signal, retainProviderSettlement) => {
if (stopping) return;
await withLock(environmentId, async () => {
const current = store.get(environmentId);
if (!current || inState(current, "destroyed", "failed", "orphaned")) return;
await providerLifecycle.reconcileRecord(current, signal, retainProviderSettlement);
});
};
const reconcileEnvironment = async (environmentId) => {
if (stopping) return;
const guard = reconcileEnvironmentGuard;
if (!guard) {
await reconcileEnvironmentCore(environmentId);
return;
}
if (reconcileEnvironmentGuardClosing) return;
const active = guardedReconcileInFlight.get(environmentId);
if (active) {
await active;
return;
}
const operation = guard(environmentId, async (signal, retainProviderSettlement) => {
await reconcileEnvironmentCore(environmentId, signal, retainProviderSettlement);
});
guardedReconcileInFlight.set(environmentId, operation);
try {
await operation;
} finally {
if (guardedReconcileInFlight.get(environmentId) === operation) guardedReconcileInFlight.delete(environmentId);
}
};
const closeReconcileEnvironmentGuard = async (expected) => {
const guard = reconcileEnvironmentGuard;
if (!guard || expected && guard !== expected) return;
reconcileEnvironmentGuardClosing = true;
while (guardedReconcileInFlight.size > 0) await Promise.allSettled(guardedReconcileInFlight.values());
if (reconcileEnvironmentGuard === guard) {
reconcileEnvironmentGuard = void 0;
reconcileEnvironmentGuardClosing = false;
}
};
const installReconcileEnvironmentGuard = (guard) => {
if (reconcileEnvironmentGuard) throw new Error("Worker environment reconciliation guard is already installed");
reconcileEnvironmentGuard = guard;
reconcileEnvironmentGuardClosing = false;
return async () => await closeReconcileEnvironmentGuard(guard);
};
const reconcilePass = async (environmentId) => {
const tasks = (environmentId === void 0 ? store.listForReconcile() : [store.get(environmentId)].filter((candidate) => candidate !== void 0)).map((candidate) => () => reconcileEnvironment(candidate.environmentId).catch(() => warn(`Worker environment reconcile failed (${candidate.environmentId}, ${candidate.providerId})`)));
await runTasksWithConcurrency({
tasks,
limit: 8
});
if (environmentId !== void 0) return;
try {
store.pruneTerminalEnvironments();
} catch (error) {
if (!isSqliteLockError(error)) throw error;
}
};
const reconcileOnce = (environmentId) => {
if (stopping) return Promise.resolve();
if (environmentId !== void 0) return trackOperation(reconcilePass(environmentId));
if (options.maintainProviders && !maintenanceInFlight) maintenanceInFlight = trackOperation(Promise.resolve().then(() => {
maintenanceAbort.signal.throwIfAborted();
return options.maintainProviders(maintenanceAbort.signal);
}).catch(() => {
if (!stopping) warn("Worker provider maintenance sweep failed; cleanup will retry");
}).finally(() => {
maintenanceInFlight = void 0;
}));
return reconcileInFlight ??= reconcilePass().finally(() => {
reconcileInFlight = void 0;
});
};
const start = () => {
if (interval || stopping) return;
unsubscribeSessionIdentityMutation = onSessionIdentityMutation((mutation) => {
const currentSessionId = "current" in mutation ? mutation.current.sessionId : void 0;
if (mutation.previous.sessionId && mutation.previous.sessionId !== currentSessionId) inference.cancelSession(mutation.previous.sessionId);
});
options.liveEvents?.start();
interval = setInterval(() => void reconcileOnce().catch(() => warn("Worker environment reconcile sweep failed")), options.reconcileIntervalMs ?? 6e4);
interval.unref?.();
reconcileOnce().catch(() => warn("Worker environment startup reconcile failed"));
};
const stop = async () => {
stopping = true;
maintenanceAbort.abort();
options.stopNodeEnrollmentWaits?.();
clearInterval(interval);
interval = void 0;
unsubscribeSessionIdentityMutation?.();
unsubscribeSessionIdentityMutation = void 0;
unsubscribeTurnClaimClosed?.();
unsubscribeTurnClaimClosed = void 0;
await closeReconcileEnvironmentGuard();
await options.closeComputers?.().catch(() => warn("Session computer cleanup failed during Gateway shutdown"));
await inference.stop();
credentialBroker.clear();
options.liveEvents?.clear();
options.stopNodeWorkerBundleTransfers?.();
try {
await joinWorkerTunnelStops([environmentAccess.stopAllTunnels(), options.nodePortalCarrier?.stopAll()]);
} finally {
const reconciliation = reconcileInFlight;
if (reconciliation) await Promise.allSettled([reconciliation]);
while (activeOperations.size > 0) await Promise.allSettled(activeOperations);
credentialBroker.clear();
turnRpc.clear();
options.liveEvents?.clear();
await options.closeNodeBootstrapArtifacts?.();
}
};
const providerSupportsExecutionMode = (providerId, mode) => options.resolveProvider(providerId)?.supportedExecutionModes?.includes(mode) === true;
const requireProviderExecutionMode = (providerId, mode) => {
if (!mode) return;
const provider = options.resolveProvider(providerId);
if (!provider) throw serviceError("provider_not_found", `Unknown worker provider: ${providerId}`);
if (!provider.supportedExecutionModes?.includes(mode)) throw serviceError("invalid_profile", `Worker provider ${providerId} does not support ${mode} placement`);
};
const configuredProfileProviderId = (profileId) => {
const profile = options.getConfig().cloudWorkers?.profiles?.[profileId];
if (!profile) throw serviceError("profile_not_found", `Unknown worker profile: ${profileId}`);
return profile.provider;
};
const service = {
list: environmentAccess.list,
supportsProviderExecutionMode: providerSupportsExecutionMode,
supportsExecutionMode: (profileId, mode) => {
const profile = options.getConfig().cloudWorkers?.profiles?.[profileId];
return profile ? providerSupportsExecutionMode(profile.provider, mode) : false;
},
requiresNodeEnrollment: (profileId, providerId) => {
const id = providerId ?? options.getConfig().cloudWorkers?.profiles?.[profileId]?.provider;
return id ? options.resolveProvider(id)?.requiresNodeEnrollment === true : false;
},
get: environmentAccess.get,
inventoryVersion: store.inventoryVersion,
supportsNodePortal: async (environmentId, ownerEpoch) => await options.nodePortalCarrier?.supports(environmentId, ownerEpoch) === true,
hasPendingNodeEnrollmentSetup: (setupId, deviceId) => store.hasPendingNodeEnrollmentSetup(setupId, deviceId),
listMachineOptions: async (profileId) => providerLifecycle.listMachineOptions(profileId),
create: async (profileId, idempotencyKey, machineClass, executionMode, projectPath, signal) => {
if (executionMode) requireProviderExecutionMode(configuredProfileProviderId(profileId), executionMode);
return environmentAccess.project(await providerLifecycle.createWithProfile(profileId, idempotencyKey, {
machineClass,
executionMode,
projectPath,
signal
}));
},
createFromProfileSnapshot: async (profile, idempotencyKey, machineClass, executionMode, projectPath, signal) => {
requireProviderExecutionMode(profile.providerId, executionMode);
return environmentAccess.project(await providerLifecycle.createWithProfile(profile.profileId, idempotencyKey, {
inherited: {
providerId: profile.providerId,
profileSnapshot: profile.profileSnapshot
},
machineClass,
executionMode,
projectPath,
signal
}));
},
destroy: async (environmentId, abandonment) => environmentAccess.project(await providerLifecycle.destroy(environmentId, { abandonment })),
destroyUnattached: async (environmentId) => environmentAccess.project(await providerLifecycle.destroy(environmentId, { requireUnattached: true })),
observeDesktop: environmentAccess.observeDesktop,
launchDesktopApp: environmentAccess.launchDesktopApp,
admitWorker: turnRpc.admitWorker,
validateWorkerConnection: turnRpc.validateWorkerConnection,
commitTranscript: turnRpc.commitTranscript,
pushLiveEvent: turnRpc.pushLiveEvent,
executeSessionTool: turnRpc.executeSessionTool,
executeComputer: turnRpc.executeComputer,
prepareComputer: options.prepareComputer,
startInference: turnRpc.startInference,
cancelInference: turnRpc.cancelInference,
cancelInferenceForSession: turnRpc.cancelInferenceForSession,
hasInferenceForSession: turnRpc.hasInferenceForSession,
resolveInferenceSessionForRunId: turnRpc.resolveInferenceSessionForRunId,
resolveSshIdentity: async (environmentId) => {
const record = store.get(environmentId);
if (!record) throw serviceError("environment_not_found", `Unknown worker environment: ${environmentId}`);
if (!record.leaseId || !record.sshEndpoint) throw serviceError("invalid_state", `Worker environment ${environmentId} has no active SSH endpoint`);
const provider = providerLifecycle.providerFor(record.providerId);
return await providerLifecycle.identityResolverFor(record, provider, record.leaseId)(record.sshEndpoint.keyRef);
},
attachSession: credentialBroker.attachSession,
takeMintedCredential: credentialBroker.takeMintedCredential,
acquireTurnCredential: credentialBroker.acquireTurnCredential,
acknowledgeCredentialDelivery: credentialBroker.acknowledgeCredentialDelivery,
startTunnel: environmentAccess.startTunnel,
stopTunnel: async (environmentId, ownerEpoch) => {
await Promise.all([
environmentAccess.stopTunnel(environmentId, ownerEpoch),
options.nodePortalCarrier?.stop(environmentId, ownerEpoch),
options.closeWorkerPortals?.(environmentId, ownerEpoch)
]);
},
stopNodeEnrollmentWaits: options.stopNodeEnrollmentWaits,
installReconcileEnvironmentGuard,
reconcileEnvironment,
reconcileOnce,
start,
stop
};
registerWorkerInferenceSessionDrain(service, (sessionId) => inferenceWithDrain.beginSessionDrain(sessionId));
return service;
}
//#endregion
export { createWorkerEnvironmentService };