@sellable/install
Version:
One-command installer for Sellable MCP in Claude Code, Codex, and Hermes
1,471 lines (1,436 loc) • 46.2 kB
JavaScript
import { execFile } from "node:child_process";
import {
createHash,
createPrivateKey,
createPublicKey,
generateKeyPairSync,
randomUUID,
sign,
} from "node:crypto";
import {
chmodSync,
chownSync,
existsSync,
lstatSync,
mkdirSync,
readFileSync,
renameSync,
writeFileSync,
} from "node:fs";
import { isAbsolute, join } from "node:path";
import { promisify } from "node:util";
import { Upload as TusUpload } from "tus-js-client";
const execFileAsync = promisify(execFile);
const SHA256 = /^[a-f0-9]{64}$/;
function imageBundleIdentity(env) {
const rawVersion = env?.SELLABLE_AGENT_DEFAULT_PROFILE_BUNDLE_VERSION;
const bundleDigest = env?.SELLABLE_AGENT_DEFAULT_PROFILE_BUNDLE_DIGEST;
if (rawVersion === undefined && bundleDigest === undefined) return null;
const bundleVersion = Number(rawVersion);
if (
!Number.isSafeInteger(bundleVersion) ||
bundleVersion < 1 ||
!SHA256.test(bundleDigest ?? "")
) {
throw new Error("runtime_bundle_identity_rejected");
}
return Object.freeze({ bundleVersion, bundleDigest });
}
const SAFE_ID = /^[A-Za-z0-9_-]{1,128}$/;
const FLY_OIDC_AUTHORIZATION_SCHEME = "FlyOIDC";
function stableJson(value) {
if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`;
if (value && typeof value === "object") {
return `{${Object.keys(value)
.sort()
.map((key) => `${JSON.stringify(key)}:${stableJson(value[key])}`)
.join(",")}}`;
}
return JSON.stringify(value);
}
export function flyRuntimeDigest(value) {
return createHash("sha256").update(stableJson(value)).digest("hex");
}
function privateDirectory(path) {
if (!isAbsolute(path)) throw new Error("fly_identity_root_rejected");
mkdirSync(path, { recursive: true, mode: 0o700 });
chmodSync(path, 0o700);
const stat = lstatSync(path);
if (
stat.isSymbolicLink() ||
!stat.isDirectory() ||
(stat.mode & 0o077) !== 0
) {
throw new Error("fly_identity_root_rejected");
}
}
function atomicPrivateFile(path, bytes) {
const temp = `${path}.${process.pid}.${randomUUID()}.tmp`;
writeFileSync(temp, bytes, { flag: "wx", mode: 0o600 });
chmodSync(temp, 0o600);
renameSync(temp, path);
chmodSync(path, 0o600);
}
export function loadOrCreateFlyRuntimeIdentity({ identityRoot, persistent }) {
privateDirectory(identityRoot);
const privateKeyPath = join(identityRoot, "ed25519-private.pem");
const publicKeyPath = join(identityRoot, "ed25519-public.pem");
if (!existsSync(privateKeyPath) || !existsSync(publicKeyPath)) {
const pair = generateKeyPairSync("ed25519");
const privateKeyPem = pair.privateKey
.export({ type: "pkcs8", format: "pem" })
.toString();
const publicKeyPem = pair.publicKey
.export({ type: "spki", format: "pem" })
.toString();
atomicPrivateFile(privateKeyPath, privateKeyPem);
atomicPrivateFile(publicKeyPath, publicKeyPem);
}
const privateLink = lstatSync(privateKeyPath);
const publicLink = lstatSync(publicKeyPath);
if (
privateLink.isSymbolicLink() ||
publicLink.isSymbolicLink() ||
!privateLink.isFile() ||
!publicLink.isFile() ||
(privateLink.mode & 0o077) !== 0 ||
(publicLink.mode & 0o077) !== 0
) {
throw new Error("fly_identity_key_rejected");
}
const privateKeyPem = readFileSync(privateKeyPath, "utf8");
const publicKeyPem = readFileSync(publicKeyPath, "utf8");
const privateKey = createPrivateKey(privateKeyPem);
const publicKey = createPublicKey(publicKeyPem);
if (
privateKey.asymmetricKeyType !== "ed25519" ||
publicKey.asymmetricKeyType !== "ed25519" ||
createPublicKey(privateKey)
.export({ type: "spki", format: "pem" })
.toString() !== publicKeyPem
) {
throw new Error("fly_identity_key_rejected");
}
return Object.freeze({
privateKeyPath,
publicKeyPem,
publicKeyHash: createHash("sha256").update(publicKeyPem).digest("hex"),
persistent: Boolean(persistent),
});
}
function parseToken(value) {
const token =
typeof value === "string"
? value.trim()
: (value?.token ?? value?.access_token);
if (typeof token !== "string" || token.length < 32 || token.length > 16_384) {
throw new Error("fly_oidc_token_rejected");
}
return token;
}
export function parseFlyOidcTokenResponse(raw) {
if (typeof raw !== "string" || raw.length > 64 * 1024) {
throw new Error("fly_oidc_token_rejected");
}
const value = raw.trim();
let parsed;
try {
parsed = JSON.parse(value);
} catch {
return parseToken(value);
}
return parseToken(parsed);
}
/**
* @param {{
* gid: number;
* path?: string;
* fsImpl?: {
* lstatSync: (path: string) => {
* uid: number;
* gid: number;
* mode: number;
* isSocket: () => boolean;
* isSymbolicLink: () => boolean;
* };
* chownSync: (path: string, uid: number, gid: number) => void;
* chmodSync: (path: string, mode: number) => void;
* };
* }} input
*/
export function handoffFlyOidcSocket({
gid,
path = "/.fly/api",
fsImpl = { lstatSync, chownSync, chmodSync },
}) {
if (
!Number.isSafeInteger(gid) ||
gid < 1 ||
path !== "/.fly/api" ||
typeof fsImpl?.lstatSync !== "function" ||
typeof fsImpl?.chownSync !== "function" ||
typeof fsImpl?.chmodSync !== "function"
) {
throw new Error("fly_oidc_socket_rejected");
}
const before = fsImpl.lstatSync(path);
if (
before.isSymbolicLink() ||
!before.isSocket() ||
before.uid !== 0 ||
(before.mode & 0o002) !== 0
) {
throw new Error("fly_oidc_socket_rejected");
}
fsImpl.chownSync(path, 0, gid);
fsImpl.chmodSync(path, 0o660);
const after = fsImpl.lstatSync(path);
if (
after.isSymbolicLink() ||
!after.isSocket() ||
after.uid !== 0 ||
after.gid !== gid ||
(after.mode & 0o777) !== 0o660
) {
throw new Error("fly_oidc_socket_handoff_failed");
}
return Object.freeze({ path, uid: 0, gid, mode: 0o660 });
}
/**
* @param {{
* user: string;
* group: string;
* beforeSetuid?: (identity: { gid: number }) => void;
* processImpl?: {
* getuid: () => number;
* getgid: () => number;
* initgroups: (user: string, extraGroup: string | number) => void;
* setgid: (id: string | number) => void;
* setuid: (id: string | number) => void;
* };
* }} input
*/
export function dropFlyRuntimePrivileges({
user,
group,
beforeSetuid,
processImpl = process,
}) {
if (
!/^[a-z_][a-z0-9_-]{0,31}$/.test(user ?? "") ||
!/^[a-z_][a-z0-9_-]{0,31}$/.test(group ?? "") ||
typeof processImpl.getuid !== "function" ||
typeof processImpl.getgid !== "function" ||
typeof processImpl.initgroups !== "function" ||
typeof processImpl.setgid !== "function" ||
typeof processImpl.setuid !== "function" ||
(beforeSetuid !== undefined && typeof beforeSetuid !== "function")
) {
throw new Error("fly_privilege_drop_rejected");
}
if (processImpl.getuid() !== 0 || processImpl.getgid() !== 0) {
throw new Error("fly_privilege_bootstrap_root_required");
}
processImpl.initgroups(user, group);
processImpl.setgid(group);
const runtimeGid = processImpl.getgid();
if (runtimeGid === 0) {
throw new Error("fly_privilege_drop_rejected");
}
beforeSetuid?.({ gid: runtimeGid });
processImpl.setuid(user);
const uid = processImpl.getuid();
const gid = processImpl.getgid();
if (uid === 0 || gid === 0) {
throw new Error("fly_privilege_drop_rejected");
}
return Object.freeze({ uid, gid });
}
export async function obtainFlyOidcToken({ audience, env = process.env }) {
if (
typeof audience !== "string" ||
audience.length < 1 ||
audience.length > 512
) {
throw new Error("fly_oidc_audience_rejected");
}
if (env.SELLABLE_AGENT_ALLOW_TEST_OIDC === "1") {
const endpoint = env.SELLABLE_AGENT_TEST_OIDC_ENDPOINT ?? "";
if (
!/^http:\/\/(?:127\.0\.0\.1|host\.docker\.internal)(?::[0-9]+)?\//.test(
endpoint
)
) {
throw new Error("fly_test_oidc_endpoint_rejected");
}
const response = await fetch(endpoint, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ aud: audience }),
signal: AbortSignal.timeout(10_000),
});
if (!response.ok) throw new Error("fly_oidc_unavailable");
return parseFlyOidcTokenResponse(await response.text());
}
const { stdout } = await execFileAsync(
"/usr/bin/curl",
[
"--fail",
"--silent",
"--show-error",
"--unix-socket",
"/.fly/api",
"--header",
"content-type: application/json",
"--request",
"POST",
"--data-binary",
JSON.stringify({ aud: audience }),
"http://localhost/v1/tokens/oidc",
],
{ timeout: 10_000, maxBuffer: 64 * 1024 }
);
return parseFlyOidcTokenResponse(stdout);
}
function exactHttpsOrigin(value) {
let parsed;
try {
parsed = new URL(value);
} catch {
throw new Error("fly_control_origin_rejected");
}
const testHttp =
process.env.SELLABLE_AGENT_ALLOW_TEST_OIDC === "1" &&
parsed.protocol === "http:" &&
["127.0.0.1", "host.docker.internal"].includes(parsed.hostname);
if (
(parsed.protocol !== "https:" && !testHttp) ||
parsed.username ||
parsed.password ||
parsed.pathname !== "/" ||
parsed.search ||
parsed.hash
) {
throw new Error("fly_control_origin_rejected");
}
return parsed.origin;
}
/**
* @param {{
* origin: string;
* runtimeId: string;
* audience: string;
* identity: { publicKeyPem: string; publicKeyHash: string };
* bootSessionId: string;
* oidcToken?: string | null;
* env?: NodeJS.ProcessEnv;
* fetchImpl?: typeof fetch;
* }} input
*/
export async function exchangeFlyCustomerIdentity({
origin,
runtimeId,
audience,
identity,
bootSessionId,
oidcToken = null,
env = process.env,
fetchImpl = fetch,
}) {
if (!/^fly-runtime_[a-f0-9]{32}$/.test(runtimeId ?? "")) {
throw new Error("fly_runtime_identity_rejected");
}
const endpoint = exactHttpsOrigin(origin);
if (!identity || !SHA256.test(identity.publicKeyHash ?? "")) {
throw new Error("fly_runtime_public_key_rejected");
}
if (
!/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(
bootSessionId ?? ""
)
) {
throw new Error("fly_boot_session_invalid");
}
const workloadToken =
oidcToken === null
? await obtainFlyOidcToken({ audience, env })
: parseToken(oidcToken);
const response = await fetchImpl(
`${endpoint}/api/v3/sellable-agent/worker/fly-exchange`,
{
method: "POST",
headers: {
authorization: `${FLY_OIDC_AUTHORIZATION_SCHEME} ${workloadToken}`,
"content-type": "application/json",
},
body: JSON.stringify({
runtimeId,
publicKeyPem: identity.publicKeyPem,
bootSessionId,
}),
signal: AbortSignal.timeout(15_000),
}
);
const value = await response.json();
if (!response.ok)
throw new Error(value?.error ?? "fly_workload_exchange_failed");
const exchange = value?.exchange;
if (
exchange?.kind !== "CUSTOMER" ||
exchange.runtimeId !== runtimeId ||
!SAFE_ID.test(exchange.hostId ?? "") ||
!SAFE_ID.test(exchange.workerId ?? "") ||
!SAFE_ID.test(exchange.enrollmentId ?? "") ||
!Number.isSafeInteger(exchange.keyGeneration) ||
exchange.keyGeneration < 1 ||
exchange.publicKeyHash !== identity.publicKeyHash ||
!Number.isFinite(Date.parse(exchange.expiresAt ?? ""))
) {
throw new Error("fly_workload_exchange_rejected");
}
return Object.freeze(exchange);
}
export async function exchangeFlyAdminIdentity({
origin,
runtimeId,
audience,
identity,
bootSessionId,
oidcToken = null,
env = process.env,
fetchImpl = fetch,
}) {
if (!/^fly-runtime_[a-f0-9]{32}$/.test(runtimeId ?? "")) {
throw new Error("fly_runtime_identity_rejected");
}
const endpoint = exactHttpsOrigin(origin);
if (!identity || !SHA256.test(identity.publicKeyHash ?? "")) {
throw new Error("fly_runtime_public_key_rejected");
}
if (
!/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(
bootSessionId ?? ""
)
) {
throw new Error("fly_boot_session_invalid");
}
const workloadToken =
oidcToken === null
? await obtainFlyOidcToken({ audience, env })
: parseToken(oidcToken);
const response = await fetchImpl(
`${endpoint}/api/v3/sellable-agent/worker/fly-exchange`,
{
method: "POST",
headers: {
authorization: `${FLY_OIDC_AUTHORIZATION_SCHEME} ${workloadToken}`,
"content-type": "application/json",
},
body: JSON.stringify({
kind: "ADMIN",
runtimeId,
publicKeyPem: identity.publicKeyPem,
bootSessionId,
}),
signal: AbortSignal.timeout(15_000),
}
);
const value = await response.json();
if (!response.ok)
throw new Error(value?.error ?? "fly_workload_exchange_failed");
const exchange = value?.exchange;
const target = exchange?.adminDeliveryTarget;
if (
exchange?.kind !== "ADMIN" ||
exchange.runtimeId !== runtimeId ||
!SAFE_ID.test(exchange.hostId ?? "") ||
!SAFE_ID.test(exchange.workerId ?? "") ||
!SAFE_ID.test(exchange.enrollmentId ?? "") ||
!Number.isSafeInteger(exchange.keyGeneration) ||
exchange.keyGeneration < 1 ||
exchange.publicKeyHash !== identity.publicKeyHash ||
!Number.isFinite(Date.parse(exchange.expiresAt ?? "")) ||
target?.agentKind !== "ADMIN" ||
target.runtimeId !== runtimeId ||
target.flyMachineId !== exchange.workerId ||
!SAFE_ID.test(target.workspaceId ?? "") ||
!SAFE_ID.test(target.agentId ?? "") ||
!SAFE_ID.test(target.flyAppId ?? "") ||
!SAFE_ID.test(target.flyMachineVersion ?? "") ||
!Number.isSafeInteger(target.runtimeGeneration) ||
target.runtimeGeneration < 1 ||
!SAFE_ID.test(target.adminRuntimeGenerationId ?? "") ||
!/^[1-9][0-9]{0,19}$/.test(target.adminRuntimeGenerationFence ?? "")
) {
throw new Error("fly_admin_workload_exchange_rejected");
}
return Object.freeze({
...exchange,
adminDeliveryTarget: Object.freeze({ ...target }),
});
}
export function signedWorkerAuthorization({
identity,
exchange,
verb,
body,
now,
unbound = false,
}) {
const privateKey = createPrivateKey(
readFileSync(identity.privateKeyPath, "utf8")
);
const claims = {
enrollmentId: exchange.enrollmentId,
hostId: exchange.hostId,
workerId: exchange.workerId,
audience: "sellable-agent-worker/v1",
keyGeneration: exchange.keyGeneration,
verb,
operationId: unbound ? null : (body.operationId ?? null),
workspaceId: unbound ? null : (body.workspaceId ?? null),
agentId: unbound ? null : (body.agentId ?? null),
fence: unbound ? null : (body.fence ?? null),
targetRevisionId: unbound ? null : (body.targetRevisionId ?? null),
bodyDigest: flyRuntimeDigest(body),
nonce: randomUUID(),
issuedAt: now.toISOString(),
expiresAt: new Date(now.getTime() + 60_000).toISOString(),
};
const signature = sign(
null,
Buffer.from(stableJson(claims)),
privateKey
).toString("base64url");
return `Worker ${Buffer.from(
JSON.stringify({ claims, publicKeyPem: identity.publicKeyPem, signature })
).toString("base64url")}`;
}
// Product claims can carry the endpoint's bounded 256 KiB input both as the
// run input and as execution context, plus bounded prompt/history metadata.
const MAX_WORKER_CONTROL_RESPONSE_CHARACTERS = 2 * 1024 * 1024;
const RETRYABLE_WORKER_CONTROL_STATUSES = new Set([
408, 425, 429, 500, 502, 503, 504,
]);
class WorkerControlResponseError extends Error {
constructor(code, { retryable = false, status = null } = {}) {
super(code);
this.name = "WorkerControlResponseError";
this.retryable = retryable;
this.status = status;
}
}
const TUS_CHUNK_BYTES = 6 * 1024 * 1024;
const TUS_TOKEN_REFRESH_MS = 45_000;
const TUS_RETRY_DELAYS_MS = [0, 3_000, 5_000, 10_000, 20_000];
function tusUploadAccepted(value) {
try {
const endpoint = new URL(value?.resumableUrl);
return (
endpoint.protocol === "https:" &&
endpoint.username === "" &&
endpoint.password === "" &&
endpoint.search === "" &&
endpoint.hash === "" &&
/^[a-z0-9-]+\.storage\.supabase\.co$/.test(endpoint.hostname) &&
endpoint.pathname === "/storage/v1/upload/resumable/sign" &&
typeof value?.signedUploadToken === "string" &&
value.signedUploadToken.length >= 32 &&
value.signedUploadToken.length <= 16_384 &&
!/\s/.test(value.signedUploadToken) &&
value.bucketName === "agent-hermes-backups-v1" &&
/^workspaces\/[A-Za-z0-9_-]{1,128}\/(?:(?:customer|admin)\/archives|agents\/[A-Za-z0-9_-]{1,128}\/operations\/[A-Za-z0-9_-]{1,128})\/[a-f0-9]{64}\.zip$/.test(
value?.objectName ?? ""
)
);
} catch {
return false;
}
}
function tusFailureCode(error) {
const response = error?.originalResponse;
const status = Number(response?.getStatus?.());
if (!Number.isSafeInteger(status) || status < 100 || status > 599) {
return "hermes_snapshot_upload_failed";
}
let detail = "";
try {
detail = String(response.getBody?.() ?? "")
.slice(0, 4_096)
.toLowerCase();
} catch {}
const category = /mime|content.?type/.test(detail)
? "mime"
: /row.?level|policy|\brls\b/.test(detail)
? "policy"
: /payload|too.?large|file.?size|size.?limit|maximum size/.test(detail)
? "size"
: /bucket/.test(detail)
? "bucket"
: /upsert|duplicate|already.?exists/.test(detail)
? "upsert"
: /token|signature|signed|(?:^|\W)exp(?:\W|$)/.test(detail)
? "token"
: /api.?key|jwt|authorization|unauthorized|forbidden/.test(detail)
? "auth"
: "other";
return `hermes_snapshot_upload_failed_http_${status}_${category}`;
}
export function uploadSupabaseSignedTus({
archive,
initial,
refresh,
now = Date.now,
UploadImpl = TusUpload,
}) {
if (
!Buffer.isBuffer(archive) ||
archive.byteLength < 1 ||
!tusUploadAccepted(initial) ||
typeof refresh !== "function" ||
typeof now !== "function" ||
typeof UploadImpl !== "function"
) {
return Promise.reject(
new WorkerControlResponseError("memory_snapshot_upload_url_rejected")
);
}
const identity = {
resumableUrl: initial.resumableUrl,
bucketName: initial.bucketName,
objectName: initial.objectName,
};
let token = initial.signedUploadToken;
let issuedAt = now();
return new Promise((resolve, reject) => {
let upload;
const fail = (error) => {
reject(new WorkerControlResponseError(tusFailureCode(error)));
};
try {
upload = new UploadImpl(archive, {
endpoint: identity.resumableUrl,
retryDelays: TUS_RETRY_DELAYS_MS,
headers: { "x-signature": token },
uploadDataDuringCreation: true,
storeFingerprintForResuming: false,
removeFingerprintOnSuccess: true,
metadata: {
bucketName: identity.bucketName,
objectName: identity.objectName,
contentType: "application/zip",
cacheControl: "3600",
},
chunkSize: TUS_CHUNK_BYTES,
async onBeforeRequest(request) {
if (now() - issuedAt >= TUS_TOKEN_REFRESH_MS) {
const next = await refresh();
if (
!tusUploadAccepted(next) ||
next.resumableUrl !== identity.resumableUrl ||
next.bucketName !== identity.bucketName ||
next.objectName !== identity.objectName
) {
throw new WorkerControlResponseError(
"memory_snapshot_upload_url_rejected"
);
}
token = next.signedUploadToken;
issuedAt = now();
}
request.setHeader("x-signature", token);
},
onError: fail,
onSuccess() {
resolve({ uploadUrl: upload.url });
},
});
upload.start();
} catch (error) {
fail(error);
}
});
}
const SOUL_OBSERVATION_GENERATION_MAX = 2_147_483_647;
export const SOUL_OBSERVATION_INTERVAL_MS = 60_000;
/**
* Content-free state for the existing worker claim tick. The bridge receipt is
* deliberately projected to hashes before it is retained here.
*/
export function createSoulClaimObservationTracker({
runtimeId,
profileId,
observationIntervalMs = SOUL_OBSERVATION_INTERVAL_MS,
now = Date.now,
}) {
if (
!SAFE_ID.test(runtimeId ?? "") ||
!/^[a-z0-9][a-z0-9_-]{0,127}$/.test(profileId ?? "")
) {
throw new Error("soul_claim_identity_rejected");
}
if (
!Number.isSafeInteger(observationIntervalMs) ||
observationIntervalMs < 1_000 ||
typeof now !== "function"
) {
throw new Error("soul_claim_observation_cadence_rejected");
}
let generation = 0;
let lastObservationTuple = null;
let lastObservationAttemptAt = null;
let pending = null;
const exactAcknowledgement = (acknowledgement) =>
Boolean(
pending &&
acknowledgement?.schemaVersion === "sellable-agent-soul-ack/v1" &&
acknowledgement.generation === pending.generation &&
acknowledgement.rawFileSha256 === pending.rawFileSha256 &&
acknowledgement.effectiveSha256 === pending.effectiveSha256 &&
stableJson(acknowledgement.journal ?? null) ===
stableJson(pending.journal ?? null)
);
return Object.freeze({
beginObservation({ force = false } = {}) {
const at = now();
if (!Number.isSafeInteger(at) || at < 0) {
throw new Error("soul_claim_observation_clock_rejected");
}
if (
!force &&
lastObservationAttemptAt !== null &&
at >= lastObservationAttemptAt &&
at - lastObservationAttemptAt < observationIntervalMs
) {
return false;
}
lastObservationAttemptAt = at;
return true;
},
observe(receipt) {
if (
receipt?.ok !== true ||
!SHA256.test(receipt.rawFileSha256 ?? "") ||
!SHA256.test(receipt.effectiveSha256 ?? "")
) {
return pending;
}
const journal =
receipt.pendingMirror === true &&
typeof receipt.generation === "string" &&
receipt.generation.length >= 8 &&
receipt.generation.length <= 128
? Object.freeze({
generation: receipt.generation,
digest: flyRuntimeDigest({
generation: receipt.generation,
rawFileSha256: receipt.rawFileSha256,
effectiveSha256: receipt.effectiveSha256,
}),
})
: undefined;
const observationTuple = stableJson({
rawFileSha256: receipt.rawFileSha256,
effectiveSha256: receipt.effectiveSha256,
pendingMirror: receipt.pendingMirror === true,
journalGeneration: journal?.generation ?? null,
});
if (observationTuple !== lastObservationTuple) {
generation =
generation >= SOUL_OBSERVATION_GENERATION_MAX ? 1 : generation + 1;
lastObservationTuple = observationTuple;
pending = Object.freeze({
schemaVersion: "sellable-agent-soul-observation/v1",
runtimeId,
profileId,
generation,
rawFileSha256: receipt.rawFileSha256,
effectiveSha256: receipt.effectiveSha256,
...(journal ? { journal } : {}),
});
}
return pending;
},
acknowledgementJournalGeneration(acknowledgement) {
return exactAcknowledgement(acknowledgement)
? (pending.journal?.generation ?? null)
: null;
},
acceptsAcknowledgement(acknowledgement) {
return exactAcknowledgement(acknowledgement);
},
acknowledge(acknowledgement) {
if (!exactAcknowledgement(acknowledgement)) return false;
pending = null;
return true;
},
pending() {
return pending;
},
});
}
/** One pass through the existing claim cadence; no timer is created here. */
export async function claimWithSoulObservation({
client,
tracker,
readSoul,
leaseSeconds = 60,
servingObservation = /** @type {any} */ (null),
}) {
// Deliver lifecycle work before any potentially slow local loader proof.
// Reconciliation uses the same cadence only after the server confirms that
// this tick did not lease ordinary work.
const tick = await client.claimTick({
leaseSeconds,
soulObservation: tracker.pending(),
...(servingObservation ? { servingObservation } : {}),
});
if (tick?.claim) return tick.claim;
const acknowledgement = tick?.soulAcknowledgement;
let observedDuringAcknowledgement = false;
if (tracker.acceptsAcknowledgement(acknowledgement)) {
const journalGeneration =
tracker.acknowledgementJournalGeneration(acknowledgement);
let locallyAcknowledged = true;
if (journalGeneration) {
try {
tracker.beginObservation({ force: true });
const receipt = await readSoul({
operation: "read",
acknowledgedGeneration: journalGeneration,
});
observedDuringAcknowledgement = true;
locallyAcknowledged = Boolean(
receipt?.ok === true &&
receipt.profileId === tracker.pending()?.profileId &&
receipt.pendingMirror === false &&
receipt.rawFileSha256 === acknowledgement.rawFileSha256 &&
receipt.effectiveSha256 === acknowledgement.effectiveSha256
);
if (!locallyAcknowledged) tracker.observe(receipt);
} catch {
locallyAcknowledged = false;
}
}
if (locallyAcknowledged) tracker.acknowledge(acknowledgement);
}
if (!observedDuringAcknowledgement && tracker.beginObservation()) {
try {
tracker.observe(await readSoul({ operation: "read" }));
} catch {
// A local proof failure must not starve the next lifecycle claim tick.
}
}
return null;
}
function workerControlVerbCode(verb) {
return String(verb).replaceAll("-", "_");
}
async function readWorkerControlResponse(response, verb) {
const status = Number(response.status);
const verbCode = workerControlVerbCode(verb);
let body;
try {
body = await response.text();
} catch {
throw new WorkerControlResponseError(
`worker_control_response_read_failed_${verbCode}_${status}`,
{
retryable: response.ok || RETRYABLE_WORKER_CONTROL_STATUSES.has(status),
}
);
}
if (body.length > MAX_WORKER_CONTROL_RESPONSE_CHARACTERS) {
throw new WorkerControlResponseError(
`worker_control_response_too_large_${verbCode}_${status}`
);
}
if (!body.trim()) {
throw new WorkerControlResponseError(
`worker_control_response_empty_${verbCode}_${status}`,
{
retryable: response.ok || RETRYABLE_WORKER_CONTROL_STATUSES.has(status),
}
);
}
let value;
try {
value = JSON.parse(body);
} catch {
throw new WorkerControlResponseError(
`worker_control_response_invalid_json_${verbCode}_${status}`,
{
retryable: response.ok || RETRYABLE_WORKER_CONTROL_STATUSES.has(status),
}
);
}
if (!value || typeof value !== "object" || Array.isArray(value)) {
throw new WorkerControlResponseError(
`worker_control_response_invalid_shape_${verbCode}_${status}`,
{
retryable: response.ok || RETRYABLE_WORKER_CONTROL_STATUSES.has(status),
}
);
}
if (!response.ok) {
const candidate =
typeof value.error === "string"
? value.error
: typeof value.code === "string"
? value.code
: null;
const code =
candidate && /^[a-z0-9_:-]{1,128}$/.test(candidate)
? candidate
: `worker_control_http_${verbCode}_${status}`;
throw new WorkerControlResponseError(code, {
retryable: RETRYABLE_WORKER_CONTROL_STATUSES.has(status),
status,
});
}
return value;
}
export function createFlyCustomerControlClient({
origin,
identity,
exchange,
env = process.env,
fetchImpl = fetch,
retryDelayMs = 150,
tusUploadImpl = uploadSupabaseSignedTus,
renewExchange = /** @type {(() => Promise<any>) | null} */ (null),
}) {
const endpoint = exactHttpsOrigin(origin);
const bundleIdentity = imageBundleIdentity(env);
if (
!Number.isSafeInteger(retryDelayMs) ||
retryDelayMs < 0 ||
retryDelayMs > 5_000
) {
throw new Error("worker_control_retry_delay_invalid");
}
let renewal = null;
const renewEnrollment = async (expiredExchange) => {
// Concurrent heartbeat, upload and completion requests share one renewal.
if (exchange !== expiredExchange) return;
if (!renewal) {
renewal = Promise.resolve()
.then(() => renewExchange())
.then((next) => {
const unchanged = [
"kind",
"runtimeId",
"enrollmentId",
"hostId",
"workerId",
"keyGeneration",
"publicKeyHash",
].every((key) => next?.[key] === expiredExchange[key]);
if (
!unchanged ||
stableJson(next?.adminDeliveryTarget ?? null) !==
stableJson(expiredExchange.adminDeliveryTarget ?? null) ||
!(Date.parse(next?.expiresAt ?? "") > Date.now())
) {
throw new Error("worker_control_renewal_identity_rejected");
}
exchange = next;
})
.finally(() => {
renewal = null;
});
}
await renewal;
};
const withEnrollment = async (request) => {
const attemptedExchange = exchange;
try {
return await request(attemptedExchange);
} catch (error) {
// This authentication rejection happens before the operation's effect.
// Never replay an ambiguous transport failure or a revoked enrollment.
if (
typeof renewExchange !== "function" ||
!(error instanceof WorkerControlResponseError) ||
error.message !== "enrollment_expired" ||
![401, 403].includes(error.status)
)
throw error;
await renewEnrollment(attemptedExchange);
return request(exchange);
}
};
const post = async (verb, path, body, { replaySafe = false } = {}) => {
const presentedBundle =
body && typeof body === "object" && !Array.isArray(body)
? imageBundleIdentity({
SELLABLE_AGENT_DEFAULT_PROFILE_BUNDLE_VERSION:
body.bundleVersion === undefined
? undefined
: String(body.bundleVersion),
SELLABLE_AGENT_DEFAULT_PROFILE_BUNDLE_DIGEST: body.bundleDigest,
})
: null;
if (
presentedBundle &&
bundleIdentity &&
(presentedBundle.bundleVersion !== bundleIdentity.bundleVersion ||
presentedBundle.bundleDigest !== bundleIdentity.bundleDigest)
) {
throw new Error("runtime_bundle_identity_rejected");
}
const signedBody =
verb === "complete" && bundleIdentity && !presentedBundle
? { ...body, ...bundleIdentity }
: body;
const maxAttempts = replaySafe ? 3 : 1;
for (let attempt = 1; ; attempt += 1) {
try {
return await withEnrollment(async (currentExchange) => {
const authorization = signedWorkerAuthorization({
identity,
exchange: currentExchange,
verb,
body: signedBody,
now: new Date(),
});
let response;
try {
response = await fetchImpl(`${endpoint}${path}`, {
method: "POST",
headers: { authorization, "content-type": "application/json" },
body: JSON.stringify(signedBody),
signal: AbortSignal.timeout(20_000),
});
} catch {
throw new WorkerControlResponseError(
`worker_control_fetch_failed_${workerControlVerbCode(verb)}`,
{ retryable: replaySafe }
);
}
return await readWorkerControlResponse(response, verb);
});
} catch (error) {
if (
!(error instanceof WorkerControlResponseError) ||
!error.retryable ||
attempt >= maxAttempts
) {
throw error;
}
if (retryDelayMs > 0) {
await new Promise((resolve) =>
setTimeout(resolve, retryDelayMs * attempt)
);
}
}
}
};
const memorySnapshotControl = async (method, body, verb, search = "") =>
withEnrollment(async (currentExchange) => {
const response = await fetchImpl(
`${endpoint}/api/v3/sellable-agent/worker/memory-snapshot${search}`,
{
method,
headers: {
authorization: signedWorkerAuthorization({
identity,
exchange: currentExchange,
verb,
body,
now: new Date(),
unbound: method === "GET",
}),
...(method === "GET" ? {} : { "content-type": "application/json" }),
},
...(method === "GET" ? {} : { body: JSON.stringify(body) }),
signal: AbortSignal.timeout(60_000),
}
);
if (method === "GET" && response.status === 404) return null;
return readWorkerControlResponse(response, verb);
});
const metadataAccepted = (value) =>
/^[a-f0-9]{64}$/.test(value?.archiveDigest ?? "") &&
Number.isSafeInteger(value?.archiveBytes) &&
value.archiveBytes > 0 &&
value.archiveBytes <= 512 * 1024 * 1024 &&
value.hermesVersion === "0.21.5" &&
/^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/.test(value?.sourceProfileId ?? "");
const boundedSnapshotIdentity = (value) =>
/^[A-Za-z0-9][A-Za-z0-9_./:-]{0,255}$/.test(value ?? "");
const metadataV2Accepted = (value) =>
metadataAccepted(value) &&
Object.keys(value).sort().join(",") ===
"archiveBytes,archiveDigest,authorityGeneration,hermesVersion,operationId,profileSchemaVersion,runtimeArtifactId,sourceBootSessionId,sourceMachineId,sourceMachineVersion,sourceProfileId" &&
boundedSnapshotIdentity(value.operationId) &&
Number.isSafeInteger(value.authorityGeneration) &&
value.authorityGeneration > 0 &&
boundedSnapshotIdentity(value.runtimeArtifactId) &&
boundedSnapshotIdentity(value.profileSchemaVersion) &&
[
value.sourceMachineId,
value.sourceMachineVersion,
value.sourceBootSessionId,
].every(
(identityValue) =>
identityValue === null || boundedSnapshotIdentity(identityValue)
);
const snapshotReadbackExpected = (value) =>
value &&
typeof value === "object" &&
!Array.isArray(value) &&
Object.keys(value).sort().join(",") ===
"authorityGeneration,operationId,profileSchemaVersion,runtimeArtifactId,sourceProfileId" &&
boundedSnapshotIdentity(value.operationId) &&
Number.isSafeInteger(value.authorityGeneration) &&
value.authorityGeneration > 0 &&
boundedSnapshotIdentity(value.runtimeArtifactId) &&
/^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/.test(value.sourceProfileId ?? "") &&
boundedSnapshotIdentity(value.profileSchemaVersion);
const signedUrlAccepted = (value) => {
try {
return new URL(value).protocol === "https:";
} catch {
return false;
}
};
const putMemorySnapshot = async (metadata, archive) => {
const isV2 = Object.hasOwn(metadata ?? {}, "operationId");
if (
!(isV2 ? metadataV2Accepted(metadata) : metadataAccepted(metadata)) ||
!Buffer.isBuffer(archive) ||
metadata.archiveBytes !== archive.byteLength ||
createHash("sha256").update(archive).digest("hex") !==
metadata.archiveDigest
) {
throw new WorkerControlResponseError("memory_snapshot_upload_rejected");
}
const beginUpload = async () => {
const value = await memorySnapshotControl(
"PUT",
{
schemaVersion: isV2
? "sellable-hermes-snapshot-upload/v2"
: "sellable-hermes-snapshot-upload/v1",
...metadata,
},
"memory-snapshot-put"
);
if (
!tusUploadAccepted(value) ||
!value.objectName.endsWith(
isV2
? `/operations/${metadata.operationId}/${metadata.archiveDigest}.zip`
: `/${metadata.archiveDigest}.zip`
)
) {
throw new WorkerControlResponseError(
"memory_snapshot_upload_url_rejected"
);
}
return value;
};
const begin = await beginUpload();
if (typeof tusUploadImpl !== "function") {
throw new WorkerControlResponseError(
"memory_snapshot_upload_url_rejected"
);
}
await tusUploadImpl({ archive, initial: begin, refresh: beginUpload });
const committed = await memorySnapshotControl(
"POST",
{
schemaVersion: isV2
? "sellable-hermes-snapshot-commit/v2"
: "sellable-hermes-snapshot-commit/v1",
...metadata,
},
"memory-snapshot-put"
);
if (
committed?.archiveDigest !== metadata.archiveDigest ||
committed?.archiveBytes !== metadata.archiveBytes
) {
throw new WorkerControlResponseError("memory_snapshot_commit_rejected");
}
return committed;
};
const getMemorySnapshot = async (expected) => {
const isV2 = expected !== undefined;
if (isV2 && !snapshotReadbackExpected(expected)) {
throw new WorkerControlResponseError("memory_snapshot_download_rejected");
}
const body = isV2
? {
schemaVersion: "sellable-hermes-snapshot-get/v2",
operationId: expected.operationId,
}
: { schemaVersion: "sellable-hermes-snapshot-get/v1" };
const search = isV2 ? `?${new URLSearchParams(body).toString()}` : "";
const result = await memorySnapshotControl(
"GET",
body,
"memory-snapshot-get",
search
);
if (!result) return null;
const identityReadbackAccepted =
!isV2 ||
(result.operationId === expected.operationId &&
result.authorityGeneration === expected.authorityGeneration &&
result.runtimeArtifactId === expected.runtimeArtifactId &&
result.sourceProfileId === expected.sourceProfileId &&
result.profileSchemaVersion === expected.profileSchemaVersion &&
/^[a-f0-9]{64}$/.test(result.archiveDigest ?? "") &&
Number.isSafeInteger(result.archiveBytes) &&
result.archiveBytes > 0 &&
result.archiveBytes <= 512 * 1024 * 1024);
if (
!(isV2 ? identityReadbackAccepted : metadataAccepted(result)) ||
!signedUrlAccepted(result.signedUrl)
) {
throw new WorkerControlResponseError(
"memory_snapshot_download_url_rejected"
);
}
const downloaded = await fetchImpl(result.signedUrl, {
signal: AbortSignal.timeout(120_000),
});
if (!downloaded.ok) {
throw new WorkerControlResponseError("memory_snapshot_download_failed");
}
const bytes = Buffer.from(await downloaded.arrayBuffer());
if (
result.archiveBytes !== bytes.byteLength ||
createHash("sha256").update(bytes).digest("hex") !== result.archiveDigest
) {
throw new WorkerControlResponseError("memory_snapshot_readback_rejected");
}
return { ...result, signedUrl: undefined, bytes };
};
const claimTick = async ({
leaseSeconds = 60,
soulObservation,
servingObservation,
} = {}) =>
post("claim", "/api/v3/sellable-agent/worker/claim", {
leaseSeconds,
...(soulObservation ? { soulObservation } : {}),
...(servingObservation ? { servingObservation } : {}),
});
return Object.freeze({
claimTick,
async claim(leaseSeconds = 60) {
return (await claimTick({ leaseSeconds })).claim ?? null;
},
async heartbeat(identityBody) {
return post(
"heartbeat",
"/api/v3/sellable-agent/worker/heartbeat",
identityBody,
{ replaySafe: true }
);
},
async claimEndpoint(leaseSeconds = 60) {
return (
(
await post(
"endpoint-claim",
"/api/v3/sellable-agent/worker/endpoint-claim",
{ leaseSeconds }
)
).claim ?? null
);
},
async heartbeatEndpoint(identityBody) {
return post(
"endpoint-heartbeat",
"/api/v3/sellable-agent/worker/endpoint-heartbeat",
identityBody
);
},
async appendEndpointEvents(identityBody) {
return post(
"endpoint-events",
"/api/v3/sellable-agent/worker/endpoint-events",
identityBody,
{ replaySafe: true }
);
},
async completeEndpoint(identityBody) {
return post(
"endpoint-complete",
"/api/v3/sellable-agent/worker/endpoint-complete",
identityBody,
{ replaySafe: true }
);
},
async acknowledgeEndpointControl(identityBody) {
return post(
"endpoint-control",
"/api/v3/sellable-agent/worker/endpoint-control",
identityBody,
{ replaySafe: true }
);
},
async fetchEndpointConnectionKey(identityBody) {
return post(
"endpoint-connection-key",
"/api/v3/sellable-agent/worker/endpoint-connection-key",
identityBody,
{ replaySafe: true }
);
},
async consumeSecret(identityBody) {
// The broker makes only this verb replay-safe for the exact claim ID.
return post(
"secret-consume",
"/api/v3/sellable-agent/worker/secret-consume",
identityBody,
{ replaySafe: true }
);
},
async complete(receipt) {
return post(
"complete",
"/api/v3/sellable-agent/worker/complete",
receipt
);
},
async putMemorySnapshot(metadata, archive) {
return putMemorySnapshot(metadata, archive);
},
async getMemorySnapshot(expected) {
return getMemorySnapshot(expected);
},
});
}
export function isFlyCustomerEnrollmentRenewalError(error) {
return error instanceof Error && error.message === "enrollment_expired";
}
export async function renewFlyCustomerControlSession({
origin,
runtimeId,
audience,
identity,
bootSessionId,
runtimeGeneration,
obtainOidcTokenImpl = obtainFlyOidcToken,
exchangeFlyCustomerIdentityImpl = exchangeFlyCustomerIdentity,
createFlyCustomerControlClientImpl = createFlyCustomerControlClient,
}) {
if (
!Number.isSafeInteger(runtimeGeneration) ||
runtimeGeneration < 1 ||
!/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(
bootSessionId ?? ""
) ||
typeof obtainOidcTokenImpl !== "function" ||
typeof exchangeFlyCustomerIdentityImpl !== "function" ||
typeof createFlyCustomerControlClientImpl !== "function"
) {
throw new Error("customer_runtime_generation_rejected");
}
const renewExchange = async () => {
let oidcToken = await obtainOidcTokenImpl({ audience });
let exchange;
try {
exchange = await exchangeFlyCustomerIdentityImpl({
origin,
runtimeId,
audience,
identity,
bootSessionId,
oidcToken,
});
} finally {
oidcToken = null;
}
if (
exchange?.kind !== "CUSTOMER" ||
exchange.runtimeId !== runtimeId ||
exchange.publicKeyHash !== identity?.publicKeyHash
) {
throw new Error("fly_workload_exchange_rejected");
}
if (exchange.keyGeneration !== runtimeGeneration) {
throw new Error("customer_runtime_generation_rejected");
}
return exchange;
};
const exchange = await renewExchange();
const client = createFlyCustomerControlClientImpl({
origin,
identity,
exchange,
renewExchange,
});
return Object.freeze({ exchange, client });
}
export async function renewFlyAdminControlSession({
origin,
runtimeId,
audience,
identity,
bootSessionId,
runtimeGeneration,
adminRuntimeGenerationId,
adminRuntimeGenerationFence,
obtainOidcTokenImpl = obtainFlyOidcToken,
exchangeFlyAdminIdentityImpl = exchangeFlyAdminIdentity,
createFlyCustomerControlClientImpl = createFlyCustomerControlClient,
}) {
if (
!Number.isSafeInteger(runtimeGeneration) ||
runtimeGeneration < 1 ||
!/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(
bootSessionId ?? ""
) ||
!SAFE_ID.test(adminRuntimeGenerationId ?? "") ||
!/^[1-9][0-9]{0,19}$/.test(adminRuntimeGenerationFence ?? "") ||
typeof obtainOidcTokenImpl !== "function" ||
typeof exchangeFlyAdminIdentityImpl !== "function" ||
typeof createFlyCustomerControlClientImpl !== "function"
) {
throw new Error("admin_runtime_generation_rejected");
}
const renewExchange = async () => {
let oidcToken = await obtainOidcTokenImpl({ audience });
let exchange;
try {
exchange = await exchangeFlyAdminIdentityImpl({
origin,
runtimeId,
audience,
identity,
bootSessionId,
oidcToken,
});
} finally {
oidcToken = null;
}
if (
exchange?.kind !== "ADMIN" ||
exchange.runtimeId !== runtimeId ||
exchange.publicKeyHash !== identity?.publicKeyHash ||
exchange.keyGeneration !== runtimeGeneration ||
exchange.adminDeliveryTarget?.runtimeGeneration !== runtimeGeneration ||
exchange.adminDeliveryTarget.adminRuntimeGenerationId !==
adminRuntimeGenerationId ||
exchange.adminDeliveryTarget.adminRuntimeGenerationFence !==
adminRuntimeGenerationFence
) {
throw new Error("admin_runtime_generation_rejected");
}
return exchange;
};
const exchange = await renewExchange();
const client = createFlyCustomerControlClientImpl({
origin,
identity,
exchange,
renewExchange,
});
return Object.freeze({ exchange, client });
}