@sellable/install
Version:
One-command installer for Sellable MCP in Claude Code, Codex, and Hermes
4,540 lines • 153 kB
JavaScript
#!/usr/bin/env node
import { spawn, spawnSync } from "node:child_process";
import {
createHash,
createPrivateKey,
createPublicKey,
randomUUID,
sign,
} from "node:crypto";
import {
chmodSync,
closeSync,
cpSync,
existsSync,
lstatSync,
mkdirSync,
mkdtempSync,
openSync,
readFileSync,
readdirSync,
renameSync,
rmSync,
writeFileSync,
} from "node:fs";
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { createRequire } from "node:module";
import { basename, dirname, join } from "node:path";
import { createInterface } from "node:readline";
import { pruneAbandonedRuntimeScratch } from "../abandoned-runtime-scratch.mjs";
import {
buildAdminSettingsManifest,
encodeAdminSettingsBundle,
importAdminSecretGeneration,
restoreAdminSecretGeneration,
} from "../admin-artifact/node_modules/@sellable/admin-install/lib/admin-secret-import.mjs";
import {
promoteAdminProfileAtomically,
stageAdminProfile,
} from "../admin-artifact/node_modules/@sellable/admin-install/lib/hermes-admin-profile-factory.mjs";
import {
buildAdminAgentServiceEnvironment,
buildNativeHermesLaunch,
buildRuntimeServingObservation,
isOperationBoundBackupClaim,
} from "../admin-artifact/node_modules/@sellable/install/lib/sellable-agent/fly-customer-worker.mjs";
import sodium from "../admin-artifact/node_modules/libsodium-wrappers/dist/modules-esm/libsodium-wrappers.mjs";
import { projectAdminEndpointRuntime } from "../admin-endpoint-runtime.mjs";
import {
nativeGatewayRunning,
nextAdminNativeGatewayRecovery,
} from "../admin-native-gateway-supervisor.mjs";
import {
bindDefaultProfileProbeCron,
buildDefaultProfileMaterializerReceipt,
buildDefaultProfileProjection,
reconcileDefaultProfileBundle,
} from "../default-profile-reconciler.mjs";
import {
SELLABLE_BROWSER_TOOLSET_DEFAULT_MARKER,
sellableSlackToolsetsForMaterialization,
} from "../fly-capabilities-bridge.mjs";
import {
startConfiguredClaudeRunsApis,
startPublicClaudeRunsApi,
} from "../fly-customer-image/claude-code-runs-api.mjs";
import { createCustomerEndpointRelay } from "../fly-customer-image/customer-runtime.mjs";
import { createHermesSessionRetention } from "../fly-customer-image/hermes-runs-client.mjs";
import {
claimWithSoulObservation,
createFlyCustomerControlClient,
createSoulClaimObservationTracker,
dropFlyRuntimePrivileges,
exchangeFlyAdminIdentity,
flyRuntimeDigest,
handoffFlyOidcSocket,
isFlyCustomerEnrollmentRenewalError,
loadOrCreateFlyRuntimeIdentity,
obtainFlyOidcToken,
renewFlyAdminControlSession,
signedWorkerAuthorization,
} from "../fly-runtime-identity.mjs";
import { proveDefaultProfileNativeSkills } from "../fly-skills-bridge.mjs";
import {
executeExternalSoulBridgeOperation,
observeProfileSoul,
prepareSoulLockRoot,
withProfileSoulLock,
} from "../fly-soul-bridge.mjs";
import {
HERMES_SNAPSHOT_HOOKS,
HERMES_SNAPSHOT_HOOK_ALLOWLIST_BYTES,
applyHermesSellableHostedPolicy,
ensureHermesClientRelationshipsCrons,
ensureHermesCustomerProgramCrons,
ensureHermesOpportunitySequencesCron,
ensureHermesRefillSendsCron,
ensureHermesSnapshotCron,
linkHermesState,
markHermesSnapshotDirty,
migrateLegacyHermesUserState,
quarantineLegacyLightfieldVenv,
reconcileHermesSnapshot,
replaceHermesState,
restoreLatestHermesSnapshot,
} from "../hermes-memory-snapshot.mjs";
import {
clearManagedAuth,
managedAuthPath,
publishManagedAuth,
} from "../managed-auth-client.mjs";
import { createMcpAdminRuntimeBindingV3 } from "../mcp-runtime-binding.mjs";
import {
evaluateModelAuthReadback,
modelAuthConfigOverlay,
modelAuthSelectionObservation,
parseModelAuthRuntimeCredential,
readModelAuthMaterialization,
reconcileHermesModelAuth,
} from "../model-auth-reconciler.mjs";
import { createModelHealthReporter } from "../model-health-reporter.mjs";
import { reconcilePublicHermesSkills } from "../public-skills.mjs";
import { createAdminRuntimePhaseTelemetry } from "./admin-runtime-phase-telemetry.mjs";
import {
completeAdminStateReceiptRecovery,
prepareAdminStateGeneration,
recoverAdminStateReceiptMismatch,
} from "./admin-state-generation.mjs";
export const ADMIN_RUNTIME_SETTINGS_DIRECTORY =
"/run/sellable-agent/admin-settings";
export const ADMIN_RUNTIME_SERVERS = Object.freeze([
"sellable",
"sellable-admin",
]);
const ARTIFACT_ROOT = "/usr/local/lib/sellable-agent/admin-artifact";
const artifactRequire = createRequire(join(ARTIFACT_ROOT, "package.json"));
const { parseDocument } = artifactRequire("yaml");
const ADMIN_PROFILE_TEMPLATE_PATH = join(
ARTIFACT_ROOT,
"profile-contract-template.json"
);
const DATA_ROOT = "/opt/data";
const PROFILES_ROOT = join(DATA_ROOT, "profiles");
const RUNTIME_ROOT = join(DATA_ROOT, ".sellable-agent", "admin-runtime");
const DELIVERY_ROOT = join(RUNTIME_ROOT, "delivery");
const SECRET_ROOT = "/run/sellable-agent/admin-runtime";
const sessionRetention = createHermesSessionRetention();
// Turn descriptor + consumed roots for the Product MCP context proxy. They are
// normally empty and receive one signed descriptor only while the fenced
// endpoint relay owns a turn; the proxy fails closed if either root is absent.
const MCP_TURN_CONTEXT_ROOT = join(SECRET_ROOT, "mcp-turn-context");
const HERMES = "/opt/hermes/.venv/bin/hermes";
const HERMES_PYTHON = "/opt/hermes/.venv/bin/python";
const NODE = "/usr/local/bin/node";
// The Product MCP runs behind the same context proxy the customer runtime
// uses, from the same stable image path. The Admin MCP stays a direct dist
// entrypoint — two distinct servers on one profile.
const MCP_CONTEXT_PROXY = "/usr/local/lib/sellable-agent/mcp-context-proxy.mjs";
const MCP_TURN_TRUST_ROOT =
"/usr/local/share/sellable-agent/mcp-turn-trust-root.pem";
const ADMIN_INSTALL_PACKAGE_ROOT = join(
ARTIFACT_ROOT,
"node_modules",
"@sellable",
"admin-install"
);
const ADMIN_INSTALL_ASSET_ROOT = join(ADMIN_INSTALL_PACKAGE_ROOT, "runtime");
const ADMIN_INSTALL_CLI = join(
ADMIN_INSTALL_PACKAGE_ROOT,
"bin",
"sellable-admin-install.mjs"
);
// The installer materializes these four dependencies next to the runtime tree
// in `--source package` mode; the sealed image already carries the exact
// locked copies, so the runtime reuses them instead of reaching the registry.
const ADMIN_INSTALL_DEPENDENCIES = Object.freeze([
"libsodium",
"libsodium-wrappers",
"playwright-core",
"yaml",
]);
const ADMIN_SETTINGS_FILES = Object.freeze([
".env",
"settings.json",
"config-state.json",
]);
const ADMIN_HOME_BINARIES = Object.freeze([
"grain",
"grain-lightfield-sync",
"lightfield",
"notion-browser",
"notion-profile-sync",
"notion-publish",
"sellable-admin-install",
"slack-pp-cli",
"with-sellable-admin-env",
]);
// Codex shims and the Lightfield SDK virtualenv are the installer's only
// symlink-bearing outputs. They live outside the profile home because
// mergeHermesState refuses to merge a profile that contains any symlink.
const ADMIN_CODEX_ROOT = join(RUNTIME_ROOT, "codex");
const ADMIN_LIGHTFIELD_VENV = join(RUNTIME_ROOT, "venvs", "lightfield");
const SYSTEM_PATH =
"/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin";
const RECEIPT_PATH =
process.env.SELLABLE_AGENT_ADMIN_CANARY_RECEIPT_PATH ??
"/run/sellable-agent/admin-canary/receipt.json";
const TIMEOUT_MS = 30_000;
const RUNTIME_ID = process.env.SELLABLE_RUNTIME_ID ?? "";
const ORIGIN = process.env.SELLABLE_AGENT_CONTROL_ORIGIN ?? "";
const AUDIENCE = process.env.SELLABLE_WORKLOAD_AUDIENCE ?? "";
const RUNTIME_GENERATION = Number(
process.env.SELLABLE_AGENT_RUNTIME_GENERATION ?? "0"
);
const BOOT_SESSION_ID = randomUUID();
const DEFAULT_PROFILE_BUNDLE_ROOT =
"/usr/local/share/sellable-agent/default-profile-bundle";
const DEFAULT_PROFILE_CRON_PYTHON = [
"import json,sys",
"from cron.jobs import create_job, update_job",
"value=json.loads(sys.argv[1])",
"job_id=value.pop('jobId', None)",
"enabled=value.pop('enabled')",
"result=update_job(job_id, {**value, 'enabled': enabled}) if job_id else create_job(prompt='', **value)",
"print(json.dumps({'id': result.get('id') if result else None}))",
].join("\n");
const sha256 = (value) => createHash("sha256").update(value).digest("hex");
const deliveryClaimId = (operationId) =>
sha256(`sellable-admin-secret-delivery-claim/v1\0${operationId}`);
const sleep = (milliseconds) =>
new Promise((resolve) => setTimeout(resolve, milliseconds));
function adminDefaultProfileCronAdapter(profileId, profileRoot) {
const mutate = ({ desired, spec, existing }) => {
const child = spawnSync(
HERMES_PYTHON,
[
"-c",
DEFAULT_PROFILE_CRON_PYTHON,
JSON.stringify({
name: desired.name,
schedule: spec.schedule,
script: desired.script,
no_agent: true,
deliver: "local",
enabled: true,
...(existing?.id ? { jobId: existing.id } : {}),
}),
],
{
env: {
...process.env,
HOME: DATA_ROOT,
HERMES_HOME: profileRoot,
HERMES_PROFILE: profileId,
NO_COLOR: "1",
},
encoding: "utf8",
timeout: 120_000,
maxBuffer: 64 * 1024,
}
);
return {
exitCode: child.status,
signal: child.signal,
stdout: child.stdout ?? "",
stderr: child.stderr ?? "",
timedOut: Boolean(child.error && child.signal === "SIGTERM"),
outputLimitExceeded: child.error?.code === "ENOBUFS",
};
};
return {
create: (input) => mutate(input),
update: (input) => mutate(input),
bind: (input) =>
bindDefaultProfileProbeCron({ ...input, bootId: BOOT_SESSION_ID }),
};
}
function adminMaterializerReceipt(profileId, profileRoot) {
return buildDefaultProfileMaterializerReceipt({
profileId,
profileRoot,
materializerReceipts: ["SOUL.md", "skills/sellable-admin/SKILL.md"],
});
}
function canonicalJson(value) {
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
if (value && typeof value === "object") {
return `{${Object.keys(value)
.sort()
.map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`)
.join(",")}}`;
}
return JSON.stringify(value);
}
function privateDirectory(path) {
mkdirSync(path, { recursive: true, mode: 0o700 });
chmodSync(path, 0o700);
const stat = lstatSync(path);
if (
stat.isSymbolicLink() ||
!stat.isDirectory() ||
(stat.mode & 0o077) !== 0
) {
throw new Error("admin_runtime_private_directory_rejected");
}
}
function atomicPrivateFile(path, bytes) {
privateDirectory(dirname(path));
const temporary = `${path}.${process.pid}.${randomUUID()}.tmp`;
writeFileSync(temporary, bytes, { flag: "wx", mode: 0o600 });
chmodSync(temporary, 0o600);
renameSync(temporary, path);
chmodSync(path, 0o600);
}
function installAdminHermesHostedPolicy(profileRoot) {
const configPath = join(profileRoot, "config.yaml");
const config = parseDocument(readFileSync(configPath, "utf8"), {
prettyErrors: false,
});
if (config.errors.length > 0) {
throw new Error("admin_runtime_hermes_config_rejected");
}
const value = config.toJSON();
if (!value || typeof value !== "object" || Array.isArray(value)) {
throw new Error("admin_runtime_hermes_config_rejected");
}
if (value.hooks) {
throw new Error("admin_runtime_hermes_hook_inventory_rejected");
}
const hosted = applyHermesSellableHostedPolicy(value);
config.set("memory", hosted.memory);
config.set("skills", hosted.skills);
config.set("onboarding", hosted.onboarding);
config.set("cron", hosted.cron);
config.set("hooks", HERMES_SNAPSHOT_HOOKS);
atomicPrivateFile(configPath, String(config));
atomicPrivateFile(
join(profileRoot, "shell-hooks-allowlist.json"),
HERMES_SNAPSHOT_HOOK_ALLOWLIST_BYTES
);
}
// The installed Admin layout is the exact shape `sellable-admin-install`
// produces for a Hermes profile: `<profileHome>/.sellable-admin` holds the
// generated CLI wrappers, the runtime tree, and the working copies of the
// delivered settings generation.
export function adminHomeLayout(profileHome) {
const adminHome = join(profileHome, ".sellable-admin");
const adminRoot = join(adminHome, "runtime", "sellable-admin");
const binRoot = join(adminHome, "bin");
return {
profileHome,
adminHome,
adminRoot,
binRoot,
configPath: join(adminHome, "settings.json"),
statePath: join(adminHome, "config-state.json"),
envPath: join(adminHome, ".env"),
lightfieldBin: join(binRoot, "lightfield"),
path: [
// Credential-aware Admin wrappers must resolve before their raw external
// binaries. `slack-pp-cli` in binRoot loads the profile's reviewed token;
// the image-owned .local binary is only its underlying executable.
binRoot,
join(profileHome, ".local", "bin"),
join(profileHome, ".slack", "bin"),
join(profileHome, ".venvs", "browser-use", "bin"),
"/opt/hermes/bin",
"/opt/hermes/.venv/bin",
SYSTEM_PATH,
].join(":"),
};
}
function adminToolEnv(layout) {
return {
PATH: layout.path,
SELLABLE_ADMIN_HOME: layout.adminHome,
SELLABLE_ADMIN_CONFIG_PATH: layout.configPath,
SELLABLE_ADMIN_STATE_PATH: layout.statePath,
SELLABLE_ADMIN_ROOT: layout.adminRoot,
SELLABLE_ADMIN_LIGHTFIELD_BIN: layout.lightfieldBin,
SELLABLE_ADMIN_LIGHTFIELD_VENV: ADMIN_LIGHTFIELD_VENV,
};
}
// The delivered generation directory stays the immutable proof subject; the
// Admin home keeps working copies exactly like an installer-managed host.
function deliverAdminSettings(settingsRoot, layout) {
const delivered = ADMIN_SETTINGS_FILES.map((name) => {
const path = join(settingsRoot, name);
if (
!existsSync(path) ||
lstatSync(path).isSymbolicLink() ||
!lstatSync(path).isFile()
) {
throw new Error("admin_runtime_settings_generation_rejected");
}
return { name, bytes: readFileSync(path) };
});
privateDirectory(layout.adminHome);
for (const { name, bytes } of delivered) {
if (name !== "config-state.json") {
atomicPrivateFile(join(layout.adminHome, name), bytes);
}
}
return prepareAdminStateGeneration({
deliveredStatePath: join(settingsRoot, "config-state.json"),
liveStatePath: layout.statePath,
});
}
function assertNoSymlinks(root, code) {
for (const entry of readdirSync(root, { withFileTypes: true })) {
if (entry.isSymbolicLink()) throw new Error(code);
if (entry.isDirectory()) assertNoSymlinks(join(root, entry.name), code);
}
}
// Staged so the promotion rename publishes the runtime tree atomically with
// the rest of the profile.
function stageAdminRuntimeTree(stageRoot) {
const stagedRoot = join(
stageRoot,
".sellable-admin",
"runtime",
"sellable-admin"
);
rmSync(stagedRoot, { recursive: true, force: true });
mkdirSync(dirname(stagedRoot), { recursive: true, mode: 0o700 });
cpSync(ADMIN_INSTALL_ASSET_ROOT, stagedRoot, { recursive: true });
for (const name of ADMIN_INSTALL_DEPENDENCIES) {
const target = join(stagedRoot, "node_modules", name);
rmSync(target, { recursive: true, force: true });
mkdirSync(dirname(target), { recursive: true });
cpSync(join(ARTIFACT_ROOT, "node_modules", name), target, {
recursive: true,
});
}
assertNoSymlinks(stagedRoot, "admin_runtime_admin_tree_symlink_rejected");
return stagedRoot;
}
// The published `bin/sellable-admin-install` shim resolves the installer from
// the registry at call time, which a sealed runtime must never do. The image
// already carries the exact locked installer, so the shim executes that copy.
function writeAdminInstallShim(layout) {
mkdirSync(layout.binRoot, { recursive: true });
const path = join(layout.binRoot, "sellable-admin-install");
writeFileSync(
path,
`#!/usr/bin/env sh\nexec ${NODE} ${ADMIN_INSTALL_CLI} "$@"\n`,
{ mode: 0o755 }
);
chmodSync(path, 0o755);
return path;
}
function installAdminHome({ settingsRoot, layout }) {
deliverAdminSettings(settingsRoot, layout);
if (!existsSync(join(layout.adminRoot, "scripts", "install-local-mcp.mjs"))) {
throw new Error("admin_runtime_admin_tree_missing");
}
privateDirectory(ADMIN_CODEX_ROOT);
writeAdminInstallShim(layout);
// `SELLABLE_ADMIN_CLI_WRAPPERS_ONLY` is the installer's own Hermes path: it
// writes bin/ plus install.json from local bytes and performs no network,
// registry, pip, or npm work.
const runInstaller = () =>
spawnSync(
NODE,
[join(layout.adminRoot, "scripts", "install-local-mcp.mjs")],
{
cwd: layout.adminRoot,
env: {
...adminToolEnv(layout),
HOME: layout.profileHome,
CODEX_HOME: ADMIN_CODEX_ROOT,
SELLABLE_ADMIN_INSTALL_HOST: "hermes",
SELLABLE_ADMIN_CLI_WRAPPERS_ONLY: "1",
SELLABLE_ADMIN_EMPLOYEE_INSTALL: "1",
SELLABLE_ADMIN_MCP_MODE: "package",
SELLABLE_ADMIN_MCP_PACKAGE_SPEC:
process.env.SELLABLE_AGENT_ADMIN_MCP_PACKAGE ?? "",
SELLABLE_ADMIN_INSTALL_PACKAGE_SPEC:
process.env.SELLABLE_AGENT_ADMIN_INSTALL_PACKAGE ?? "",
NO_COLOR: "1",
},
encoding: "utf8",
timeout: TIMEOUT_MS,
}
);
let generated = runInstaller();
if (generated.stderr?.includes("admin_state_receipt_state_mismatch")) {
recoverAdminStateReceiptMismatch({
deliveredStatePath: join(settingsRoot, "config-state.json"),
liveStatePath: layout.statePath,
failureCode: "admin_state_receipt_state_mismatch",
});
generated = runInstaller();
}
let summary;
try {
summary = JSON.parse(generated.stdout);
} catch {
const exactStateFailure = generated.stderr?.match(
/admin_state_[a-z0-9_]+/
)?.[0];
if (exactStateFailure) throw new Error(exactStateFailure);
throw new Error("admin_runtime_admin_home_wrappers_rejected");
}
// The sealed image carries the checksum-pinned Printing Press binary, so
// every wrapper generated from local bytes must be immediately usable.
if (
generated.error ||
summary?.adminHome !== layout.adminHome ||
!Array.isArray(summary.wrappers) ||
summary.wrappers.length < 1 ||
summary.wrappers.some((wrapper) => wrapper?.ready !== true)
) {
throw new Error("admin_runtime_admin_home_wrappers_rejected");
}
completeAdminStateReceiptRecovery({ liveStatePath: layout.statePath });
for (const name of ADMIN_HOME_BINARIES) {
const stat = lstatSync(join(layout.binRoot, name));
if (stat.isSymbolicLink() || !stat.isFile() || (stat.mode & 0o111) === 0) {
throw new Error("admin_runtime_admin_home_binary_rejected");
}
}
const install = JSON.parse(
readFileSync(join(layout.adminHome, "install.json"), "utf8")
);
if (
install.adminHome !== layout.adminHome ||
install.adminRoot !== layout.adminRoot
) {
throw new Error("admin_runtime_admin_home_install_rejected");
}
for (const name of ADMIN_SETTINGS_FILES) {
const stat = lstatSync(join(layout.adminHome, name));
if (stat.isSymbolicLink() || !stat.isFile() || (stat.mode & 0o077) !== 0) {
throw new Error("admin_runtime_admin_home_settings_rejected");
}
}
return {
home: layout.adminHome,
root: layout.adminRoot,
binaries: [...ADMIN_HOME_BINARIES],
settings: [...ADMIN_SETTINGS_FILES],
marketplaceRoot: install.marketplaceRoot,
pluginCacheRoot: install.pluginCacheRoot,
};
}
async function installedPackage(packageName) {
const path = `${ARTIFACT_ROOT}/node_modules/${packageName}/package.json`;
const value = JSON.parse(await readFile(path, "utf8"));
return { name: value.name, version: value.version };
}
function requestedVersion(spec) {
const marker = spec.lastIndexOf("@");
if (marker <= 0 || marker === spec.length - 1)
throw new Error("admin_runtime_package_spec_invalid");
return spec.slice(marker + 1);
}
function packageSpecs() {
return {
"@sellable/mcp": process.env.SELLABLE_AGENT_PRODUCT_MCP_PACKAGE,
"@sellable/install": process.env.SELLABLE_AGENT_PRODUCT_INSTALL_PACKAGE,
"@sellable/admin-mcp": process.env.SELLABLE_AGENT_ADMIN_MCP_PACKAGE,
"@sellable/admin-install": process.env.SELLABLE_AGENT_ADMIN_INSTALL_PACKAGE,
};
}
async function proveInstalledPackages() {
const packages = {};
for (const [name, spec] of Object.entries(packageSpecs())) {
if (!spec) throw new Error("admin_runtime_package_identity_missing");
const installed = await installedPackage(name);
if (
installed.name !== name ||
installed.version !== requestedVersion(spec)
) {
throw new Error("admin_runtime_package_identity_mismatch");
}
packages[name] = installed;
}
return packages;
}
async function inspectMcp({
alias,
command = "/usr/local/bin/node",
args,
env,
packageProof,
probeCall = null,
}) {
const child = spawn(command, args, {
env: { ...process.env, ...env },
stdio: ["pipe", "pipe", "pipe"],
});
let stderr = "";
child.stderr.on("data", (chunk) => {
stderr = `${stderr}${chunk}`.slice(-4096);
});
const lines = createInterface({ input: child.stdout });
const pending = new Map();
lines.on("line", (line) => {
try {
const message = JSON.parse(line);
if (message.id != null && pending.has(message.id)) {
pending.get(message.id)(message);
pending.delete(message.id);
}
} catch {
// The JSON-RPC proof fails closed on timeout or an invalid response.
}
});
const request = (id, method, params = {}) =>
new Promise((resolve, reject) => {
const timer = setTimeout(() => {
pending.delete(id);
reject(new Error(`admin_runtime_${alias}_${method}_timeout`));
}, TIMEOUT_MS);
pending.set(id, (message) => {
clearTimeout(timer);
if (message.error) {
reject(new Error(`admin_runtime_${alias}_${method}_failed`));
} else {
resolve(message.result);
}
});
child.stdin.write(
`${JSON.stringify({ jsonrpc: "2.0", id, method, params })}\n`
);
});
try {
const initialized = await request(1, "initialize", {
protocolVersion: "2024-11-05",
capabilities: {},
clientInfo: { name: "sellable-admin-runtime", version: "1" },
});
child.stdin.write(
`${JSON.stringify({
jsonrpc: "2.0",
method: "notifications/initialized",
params: {},
})}\n`
);
const listed = await request(2, "tools/list");
const tools = Array.isArray(listed?.tools)
? listed.tools
.map((tool) => tool?.name)
.filter(Boolean)
.sort()
: [];
if (tools.length === 0)
throw new Error(`admin_runtime_${alias}_inventory_empty`);
const safeRead = probeCall
? await request(3, "tools/call", probeCall)
: null;
return {
alias,
packageName: packageProof.name,
packageVersion: packageProof.version,
packageIntegrity: packageProof.integrity,
serverInfoName: initialized?.serverInfo?.name ?? "",
serverInfoVersion: initialized?.serverInfo?.version ?? "",
toolCount: tools.length,
toolInventoryDigest: sha256(JSON.stringify(tools)),
tools,
...(safeRead === null ? {} : { safeRead }),
};
} catch (error) {
throw new Error(
`${error instanceof Error ? error.message : "admin_runtime_mcp_failed"}:${sha256(stderr)}`
);
} finally {
child.kill("SIGTERM");
lines.close();
}
}
async function proveAdminProductMcpReadiness({ profile, claim, installed }) {
const slack = claim?.runtimeContract?.slack;
const channelId = slack?.selectedBindings?.[0]?.channelId;
const requesterId = slack?.memberAllowlist?.[0];
const packageProof = profile?.stage?.packages?.product;
if (
!/^[A-Za-z0-9_-]{1,128}$/.test(channelId ?? "") ||
!/^[A-Za-z0-9_-]{1,128}$/.test(requesterId ?? "") ||
!packageProof ||
typeof installed?.productConfigPath !== "string"
) {
throw new Error("admin_product_mcp_service_context_rejected");
}
try {
const layout = adminHomeLayout(profile.profileHome);
const proof = await inspectMcp({
alias: "sellable",
command: NODE,
args: [MCP_CONTEXT_PROXY],
env: {
HOME: profile.profileHome,
HERMES_HOME: DATA_ROOT,
...adminAgentServiceEnvironment(claim),
...productMcpContextProxyEnvironment(layout),
SELLABLE_CONFIG_PATH: installed.productConfigPath,
},
packageProof,
probeCall: {
name: "get_company_info",
arguments: {},
_meta: {
sellableAgent: {
channelId,
requesterId: claim.runtimeContract.slack.memberAllowlist[0],
providerRequestId: `admin-runtime-readiness-${claim.operation.id}`,
},
},
},
});
if (
!proof.safeRead ||
typeof proof.safeRead !== "object" ||
proof.safeRead.isError === true
) {
throw new Error("admin_product_mcp_service_context_rejected");
}
return Object.freeze({
authenticated: true,
safeRead: true,
packageName: proof.packageName,
packageVersion: proof.packageVersion,
toolInventoryDigest: proof.toolInventoryDigest,
});
} catch {
throw new Error("admin_product_mcp_service_context_rejected");
}
}
function exactSoul(claim) {
const artifacts = claim?.revision?.artifacts?.filter(
(artifact) =>
artifact?.type === "SOUL" && typeof artifact.safeContent === "string"
);
if (artifacts?.length !== 1) throw new Error("admin_runtime_soul_missing");
const soul = Buffer.from(artifacts[0].safeContent, "utf8");
if (
soul.byteLength !== artifacts[0].sizeBytes ||
sha256(soul) !== artifacts[0].contentHash
) {
throw new Error("admin_runtime_soul_digest_rejected");
}
return { bytes: soul, sha256: sha256(soul), byteLength: soul.byteLength };
}
// The context proxy fails closed at startup unless its descriptor roots
// exist as private directories, its trust root file matches the fingerprint
// it is handed, and its downstream command is absolute. The materializer owns
// every one of those inputs so the `sellable` server can never enter a
// fail-closed respawn loop from missing environment.
function ensureProductMcpContextProxyRoots() {
privateDirectory(MCP_TURN_CONTEXT_ROOT);
privateDirectory(join(MCP_TURN_CONTEXT_ROOT, "consumed"));
}
function readMcpTurnTrustRoot() {
let trustRootPem;
try {
const entry = lstatSync(MCP_TURN_TRUST_ROOT);
if (
entry.isSymbolicLink() ||
!entry.isFile() ||
(entry.mode & 0o022) !== 0
) {
throw new Error("admin_runtime_trust_root_rejected");
}
trustRootPem = readFileSync(MCP_TURN_TRUST_ROOT, "utf8");
} catch {
throw new Error("admin_runtime_trust_root_rejected");
}
return trustRootPem;
}
function productMcpContextProxyEnvironment(layout) {
const trustRootPem = readMcpTurnTrustRoot();
ensureProductMcpContextProxyRoots();
return {
SELLABLE_AGENT_TURN_DESCRIPTOR_PATH: join(
MCP_TURN_CONTEXT_ROOT,
"turn-context.json"
),
SELLABLE_AGENT_TURN_CONSUMED_ROOT: join(MCP_TURN_CONTEXT_ROOT, "consumed"),
SELLABLE_AGENT_MCP_RUNTIME_BINDING_PATH: join(
layout.profileHome,
"sellable",
"runtime-binding.json"
),
SELLABLE_AGENT_MCP_TRUST_ROOT_PATH: MCP_TURN_TRUST_ROOT,
SELLABLE_AGENT_MCP_TRUST_ROOT_FINGERPRINT: sha256(trustRootPem),
SELLABLE_AGENT_MCP_DOWNSTREAM_COMMAND: NODE,
SELLABLE_AGENT_MCP_DOWNSTREAM_ARGS_JSON: JSON.stringify([
`${ARTIFACT_ROOT}/node_modules/@sellable/mcp/dist/index.js`,
]),
};
}
// Inventory proof must exercise the real context proxy, but it must not share
// the live gateway's turn descriptor. A live endpoint run intentionally
// projects `tools/list` to its signed allowlist; treating that narrow view as
// package drift makes profile materialization race with ordinary Hermes work.
// Give each proof an unguessable private, descriptor-free context root instead.
function productMcpInventoryProofEnvironment() {
const root = mkdtempSync(join(SECRET_ROOT, "mcp-inventory-proof-"));
privateDirectory(root);
privateDirectory(join(root, "consumed"));
const configPath = join(root, "sellable-config.json");
atomicPrivateFile(
configPath,
`${JSON.stringify({
apiUrl: "https://example.invalid",
token: "skt_live_admin_inventory_proof_nonsecret",
})}\n`
);
return {
root,
env: {
SELLABLE_AGENT_TURN_DESCRIPTOR_PATH: join(root, "turn-context.json"),
SELLABLE_AGENT_TURN_CONSUMED_ROOT: join(root, "consumed"),
SELLABLE_CONFIG_PATH: configPath,
SELLABLE_AGENT_SERVICE_CREDENTIAL_FILE: "",
// This disposable subprocess only inventories package tools with a fake
// credential. It has neither the gateway identity nor live authority.
SELLABLE_AGENT_MANAGED_AUTH_PATH: "",
SELLABLE_AGENT_RUNTIME: "",
SELLABLE_AGENT_SERVER_AUTHENTICATED_KIND: "",
},
};
}
function profilePackageReceipt(
profileContract,
alias,
role,
layout,
agentServiceEnv = {}
) {
const entry = profileContract.packages[alias][role];
return {
...entry,
command: "/usr/local/bin/node",
args:
alias === "sellable"
? [MCP_CONTEXT_PROXY]
: [`${ARTIFACT_ROOT}/node_modules/@sellable/admin-mcp/dist/index.js`],
// The Admin MCP resolves its packaged gateway scripts through
// SELLABLE_ADMIN_ROOT and its CLI shell-outs through PATH, so the server
// entry carries them instead of relying on gateway inheritance.
...(alias === "sellable-admin" ? { env: adminToolEnv(layout) } : {}),
...(alias === "sellable"
? {
env: {
...agentServiceEnv,
...productMcpContextProxyEnvironment(layout),
},
}
: {}),
};
}
function installAdminMcpRuntimeBinding(
profileRoot,
profileId,
agentServiceEnv
) {
const serviceCredentialGeneration = Number(
agentServiceEnv.SELLABLE_AGENT_CREDENTIAL_GENERATION
);
const binding = createMcpAdminRuntimeBindingV3({
workspaceId: agentServiceEnv.SELLABLE_AGENT_WORKSPACE_ID,
agentId: agentServiceEnv.SELLABLE_AGENT_ID,
runtimeId: RUNTIME_ID,
profileId,
serviceCredentialGeneration,
runtimeGeneration: RUNTIME_GENERATION,
});
atomicPrivateFile(
join(profileRoot, "sellable", "runtime-binding.json"),
`${JSON.stringify(binding, null, 2)}\n`
);
}
// The profile's `sellable` MCP server exactly as Hermes runs it, so a Claude
// Code tool-lane run sees the same tools, through the same context proxy.
function profileSellableMcpServer(profileRoot) {
const path = join(profileRoot, "config.yaml");
const stat = lstatSync(path);
if (stat.isSymbolicLink() || !stat.isFile()) return null;
const document = parseDocument(readFileSync(path, "utf8"));
if (document.errors.length > 0) return null;
const server = document.toJS().mcp_servers?.sellable;
if (
server?.command !== NODE ||
JSON.stringify(server.args) !== JSON.stringify([MCP_CONTEXT_PROXY]) ||
!server.env ||
typeof server.env !== "object"
) {
return null;
}
return {
command: NODE,
args: [MCP_CONTEXT_PROXY],
env: Object.fromEntries(
Object.entries(server.env).map(([key, value]) => [key, String(value)])
),
};
}
function installAdminModelConfig(profileRoot, modelConfig) {
const path = join(profileRoot, "config.yaml");
const stat = lstatSync(path);
if (stat.isSymbolicLink() || !stat.isFile()) {
throw new Error("admin_model_config_rejected");
}
const document = parseDocument(readFileSync(path, "utf8"));
if (document.errors.length > 0) {
throw new Error("admin_model_config_rejected");
}
document.set("agent", modelConfig.agent);
document.set("providers", modelConfig.providers);
document.set("model", modelConfig.model);
document.set(
"credential_pool_strategies",
modelConfig.credential_pool_strategies
);
document.set("fallback_providers", modelConfig.fallback_providers);
if (document.get("provider_routing") === undefined) {
// OpenRouter provider selection defaults to highest throughput; an
// explicit admin config value wins.
document.set("provider_routing", { sort: "throughput" });
}
const currentAuxiliary = document.toJS().auxiliary ?? {};
if (!currentAuxiliary.vision) {
// Non-vision main models need a working vision auxiliary; the Nous
// portal has no credentials on these machines, so image input would
// silently fail without this default.
document.set("auxiliary", {
...currentAuxiliary,
vision: { provider: "openrouter", model: "google/gemini-3.5-flash" },
});
}
document.setIn(["platforms", "api_server"], {
enabled: true,
extra: { host: "127.0.0.1", port: 8642, cors_origins: [] },
});
atomicPrivateFile(path, document.toString());
const readback = parseDocument(readFileSync(path, "utf8")).toJS();
if (
canonicalJson(readback.agent) !== canonicalJson(modelConfig.agent) ||
canonicalJson(readback.providers) !==
canonicalJson(modelConfig.providers) ||
canonicalJson(readback.model) !== canonicalJson(modelConfig.model) ||
canonicalJson(readback.credential_pool_strategies) !==
canonicalJson(modelConfig.credential_pool_strategies) ||
canonicalJson(readback.fallback_providers) !==
canonicalJson(modelConfig.fallback_providers) ||
canonicalJson(readback.platforms?.api_server) !==
canonicalJson({
enabled: true,
extra: { host: "127.0.0.1", port: 8642, cors_origins: [] },
})
) {
throw new Error("admin_model_config_rejected");
}
}
const ADMIN_LIVE_SKILLS_CONFIG_KEYS = Object.freeze([
"disabled",
"platform_disabled",
]);
const ADMIN_LIVE_PLUGIN_CONFIG_KEYS = Object.freeze(["enabled", "disabled"]);
const ADMIN_SEALED_SKILL_PATHS = new Set([
// The package owns only the root routing skill. Customer/operator skills
// live below this namespace and must survive profile materialization.
"sellable/SKILL.md",
// The Admin profile factory owns this whole namespace. Preserving an old
// package version as mutable operator state makes every package upgrade
// restage forever when any Admin skill changes.
"sellable-admin",
]);
function adminPreservedSlackToolsets(config, profileRoot) {
return sellableSlackToolsetsForMaterialization(config, {
browserDefaultApplied: existsSync(
join(profileRoot, SELLABLE_BROWSER_TOOLSET_DEFAULT_MARKER)
),
});
}
function adminPreservedSkillsIdentity(profileRoot) {
if (!existsSync(profileRoot)) return null;
const entries = [];
const preservedRoots = ["skills", "plugins", "plugin-data"];
const configPath = join(profileRoot, "config.yaml");
// Admin settings are delivered into .sellable-admin before the Hermes
// profile is staged. That settings-only directory is not an existing live
// profile and therefore has no skills state to preserve.
if (
!preservedRoots.some((name) => existsSync(join(profileRoot, name))) &&
!existsSync(configPath)
)
return null;
const walk = (rootName, directory, relativePath = "") => {
for (const name of readdirSync(directory).sort()) {
const nextRelative = relativePath ? `${relativePath}/${name}` : name;
if (
rootName === "skills" &&
[...ADMIN_SEALED_SKILL_PATHS].some(
(path) => nextRelative === path || nextRelative.startsWith(`${path}/`)
)
)
continue;
const path = join(directory, name);
const stat = lstatSync(path);
if (stat.isSymbolicLink()) {
throw new Error("admin_runtime_live_state_symlink_rejected");
}
if (stat.isDirectory()) walk(rootName, path, nextRelative);
else if (stat.isFile()) {
entries.push({
path: `${rootName}/${nextRelative}`,
sha256: sha256(readFileSync(path)),
});
} else throw new Error("admin_runtime_live_state_entry_rejected");
}
};
for (const rootName of preservedRoots) {
const root = join(profileRoot, rootName);
if (existsSync(root)) walk(rootName, root);
}
const mutableConfig = {};
let slackToolsets = null;
if (existsSync(configPath)) {
const document = parseDocument(readFileSync(configPath, "utf8"));
if (document.errors.length > 0) {
throw new Error("admin_runtime_live_skills_config_rejected");
}
const config = document.toJS();
const skills = config?.skills;
if (skills && typeof skills === "object" && !Array.isArray(skills)) {
mutableConfig.skills = Object.fromEntries(
ADMIN_LIVE_SKILLS_CONFIG_KEYS.filter((key) =>
Object.prototype.hasOwnProperty.call(skills, key)
).map((key) => [key, skills[key]])
);
}
const plugins = config?.plugins;
if (plugins && (typeof plugins !== "object" || Array.isArray(plugins))) {
throw new Error("admin_runtime_live_plugins_config_rejected");
}
if (plugins) {
mutableConfig.plugins = Object.fromEntries(
ADMIN_LIVE_PLUGIN_CONFIG_KEYS.filter((key) =>
Object.prototype.hasOwnProperty.call(plugins, key)
).map((key) => [key, plugins[key]])
);
}
slackToolsets = adminPreservedSlackToolsets(config, profileRoot);
}
return sha256(canonicalJson({ mutableConfig, entries, slackToolsets }));
}
function mergeAdminLiveSkillsConfig(sourceRoot, targetRoot) {
const sourcePath = join(sourceRoot, "config.yaml");
const targetPath = join(targetRoot, "config.yaml");
if (!existsSync(sourcePath) || !existsSync(targetPath)) return;
const sourceDocument = parseDocument(readFileSync(sourcePath, "utf8"));
const targetDocument = parseDocument(readFileSync(targetPath, "utf8"));
if (sourceDocument.errors.length > 0 || targetDocument.errors.length > 0) {
throw new Error("admin_runtime_live_skills_config_rejected");
}
const sourceConfig = sourceDocument.toJS();
const sourceSkills = sourceConfig?.skills;
if (sourceSkills !== undefined) {
if (
!sourceSkills ||
typeof sourceSkills !== "object" ||
Array.isArray(sourceSkills)
) {
throw new Error("admin_runtime_live_skills_config_rejected");
}
for (const key of ADMIN_LIVE_SKILLS_CONFIG_KEYS) {
if (Object.prototype.hasOwnProperty.call(sourceSkills, key)) {
targetDocument.setIn(
["skills", key],
structuredClone(sourceSkills[key])
);
}
}
}
const sourcePlugins = sourceConfig?.plugins;
if (sourcePlugins !== undefined) {
if (
!sourcePlugins ||
typeof sourcePlugins !== "object" ||
Array.isArray(sourcePlugins)
) {
throw new Error("admin_runtime_live_plugins_config_rejected");
}
for (const key of ADMIN_LIVE_PLUGIN_CONFIG_KEYS) {
if (Object.prototype.hasOwnProperty.call(sourcePlugins, key)) {
targetDocument.setIn(
["plugins", key],
structuredClone(sourcePlugins[key])
);
}
}
}
const slackToolsets = sellableSlackToolsetsForMaterialization(sourceConfig, {
browserDefaultApplied: existsSync(
join(sourceRoot, SELLABLE_BROWSER_TOOLSET_DEFAULT_MARKER)
),
});
if (slackToolsets !== null)
targetDocument.setIn(["platform_toolsets", "slack"], slackToolsets);
atomicPrivateFile(targetPath, targetDocument.toString());
}
function installAdminBrowserToolsetDefault(sourceRoot, stageRoot) {
const configPath = join(stageRoot, "config.yaml");
const markerPath = join(stageRoot, SELLABLE_BROWSER_TOOLSET_DEFAULT_MARKER);
const document = parseDocument(readFileSync(configPath, "utf8"));
if (document.errors.length > 0) {
throw new Error("admin_runtime_live_skills_config_rejected");
}
document.setIn(
["platform_toolsets", "slack"],
sellableSlackToolsetsForMaterialization(document.toJS(), {
browserDefaultApplied: existsSync(
join(sourceRoot, SELLABLE_BROWSER_TOOLSET_DEFAULT_MARKER)
),
})
);
atomicPrivateFile(configPath, document.toString());
privateDirectory(dirname(markerPath));
atomicPrivateFile(markerPath, "enabled\n");
}
function installWritableAdminSlackChannelPolicy(stageRoot, slack) {
const envPath = join(stageRoot, ".env");
if (!existsSync(envPath)) {
throw new Error("admin_runtime_slack_environment_rejected");
}
const stat = lstatSync(envPath);
if (stat.isSymbolicLink() || !stat.isFile()) {
throw new Error("admin_runtime_slack_environment_rejected");
}
if (
!Array.isArray(slack?.selectedChannelIds) ||
slack.selectedChannelIds.length < 1 ||
!slack.selectedChannelIds.includes(slack.primaryChannel?.id)
) {
throw new Error("admin_runtime_slack_contract_rejected");
}
let allowedPolicyCount = 0;
let readOnlyPolicyCount = 0;
const updated = readFileSync(envPath, "utf8")
.split("\n")
.map((line) => {
if (line.startsWith("SLACK_ALLOWED_CHANNELS=")) {
allowedPolicyCount += 1;
return `SLACK_ALLOWED_CHANNELS=${slack.selectedChannelIds.join(",")}`;
}
if (line.startsWith("SLACK_READ_ONLY_CHANNELS=")) {
readOnlyPolicyCount += 1;
return "SLACK_READ_ONLY_CHANNELS=";
}
return line;
})
.join("\n");
if (allowedPolicyCount !== 1 || readOnlyPolicyCount !== 1) {
throw new Error("admin_runtime_slack_environment_rejected");
}
atomicPrivateFile(envPath, updated);
}
async function materializeAdminProfile({
profileContract,
settingsRoot,
rootAuthPath,
modelConfig,
productConfigPath,
soul,
slack = null,
customerProgram,
agentServiceEnv = {},
runPhase = async (_phase, work) => work(),
}) {
const profileHome = profileContract.profileHome;
const profile = basename(profileHome);
if (join(PROFILES_ROOT, profile) !== profileHome) {
throw new Error("admin_runtime_profile_home_rejected");
}
const stageRoot = join(PROFILES_ROOT, ".staging", profile);
const backupRoot = join(PROFILES_ROOT, ".backup", profile);
privateDirectory(PROFILES_ROOT);
const readAuthoritativeSoul = () => {
const activeSoulPath = join(profileHome, "SOUL.md");
if (!existsSync(activeSoulPath)) return soul;
const activeSoul = lstatSync(activeSoulPath);
if (activeSoul.isSymbolicLink() || !activeSoul.isFile()) {
throw new Error("admin_runtime_live_soul_rejected");
}
const bytes = readFileSync(activeSoulPath);
return {
bytes,
sha256: sha256(bytes),
byteLength: bytes.byteLength,
};
};
const layout = adminHomeLayout(profileHome);
// stageAdminProfile canonicalizes these two paths before staging, so the
// Admin home working copies must exist before the factory runs.
await runPhase("profile_settings_delivery", async () => {
deliverAdminSettings(settingsRoot, layout);
});
const serverProofs = {};
const stageWithAuthority = async (stagedAuthority) =>
runPhase("profile_stage", async () => {
const stage = await stageAdminProfile({
mode: slack ? "direct_soul_native_slack" : "direct_soul",
lifecycleAuthorized: slack ? true : undefined,
ephemeralProductConfig: Boolean(slack),
ephemeralSlackSecrets: Boolean(slack),
profile,
profilesRoot: PROFILES_ROOT,
stageRoot,
finalProfileRoot: profileHome,
rootAuthPath,
canonicalAdminConfigPath: layout.configPath,
canonicalAdminStatePath: layout.statePath,
productConfigPath,
// The pinned Admin installer validates this bootstrap template model.
// installAdminModelConfig below replaces it with the managed model
// (currently GPT-6.1 Sol high) before promotion or gateway startup.
model: "gpt-5.6-sol",
reasoningEffort: "high",
soul: stagedAuthority,
...(slack
? {
slackSecretFile: slack.secretFile,
teamId: slack.teamId,
primaryChannel: slack.primaryChannel,
secondaryReadChannelIds: slack.secondaryReadChannelIds,
allowedSlackUserIds: slack.allowedSlackUserIds,
slackApp: slack.slackApp,
}
: {}),
skillsRoot: join(ADMIN_INSTALL_ASSET_ROOT, "mcp", "skills"),
installRoot: layout.adminRoot,
adminHome: layout.adminHome,
installAdmin: async () =>
profilePackageReceipt(
profileContract,
"sellable-admin",
"mcp",
layout
),
installProduct: async () =>
profilePackageReceipt(
profileContract,
"sellable",
"mcp",
layout,
agentServiceEnv
),
verifyMcp: async ({ kind, profileRoot, config }) => {
const alias = kind === "admin" ? "sellable-admin" : "sellable";
const server = config.mcp_servers[alias];
const inventoryProof =
alias === "sellable" ? productMcpInventoryProofEnvironment() : null;
let proof;
try {
proof = await inspectMcp({
alias,
command: server.command,
args: server.args,
env: {
HOME: profileRoot,
HERMES_HOME: profileRoot,
...adminToolEnv(layout),
...agentServiceEnv,
SELLABLE_CONFIG_PATH: productConfigPath,
...(server.env ?? {}),
...(inventoryProof?.env ?? {}),
},
packageProof: profileContract.packages[alias].mcp,
});
} finally {
if (inventoryProof) {
rmSync(inventoryProof.root, { recursive: true, force: true });
}
}
if (
JSON.stringify(proof.tools) !==
JSON.stringify(profileContract.tools[alias])
) {
throw new Error("admin_runtime_tool_inventory_mismatch");
}
serverProofs[alias] = proof;
return { ok: true, ...proof };
},
});
if (slack) installWritableAdminSlackChannelPolicy(stageRoot, slack);
installAdminModelConfig(stageRoot, modelConfig);
installAdminHermesHostedPolicy(stageRoot);
stageAdminRuntimeTree(stageRoot);
installAdminMcpRuntimeBinding(stageRoot, profile, agentServiceEnv);
return stage;
});
let stagedAuthority = withProfileSoulLock(profile, readAuthoritativeSoul, {
dataRoot: DATA_ROOT,
});
let stage = null;
let promoted = null;
for (let attempt = 0; attempt < 2 && !promoted; attempt += 1) {
stage = await stageWithAuthority(stagedAuthority);
const committed = await runPhase("profile_live_merge_promote", async () =>
withProfileSoulLock(
profile,
async () => {
const finalAuthority = readAuthoritativeSoul();
if (
finalAuthority.sha256 !== stagedAuthority.sha256 ||
!finalAuthority.bytes.equals(stagedAuthority.bytes)
) {
rmSync(stageRoot, { recursive: true, force: true });
return { status: "RESTAGE", stagedAuthority: finalAuthority };
}
const liveSkillsIdentity = adminPreservedSkillsIdentity(profileHome);
if (liveSkillsIdentity !== null) {
replaceHermesState(
join(profileHome, "skills"),
join(stageRoot, "skills"),
{
preserveTargetPaths: [...ADMIN_SEALED_SKILL_PATHS],
}
);
}
// The gateway and endpoint relay are stopped before materialization.
// Reuse durable inodes on this volume, as Customer profiles do, so a
// large conversation DB cannot exhaust disk during an upgrade.
linkHermesState(profileHome, stageRoot, {
skipSourceTopLevelNames:
liveSkillsIdentity === null ? [] : ["skills"],
});
mergeAdminLiveSkillsConfig(profileHome, stageRoot);
installAdminBrowserToolsetDefault(profileHome, stageRoot);
if (
liveSkillsIdentity !== null &&
adminPreservedSkillsIdentity(stageRoot) !== liveSkillsIdentity
) {
rmSync(stageRoot, { recursive: true, force: true });
return { status: "RESTAGE", stagedAuthority: finalAuthority };
}
const stagedSoulPath = join(stageRoot, "SOUL.md");
const stagedSoul = lstatSync(stagedSoulPath);
const stagedSoulBytes = readFileSync(stagedSoulPath);
if (
stagedSoul.isSymbolicLink() ||
!stagedSoul.isFile() ||
!stagedSoulBytes.equals(finalAuthority.bytes)
) {
throw new Error("admin_runtime_staged_soul_rejected");
}
const promotion = await promoteAdminProfileAtomically({
profile,
currentRoot: profileHome,
stagedRoot: stageRoot,
backupRoot,
});
const finalSoulPath = join(profileHome, "SOUL.md");
const finalSoul = lstatSync(finalSoulPath);
const finalSoulBytes = readFileSync(finalSoulPath);
if (
finalSoul.isSymbolicLink() ||
!finalSoul.isFile() ||
finalSoul.uid !== (process.getuid?.() ?? finalSoul.uid) ||
finalSoul.gid !== (process.getgid?.() ?? finalSoul.gid) ||
!finalSoulBytes.equals(finalAuthority.bytes)
) {
throw new Error("admin_runtime_live_soul_readback_rejected");
}
chmodSync(finalSoulPath, 0o600);
if ((lstatSync(finalSoulPath).mode & 0o777) !== 0o600) {
throw new Error("admin_runtime_live_soul_mode_rejected");
}
return {
status: "PROMOTED",
promotion,
observedSoulRawSha256: sha256(finalSoulBytes),
};
},
{ dataRoot: DATA_ROOT }
)
);
if (committed.status === "PROMOTED") {
promoted = committed;
} else {
stagedAuthority = committed.stagedAuthority;
}
}
if (!promoted || !stage) {
throw new Error("admin_runtime_live_soul_retry_required");
}
// The installer bakes absolute paths into every generated wrapper, so bin/
// is generated once the promoted Admin home is at its final path.
const {
adminHome,
memoryCron,
opportunitySequencesCron,
clientRelationshipsCrons,
customerProgramCrons,
adminRefillCron,
} = await runPhase("profile_crons", async () => {
const adminHome = installAdminHome({ settingsRoot, layout });
const memoryCron = await ensureHermesSnapshotCron({
profileId: profile,
profileRoot: profileHome,
hermesRoot: DATA_ROOT,
hermesCli: HERMES,
});
if (!memoryCron.ok) throw new Error(memoryCron.code);
const opportunitySequencesCron = await ensureHermesOpportunitySequencesCron(
{
profileId: profile,
profileRoot: profileHome,
hermesRoot: DATA_ROOT,
statePath: layout.statePath,
}
);
// The production Admin profile always has native Slack authority. The
// recovery canary intentionally materializes a direct-SOUL profile with no
// Slack identity, so it must not invent customer delivery jobs.
const customerProgramCrons = slack
? await ensureHermesCustomerProgramCrons({
profileId: profile,
profileRoot: profileHome,
hermesRoot: DATA_ROOT,
workspaceId: agentServiceEnv.SELLABLE_AGENT_WORKSPACE_ID,
channelId: slack.primaryChannel.id,
timezone: customerProgram.timezone,
apiUrl: ORIGIN,
authorityPath: managedAuthPath(profile),
})
: { ok: true, status: "SKIPPED", jobs: [] };
if (!customerProgramCrons.ok) throw new Error(customerProgramCrons.code);
// NON-FATAL like its siblings: a missed refill tick is harmless (refill
// no-ops when queues are full) and must not block an Admin promotion.
const adminRefillCron = slack
? await ensureHermesRefillSendsCron({
profileId: profile,
profileRoot: profileHome,
hermesRoot: DATA_ROOT,
workspaceId: agentServiceEnv.SELLABLE_AGENT_WORKSPACE_ID,
})
: { ok: true, status: "SKIPPED" };
// Ordered after its sibling and NON-FATAL: an Admin runtime must not fail
// to promote because one client's digest cron could not be written.
const clientRelationshipsCrons = await ensureHermesClientRelationshipsCrons(
{
profileId: profile,
profileRoot: profileHome,
hermesRoot: DATA_ROOT,
statePath: layout.statePath,
}
);
return {
adminHome,
memoryCron,
opportunitySequencesCron,
clientRelationshipsCrons,
customerProgramCrons,
adminRefillCron,
};
});
return {
stage,
promotion: promoted.promotion,
profile,
profileHome,
adminHome,
memoryCron,
opportunitySequencesCron,
clientRelationshipsCrons,
customerProgramCrons,
adminRefillCron,
observedSoulRawSha256: promoted.observedSoulRawSha256,
servers: [serverProofs.sellable, serverProofs["sellable-admin"]].map(
({ tools: _tools, ...proof }) => proof
),
};
}
function loadOrCreateDeliveryKey() {
const path = join(DELIVERY_ROOT, "x25519.json");
privateDirectory(DELIVERY_ROOT);
if (!existsSync(path)) {
const pair = sodium.crypto_box_keypair();
atomicPrivateFile(
path,
`${JSON.stringify({
publicKey: Buffer.from(pair.publicKey).toString("base64url"),
privateKey: Buffer.from(pair.privateKey).toString("base64url"),
})}\n`
);
}
const stat = lstatSync(path);
if (stat.isSymbolicLink() || !stat.isFile() || (stat.mode & 0o077) !== 0) {
throw new Error("admin_runtime_delivery_key_rejected");
}
const parsed = JSON.parse(readFileSync(path, "utf8"));
const publicKey = Buffer.from(parsed.publicKey ?? "", "base64url");
const privateKey = Buffer.from(parsed.privateKey ?? "", "base64url");
if (
publicKey.byteLength !== sodium.crypto_box_PUBLICKEYBYTES ||
privateKey.byteLength !== sodium.crypto_box_SECRETKEYBYTES ||
publicKey.toString("base64url") !== parsed.publicKey ||
privateKey.toString("base64url") !== parsed.privateKey
) {
throw new Error("admin_runtime_delivery_key_rejected");
}
return {
path,
publicKey: new Uint8Array(publicKey),
privateKey: new Uint8Array(privateKey),
publicKeyEncoded: parsed.publicKey,
publicKeyHash: sha256(publicKey),
};
}
async function controlJson({
path,
method = "POST",
verb,
body,
identity,
exchange,
unbound = false,
}) {
const authorization = signedWorkerAuthorization({
identity,
exchange,
verb,
body,
now: new Date(),
unbound,
});
const response = await fetch(`${ORIGIN}${path}`, {
method,
headers: {
authorization,
"content-type": "application/json",
},
...(method === "POST" ? { body: JSON.stringify(body) } : {}),
signal: AbortSignal.timeout(30_000),
});
const value = await response.json();
if (!response.ok) throw new Error(value?.error ?? "admin_control_failed");
return value;
}
async function enrollDeliveryKey({ identity, exchange, deliveryKey }) {
const body = {
schemaVersion: "sellable-admin-delivery-key/v1",
...exchange.adminDeliveryTarget,
deliveryPublicKey: deliveryKey.publicKeyEncoded,
deliveryKeyHash: deliveryKey.publicKeyHash,
alg: "crypto_box_seal",
kty: "OKP",
crv: "X25519",
keyUse: "enc",
nonce: randomUUID(),
issuedAt: new Date().toISOString(),
};
const privateKey = createPrivateKey(readFileSync(identity.privateKeyPath));
const attestation = {
body,
signature: sign(
null,
Buffer.from(JSON.stringify(body), "utf8"),
privateKey
).toString("base64url"),
};
const result = await controlJson({
path: "/api/v3/sellable-agent/worker/admin-delivery-key",
verb: "admin-delivery-key-enroll",
body: attestation,
identity,
exchange,
unbound: true,
});
if (
result?.enrollment?.runtimeId !== exchange.runtimeId ||
result.enrollment.adminRuntimeGenerationId !==
exchange.adminDeliveryTarget.adminRuntimeGenerationId ||
result.enrollment.adminRuntimeGenerationFence !==
exchange.adminDeliveryTarget.adminRuntimeGenerationFence ||
result.enrollment.deliveryKeyHash !== deliveryKey.publicKeyHash
) {
throw new Error("admin_delivery_key_enrollment_rejected");
}
}
async function pollDelivery({ identity, exchange }) {
const target = exchange.adminDeliveryTarget;
const body = {
schemaVersion: "sellable-admin-secret-delivery-poll/v1",
runtimeId: exchange.runtimeId,
adminRuntimeGenerationId: target.adminRuntimeGenerationId,
adminRuntimeGenerationFence: target.adminRuntimeGenerationFence,
};
return (
await controlJson({
path: "/api/v3/sellable-agent/admin-secret-deliveries/pending",
verb: "admin-secret-poll",
body,
identity,
exchange,
unbound: true,
})
).operation;
}
function operationIdentity(claim, exchange) {
return {
operationId: claim.operation.id,
workspaceId: claim.operation.workspaceId,
agentId: claim.operation.agentId,
hostId: exchange.hostId,
workerEnrollmentId: exchange.enrollmentId,
workerId: exchange.workerId,
fence: claim.operation.fence,
targetRevisionId: claim.operation.targetRevisionId,
};
}
function adminRuntimeFailureErrorClass(error) {
const raw =
error instanceof Error && error.message
? error.message
: "admin_runtime_failed";
const normalized = raw
.toUpperCase()
.replace(/[^A-Z0-9_]/g, "_")
.replace(/_+/g, "_")
.slice(0, 64);
return /^[A-Z]/.test(normalized)
? normalized
: `ADMIN_${normalized}`.slice(0, 64);
}
function adminRuntimeFailureCompletionReceipt({
claim,
exchange,
error,
secretsConsumed,
}) {
const hasRuntimeSecrets = [
claim.secretEnvelopes?.mcpService,
claim.secretEnvelopes?.slackApp,
claim.secretEnvelopes?.slackBot,
claim.secretEnvelopes?.modelCodex,
].some(Boolean);
const uncertain = secretsConsumed === true;
const unsigned = {
...operationIdentity(claim, exchange),
status: uncertain ? "UNCERTAIN" : "FAILED",
secretDisposition: uncertain
? "UNCERTAIN"
: hasRuntimeSecrets
? "NOT_CONSUMED"
: "NOT_PRESENT",
observedLifecycle: "DEGRADED",
observedRevisionId: null,
signerGeneration: exchange.keyGeneration,
runtimeHashes: {
package: null,
config: null,
content: null,
process: null,
},
conditions: [
{
code: uncertain ? "RUNTIME_UNCERTAIN" : "PROVISIONING_FAILED",
status: uncertain ? "UNKNOWN" : "FALSE",
},
],
errorClass: adminRuntimeFailureErrorClass(error),
auditIds: [`fly-admin-${claim.operation.id}`],
};
return { ...unsigned, signedDigest: flyRuntimeDigest(unsigned) };
}
function operationSnapshotIdentity(claim, exchange) {
const authority = claim?.runtimeAuthority;
if (
!isOperationBoundBackupClaim(claim) ||
authority?.scope !== "ASSIGNED" ||
authority.runtimeArtifactId !== authority.assignedRuntimeArtifactId
) {
return undefined;
}
return {
operationId: claim.operation.id,
authorityGeneration: authority.authorityGeneration,
runtimeArtifactId: authority.assignedRuntimeArtifactId,
sourceMachineId: exchange.workerId,
sourceMachineVersion: authority.machineVersion,
sourceBootSessionId: BOOT_SESSION_ID,
profileSchemaVersion: "admin-native-profile/v2",
};
}
function adminServingObservation(activeLaunch, activeGateway, exchange) {
try {
const authority = activeLaunch?.claim?.runtimeAuthority;
const running = nativeGatewayRunning(activeGateway);
const productMcp = activeLaunch?.profile?.servers?.find(
(server) => server.alias === "sellable"
);
const adminMcp = activeLaunch?.profile?.servers?.find(
(server) => server.alias === "sellable-admin"
);
if (!authority || !running || !activeLaunch.slackProof) return null;
const observedAt = new Date();
return buildRuntimeServingObservation({
scope: authority.scope,
kind: "ADMIN",
operationId: activeLaunch.claim.operation.id,
workspaceId: activeLaunch.claim.operation.workspaceId,
agentId: activeLaunch.claim.operation.agentId,
runtimeId: authority.runtimeId,
authorityGeneration: authority.authorityGeneration,
runtimeArtifactId: authority.runtimeArtifactId,
machineId: exchange.workerId,
machineVersion: authority.machineVersion,
bootSessionId: BOOT_SESSION_ID,
profileIdentity: activeLaunch.profile.profile,
processIdentity: `gateway-${activeGateway.child.pid}-${activeLaunch.claim.operation.fence}`,
observedAt: observedAt.toISOString(),
expiresAt: new Date(observedAt.getTime() + 120_000).toISOString(),
slack: {
authenticated: true,
socketModeConnected: true,
apiReachable: true,
},
productMcp: {
serverInfoName: productMcp?.serverInfoName,
toolCount: productMcp?.toolCount,
},
adminMcp: {
serverInfoName: adminMcp?.serverInfoName,
toolCount: adminMcp?.toolCount,
},
...(authority.bundleVersion === undefined
? {}
: {
bundleVersion: authority.bundleVersion,
bundleDigest: authority.bundleDigest,
}),
nativeGateway: true,
customerRouting: false,
});
} catch {
return null;
}
}
function startOperationHeartbeat({ claim, exchange, client }) {
const identity = operationIdentity(claim, exchange);
let stopped = false;
let failure = null;
let inFlight = Promise.resolve();
const beat = () => {
if (stopped) return;
inFlight = client
.heartbeat({ ...identity, leaseSeconds: 60 })
.then((value) => {
if (value?.renewed !== true) {
throw new Error("admin_runtime_heartbeat_rejected");
}
})
.catch((error) => {
failure =
error instanceof Error
? error
: new Error("admin_runtime_heartbeat_failed");
});
};
const timer = setInterval(beat, 20_000);
beat();
return {
assertHealthy() {
if (failure) throw failure;
},
async reattest() {
beat();
await inFlight;
if (failure) throw failure;
},
async stop() {
stopped = true;
clearInterval(timer);
await inFlight;
if (failure) throw failure;
},
};
}
async function consumeRuntimeSecrets({ claim, exchange, client }) {
const identity = operationIdentity(claim, exchange);
const consume = async (descriptor) => {
if (!descriptor) throw new Error("admin_runtime_secret_missing");
const value = await client.consumeSecret({
...identity,
envelopeId: descriptor.id,
claimId: randomUUID(),
});
if (
value.envelopeId !== descriptor.id ||
value.generation !== descriptor.generation ||
typeof value.secret !== "string" ||
value.secret.length < 1 ||
sha256(value.secret).slice(0, 16) !== descriptor.fingerprint
) {
throw new Error("admin_runtime_secret_delivery_rejected");
}
return value.secret;
};
const [mcpService, slackApp, slackBot, modelCodex] = await Promise.all([
consume(claim.secretEnvelopes?.mcpService),
consume(claim.secretEnvelopes?.slackApp),
consume(claim.secretEnvelopes?.slackBot),
consume(claim.secretEnvelopes?.modelCodex),
]);
return { mcpService, slackApp, slackBot, modelCodex };
}
function adminProfileFactoryAuthJson(model) {
if (model.value.schemaVersion !== "sellable-agent-model-auth-runtime/v3") {
return model.hermesRootAuthJson;
}
let store;
try {
store = JSON.parse(model.hermesRootAuthJson);
} catch {
throw new Error("admin_model_auth_profile_factory_compat_rejected");
}
const records = store?.credential_pool?.["openai-codex"];
const expected = [
{ sourceId: "sellable-primary", priority: 1 },
{ sourceId: "dittto-secondary", priority: 2 },
{ sourceId: "kaigo-tertiary", priority: 3 },
];
if (
!Array.isArray(records) ||
records.length !== expected.length ||
records.some(
(record, index) =>
record?.id !== expected[index].sourceId ||
record?.priority !== expected[index].priority
)
) {
throw new Error("admin_model_auth_profile_factory_compat_rejected");
}
// The pinned Admin profile factory predates the three-slot runtime schema and
// accepts only its former Dittto/Kaigo access-only proof. It does not install
// or copy this file; the runtime and Hermes continue to use DATA_ROOT/auth.json
// with all three slots. Preserve the two matching accounts solely for that
// redundant factory validation until the pinned package can consume v3.
const legacyRecords = records.slice(1).map((record, index) => {
const legacy = [
{ id: "dittto-primary", priority: 1 },
{ id: "kaigo-secondary", priority: 2 },
][index];
return {
...record,
id: legacy.id,
label: legacy.id,
priority: legacy.priority,
};
});
return `${JSON.stringify(
{
...store,
credential_pool: {
...store.credential_pool,
"openai-codex": legacyRecords,
},
},
null,
2
)}\n`;
}
function assertAdminSellableCredentialBinding({ claim, secret }) {
const serviceAuthority = claim?.runtimeContract?.service;
const serviceEnvelope = claim?.secretEnvelopes?.mcpService;
if (
!serviceAuthority ||
serviceEnvelope?.provider !== "sellable" ||
serviceEnvelope?.purpose !== "service-credential" ||
serviceEnvelope.workspaceId !== claim.operation.workspaceId ||
serviceEnvelope.agentId !== claim.operation.agentId ||
serviceEnvelope.operationId !== claim.operation.id ||
serviceAuthority.credentialGeneration !== serviceEnvelope.generation ||
serviceAuthority.credentialFingerprint !== serviceEnvelope.fingerprint ||
typeof secret !== "string" ||
!secret.startsWith("sat_") ||
sha256(secret).slice(0, 16) !== serviceEnvelope.fingerprint
) {
throw new Error("admin_sellable_credential_binding_rejected");
}
}
let installedManagedAuthProfile = null;
async function installRuntimeSecrets({ claim, secrets, desiredLifecycle }) {
privateDirectory(SECRET_ROOT);
assertAdminSellableCredentialBinding({
claim,
secret: secrets.mcpService,
});
const modelAuthority = claim?.runtimeContract?.model;
const modelEnvelope = claim?.secretEnvelopes?.modelCodex;
const modelTuple = claim?.adminRuntimeGeneration?.tuple;
if (
modelAuthority?.provider !== "codex" ||
!/^[A-Za-z0-9_.:-]{1,256}$/.test(modelAuthority.activationId ?? "") ||
!Number.isSafeInteger(modelAuthority.bundleGeneration) ||
modelAuthority.bundleGeneration < 1 ||
!/^[a-f0-9]{64}$/.test(modelAuthority.policyHash ?? "") ||
!/^[a-f0-9]{16}$/.test(modelAuthority.credentialFingerprint ?? "") ||
modelAuthority.bundleGeneration !== modelEnvelope?.generation ||
modelAuthority.credentialFingerprint !== modelEnvelope?.fingerprint ||
modelEnvelope?.activationId !== modelAuthority.activationId ||
modelEnvelope?.bundleGeneration !== modelAuthority.bundleGeneration ||
modelEnvelope?.policyHash !== modelAuthority.policyHash ||
modelTuple?.modelAuthActivationId !== modelAuthority.activationId ||
modelTuple?.modelAuthBundleGeneration !== modelAuthority.bundleGeneration ||
modelTuple?.modelAuthPolicyHash !== modelAuthority.policyHash ||
!/^[A-Za-z0-9_-]{1,128}$/.test(claim?.operation?.profileId ?? "")
) {
throw new Error("admin_model_auth_identity_rejected");
}
const model = parseModelAuthRuntimeCredential(secrets.modelCodex);
if (
model.activationId !== modelAuthority.activationId ||
model.bundleGeneration !== modelAuthority.bundleGeneration ||
model.policyHash !== modelAuthority.policyHash
) {
throw new Error("admin_model_auth_identity_rejected");
}
const rootAuthPath = join(DATA_ROOT, "auth.json");
const markerPath = join(RUNTIME_ROOT, "model-auth-source.json");
const localAuthPath = join(
PROFILES_ROOT,
claim.operation.profileId,
"auth.json"
);
const identityPath = join(RUNTIME_ROOT, "model-auth-identity.json");
const modelBundlePath = join(SECRET_ROOT, "model-auth-runtime-v2.json");
const validateAuthFile = (path) => {
const stat = lstatSync(path);
if (stat.isSymbolicLink() || !stat.isFile() || (stat.mode & 0o077) !== 0) {
throw new Error("admin_model_auth_store_rejected");
}
};
if (existsSync(localAuthPath)) {
validateAuthFile(localAuthPath);
rmSync(localAuthPath, { force: true });
}
atomicPrivateFile(modelBundlePath, model.authJson);
const modelAuthState = reconcileHermesModelAuth({
authStorePath: rootAuthPath,
markerPath,
credential: model,
lifecycle: desiredLifecycle,
});
const profileAuthPath =
model.value.schemaVersion === "sellable-agent-model-auth-runtime/v3"
? join(SECRET_ROOT, "model-auth-profile-factory-compat.json")
: desiredLifecycle === "ON"
? rootAuthPath
: join(SECRET_ROOT, "model-auth-pending-start.json");
if (profileAuthPath !== rootAuthPath) {
atomicPrivateFile(profileAuthPath, adminProfileFactoryAuthJson(model));
}
atomicPrivateFile(
identityPath,
`${JSON.stringify({
schemaVersion: "sellable-admin-model-auth/v2",
provider: "codex",
activationId: modelAuthority.activationId,
bundleGeneration: modelAuthority.bundleGeneration,
policyHash: modelAuthority.policyHash,
credentialFingerprint: modelAuthority.credentialFingerprint,
materializationStatus: modelAuthState.materialized
? "current"
: "pending_start",
})}\n`
);
atomicPrivateFile(join(SECRET_ROOT, "slack-app"), `${secrets.slackApp}\n`);
atomicPrivateFile(join(SECRET_ROOT, "slack-bot"), `${secrets.slackBot}\n`);
atomicPrivateFile(
join(SECRET_ROOT, "api-server-key"),
sha256(`admin-endpoint-api\0${randomUUID()}\0${Date.now()}`)
);
const slackSecretFile = join(SECRET_ROOT, "profile-slack.env");
atomicPrivateFile(
slackSecretFile,
`SLACK_BOT_TOKEN=${secrets.slackBot}\nSLACK_APP_TOKEN=${secrets.slackApp}\n`
);
const serviceEnv = adminAgentServiceEnvironment(claim);
await publishManagedAuth({
schemaVersion: "sellable-managed-auth/v1",
profileId: claim.operation.profileId,
runtimeId: RUNTIME_ID,
runtimeGeneration: RUNTIME_GENERATION,
agentKind: "ADMIN",
apiUrl: ORIGIN,
token: secrets.mcpService,
agent: {
workspaceId: claim.operation.workspaceId,
agentId: claim.operation.agentId,
generation: Number(serviceEnv.SELLABLE_AGENT_CREDENTIAL_GENERATION),
policyHash: serviceEnv.SELLABLE_AGENT_POLICY_HASH,
selectedChannelIds: JSON.parse(
serviceEnv.SELLABLE_AGENT_SELECTED_CHANNEL_IDS
),
},
});
installedManagedAuthProfile = claim.operation.profileId;
// This legacy path now holds preferences only. The root supervisor owns
// service authority; a shell copy here cannot replace credentials or origin.
const productConfigPath = join(SECRET_ROOT, "sellable-config.json");
atomicPrivateFile(
productConfigPath,
`${JSON.stringify({
apiUrl: ORIGIN,
activeWorkspaceId: claim.operation.workspaceId,
workspaceId: claim.operation.workspaceId,
})}\n`
);
return {
rootAuthPath,
profileAuthPath,
markerPath,
model,
modelAuthState,
modelConfig: modelAuthConfigOverlay(model),
productConfigPath,
slackSecretFile,
};
}
function pendingAdminModelAuthReadback(model) {
return Object.freeze({
schemaVersion: model.value.schemaVersion.endsWith("/v3")
? "sellable-agent-model-auth-readback/v3"
: "sellable-agent-model-auth-readback/v2",
activationId: model.activationId,
bundleGeneration: model.bundleGeneration,
policyHash: model.policyHash,
provider: model.provider,
model: model.model,
fallbackDepth: model.value.openrouter.models.length,
desiredStatus: "converged",
materializedStatus: "pending_start",
slots: Object.freeze(
model.identity.slots.map((slot) =>
Object.freeze({
slotId: slot.slotId,
...(model.value.schemaVersion.endsWith("/v3")
? { priority: slot.priority }
: {}),
grantFingerprint: slot.grantFingerprint,
expiresAt: new Date(slot.expiresAt).toISOString(),
state: "ok",
lastErrorReason: null,
lastErrorResetAt: null,
})
)
),
openrouter: Object.freeze({
keyFingerprint: model.identity.openrouterKeyFingerprint,
state: "ok",
}),
});
}
function proveAdminModelAuth(profile, installed, desiredLifecycle) {
if (desiredLifecycle === "OFF") {
return Object.freeze({
...pendingAdminModelAuthReadback(installed.model),
source: "desired_converged_materialization_pending_start",
sourceClass: "access_only_pool_pending_start",
runtimeRefreshOwned: false,
});
}
const materialization = readModelAuthMaterialization({
authStorePath: installed.rootAuthPath,
markerPath: installed.markerPath,
});
const verdict = evaluateModelAuthReadback(materialization, {
activationId: installed.model.activationId,
bundleGeneration: installed.model.bundleGeneration,
policyHash: installed.model.policyHash,
});
if (!verdict.accepted) {
throw new Error(
`admin_model_auth_readback_rejected:${verdict.rejection ?? "unknown"}`
);
}
const environment = {
...process.env,
HOME: DATA_ROOT,
HERMES_HOME: profile.profileHome,
SELLABLE_HERMES_PROFILE_ID: profile.profile,
NO_COLOR: "1",
};
delete environment.OPENAI_BASE_URL;
delete environment.OPENAI_API_KEY;
delete environment.OPENROUTER_API_KEY;
const probe = spawnSync(
HERMES_PYTHON,
[
"-c",
[
"import json",
"from agent.credential_pool import load_pool",
'pool = load_pool("openai-codex")',
"entry = pool.select()",
'print(json.dumps({"provider": "openai-codex", "source": getattr(entry, "source", None), "source_class": "access_only_pool", "credential_id": getattr(entry, "id", None), "credential_present": bool(getattr(entry, "access_token", None)), "runtime_refresh_owned": bool(getattr(entry, "refresh_token", None))}))',
].join(";"),
],
{
env: environment,
encoding: "utf8",
timeout: TIMEOUT_MS,
maxBuffer: 64 * 1024,
stdio: ["ignore", "pipe", "pipe"],
}
);
let readback = null;
try {
readback = JSON.parse(probe.stdout ?? "");
} catch {
readback = null;
}
if (
probe.status !== 0 ||
probe.error ||
readback?.provider !== "openai-codex" ||
readback?.source !== "manual:device_code" ||
readback?.source_class !== "access_only_pool" ||
![
"dittto-primary",
"kaigo-secondary",
"sellable-primary",
"dittto-secondary",
"kaigo-tertiary",
].includes(readback?.credential_id) ||
readback?.credential_present !== true ||
readback?.runtime_refresh_owned !== false
) {
throw new Error("access_only_snapshot_infeasible");
}
return Object.freeze({
...materialization,
source: readback.source,
sourceClass: readback.source_class,
selectedCredentialId: readback.credential_id,
modelAuthSelection: modelAuthSelectionObservation(materialization),
runtimeRefreshOwned: readback.runtime_refresh_owned,
});
}
function proveAdminAttachmentHandoff(profile) {
const mediaPath = join(profile.profileHome, "attachment-canary.csv");
writeFileSync(mediaPath, "name,status\nphase158,ready\n", { mode: 0o600 });
let probe;
try {
probe = spawnSync(
HERMES_PYTHON,
[
"-c",
[
"import json, sys",
"from gateway.platforms.base import validate_media_delivery_path",
"observed = validate_media_delivery_path(sys.argv[1])",
'print(json.dumps({"accepted": observed == sys.argv[1]}))',
].join(";"),
mediaPath,
],
{
env: {
...process.env,
HOME: profile.profileHome,
HERMES_HOME: DATA_ROOT,
HERMES_WRITE_SAFE_ROOT: DATA_ROOT,
SELLABLE_HERMES_PROFILE_ID: profile.profile,
NO_COLOR: "1",
},
encoding: "utf8",
timeout: TIMEOUT_MS,
maxBuffer: 64 * 1024,
stdio: ["ignore", "pipe", "pipe"],
}
);
} finally {
rmSync(mediaPath, { force: true });
}
let readback = null;
try {
readback = JSON.parse(probe.stdout ?? "");
} catch {
readback = null;
}
if (probe.status !== 0 || probe.error || readback?.accepted !== true) {
throw new Error("admin_attachment_media_path_rejected");
}
return Object.freeze({
home: profile.profileHome,
hermesHome: DATA_ROOT,
writeSafeRoot: DATA_ROOT,
mediaPathAccepted: true,
deliveryMode: "native",
});
}
async function clearRuntimeSecrets() {
if (installedManagedAuthProfile) {
await clearManagedAuth(installedManagedAuthProfile);
installedManagedAuthProfile = null;
}
for (const name of [
"slack-app",
"slack-bot",
"api-server-key",
"profile-slack.env",
"sellable-config.json",
]) {
rmSync(join(SECRET_ROOT, name), { force: true });
}
}
function adminAgentServiceEnvironment(claim) {
const operation = claim?.operation;
const adminGeneration = claim?.adminRuntimeGeneration;
const service = claim?.runtimeContract?.service;
const bindings = claim?.runtimeContract?.slack?.selectedBindings;
const toolInclude = service?.toolInclude;
// The operation lease fence and immutable Admin-generation fence are
// independent CAS domains. The broker already loads the generation through
// operation.adminRuntimeGenerationId and binds it to this workspace/agent.
if (
claim?.agentKind !== "ADMIN" ||
adminGeneration?.agentKind !== "ADMIN" ||
adminGeneration.workspaceId !== operation?.workspaceId ||
adminGeneration.agentId !== operation?.agentId ||
!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(adminGeneration.id ?? "") ||
!/^[1-9][0-9]{0,19}$/.test(String(adminGeneration.fence ?? "")) ||
!Number.isSafeInteger(RUNTIME_GENERATION) ||
RUNTIME_GENERATION < 1 ||
!operation ||
!/^[A-Za-z0-9_-]{1,128}$/.test(operation.profileId ?? "") ||
!service ||
!/^[A-Za-z0-9_-]{1,128}$/.test(service.principalId ?? "") ||
!Array.isArray(toolInclude) ||
toolInclude.length < 1 ||
JSON.stringify(toolInclude) !==
JSON.stringify([...new Set(toolInclude)].sort()) ||
!Array.isArray(bindings) ||
bindings.length < 1 ||
bindings.some(
(binding) =>
!binding ||
typeof binding.channelId !== "string" ||
!/^[A-Za-z0-9_-]{1,128}$/.test(binding.channelId)
)
) {
throw new Error("admin_runtime_agent_service_environment_rejected");
}
return Object.freeze({
...buildAdminAgentServiceEnvironment({
agentKind: claim.agentKind,
workspaceId: operation.workspaceId,
agentId: operation.agentId,
serviceCredentialGeneration: service.credentialGeneration,
policyHash: service.policyHash,
selectedChannelIds: [
...new Set(bindings.map((binding) => binding.channelId)),
].sort(),
}),
// Admin intentionally has no customer workspace lock. This explicit kind
// lets Product MCP distinguish that omission from a partially materialized
// customer profile while the signed Agent service context remains the home
// workspace authority.
SELLABLE_AGENT_SERVER_AUTHENTICATED_KIND: "ADMIN",
SELLABLE_AGENT_RUNTIME: "1",
SELLABLE_AGENT_RUNTIME_GENERATION: String(RUNTIME_GENERATION),
SELLABLE_RUNTIME_ID: RUNTIME_ID,
SELLABLE_AGENT_MANAGED_AUTH_PATH: managedAuthPath(
claim.operation.profileId
),
SELLABLE_AGENT_ADMIN_GENERATION_ID: claim.adminRuntimeGeneration.id,
SELLABLE_AGENT_ADMIN_GENERATION_FENCE: String(
claim.adminRuntimeGeneration.fence
),
});
}
function nativeSlackProfileInput(claim, installed) {
const slack = claim.runtimeContract?.slack;
const bindings = slack?.selectedBindings;
if (
!slack ||
!Array.isArray(bindings) ||
bindings.length < 1 ||
!bindings.every(
(binding) =>
typeof binding.channelId === "string" &&
binding.channelId &&
Number.isSafeInteger(binding.routeGeneration) &&
binding.routeGeneration > 0
)
) {
throw new Error("admin_runtime_slack_contract_rejected");
}
const primary =
bindings.find((binding) => binding.id === slack.defaultBindingId) ??
bindings.find((binding) => binding.isDefaultVerification === true);
if (!primary) throw new Error("admin_runtime_slack_contract_rejected");
return {
secretFile: installed.slackSecretFile,
teamId: slack.teamId,
primaryChannel: {
id: primary.channelId,
name: primary.channelName ?? primary.channelId,
},
selectedChannelIds: [
...new Set(bindings.map((binding) => binding.channelId)),
].sort(),
secondaryReadChannelIds: [],
allowedSlackUserIds: [...slack.memberAllowlist],
slackApp: {
appId: slack.appId,
botId: slack.botUserId,
},
};
}
async function proveAdminSlackServing(claim, secrets) {
const slack = claim.runtimeContract?.slack;
if (!slack) throw new Error("admin_runtime_slack_contract_rejected");
const post = (method, token) =>
fetch(`https://slack.com/api/${method}`, {
method: "POST",
headers: {
authorization: `Bearer ${token}`,
"content-type": "application/x-www-form-urlencoded",
},
body: "",
signal: AbortSignal.timeout(15_000),
});
const [authResponse, socketResponse] = await Promise.all([
post("auth.test", secrets.slackBot),
post("apps.connections.open", secrets.slackApp),
]);
const [auth, socket] = await Promise.all([
authResponse.json(),
socketResponse.json(),
]);
if (
!authResponse.ok ||
auth?.ok !== true ||
auth.team_id !== slack.teamId ||
auth.user_id !== slack.botUserId ||
!socketResponse.ok ||
socket?.ok !== true ||
typeof socket.url !== "string" ||
!/^wss:\/\//.test(socket.url)
) {
throw new Error("admin_runtime_slack_serving_rejected");
}
return true;
}
async function fetchDeliveryCiphertext({
operation,
identity,
exchange,
claimId,
}) {
const body = {
schemaVersion: "sellable-admin-secret-delivery-claim/v1",
operationId: operation.operationId,
claimId,
workspaceId: operation.workspaceId,
agentId: operation.agentId,
};
const authorization = signedWorkerAuthorization({
identity,
exchange,
verb: "secret-consume",
body,
now: new Date(),
});
const query = new URLSearchParams({ claimId });
const response = await fetch(
`${ORIGIN}/api/v3/sellable-agent/admin-secret-deliveries/${operation.operationId}/ciphertext?${query}`,
{
headers: { authorization },
signal: AbortSignal.timeout(30_000),
}
);
if (!response.ok) {
const value = await response.json().catch(() => null);
throw new Error(value?.error ?? "admin_delivery_ciphertext_failed");
}
const bytes = new Uint8Array(await response.arrayBuffer());
if (
bytes.byteLength !== operation.ciphertextBytes ||
sha256(bytes) !== operation.ciphertextDigest
) {
throw new Error("admin_delivery_ciphertext_rejected");
}
return bytes;
}
function recoveryKey(expected = null) {
const path = process.env.SELLABLE_ADMIN_RECOVERY_PRIVATE_KEY_FILE ?? "";
if (
Object.prototype.hasOwnProperty.call(
process.env,
"SELLABLE_ADMIN_RECOVERY_PRIVATE_KEY_PEM"
) ||
!path ||
!existsSync(path)
) {
throw new Error("admin_recovery_private_key_missing");
}
const stat = lstatSync(path);
if (stat.isSymbolicLink() || !stat.isFile() || (stat.mode & 0o077) !== 0) {
throw new Error("admin_recovery_private_key_rejected");
}
const privateKey = createPrivateKey(readFileSync(path));
if (privateKey.asymmetricKeyType !== "x25519") {
throw new Error("admin_recovery_private_key_rejected");
}
const jwk = privateKey.export({ format: "jwk" });
const publicJwk = createPublicKey(privateKey).export({ format: "jwk" });
const secret = Buffer.from(jwk.d ?? "", "base64url");
const publicKey = Buffer.from(publicJwk.x ?? "", "base64url");
if (secret.byteLength !== 32 || publicKey.byteLength !== 32) {
throw new Error("admin_recovery_private_key_rejected");
}
const key = {
publicKey: new Uint8Array(publicKey),
privateKey: new Uint8Array(secret),
publicKeyEncoded: publicKey.toString("base64url"),
keyId: `recovery_${sha256(publicKey).slice(0, 32)}`,
};
if (
expected &&
(key.publicKeyEncoded !== expected.recoveryPublicKey ||
key.keyId !== expected.recoveryKeyId)
) {
throw new Error("admin_recovery_private_key_binding_rejected");
}
return key;
}
async function publishRecovery({
operation,
manifest,
files,
identity,
exchange,
generation,
}) {
const key = recoveryKey(generation);
const plaintext = encodeAdminSettingsBundle({ manifest, files });
const ciphertext = sodium.crypto_box_seal(plaintext, key.publicKey);
const body = {
schemaVersion: "sellable-admin-recovery-publish/v1",
operationId: operation.operationId,
runtimeId: operation.runtimeId,
adminRuntimeGenerationId: operation.runtimeGenerationId,
adminRuntimeGenerationFence: operation.generationFence,
settingsDigest: operation.settingsDigest,
recoveryKeyId: key.keyId,
recoveryPublicKey: key.publicKeyEncoded,
ciphertextDigest: sha256(ciphertext),
ciphertextBytes: ciphertext.byteLength,
ciphertextBase64: Buffer.from(ciphertext).toString("base64"),
};
const result = await controlJson({
path: `/api/v3/sellable-agent/admin-secret-deliveries/${operation.operationId}/recovery`,
verb: "admin-recovery-publish",
body,
identity,
exchange,
unbound: true,
});
if (
result?.published !== true ||
result.recovery?.recoveryKeyId !== key.keyId ||
result.recovery?.ciphertextDigest !== body.ciphertextDigest
) {
throw new Error("admin_recovery_publish_rejected");
}
return {
...result.recovery,
ciphertextBytes: body.ciphertextBytes,
};
}
async function importDelivery({
operation,
identity,
exchange,
deliveryKey,
generation,
}) {
const claimId = deliveryClaimId(operation.operationId);
let recovery = null;
let decodedManifest = null;
let decodedFiles = null;
const targetRoot = join(RUNTIME_ROOT, "settings");
privateDirectory(targetRoot);
const target = {
workspaceId: operation.workspaceId,
agentId: operation.agentId,
agentKind: "ADMIN",
deliveryMode: operation.deliveryMode,
flyAppName: operation.flyAppName,
machineId: operation.flyMachineId,
machineVersion: operation.flyMachineVersion,
workloadKeyHash: operation.workloadKeyHash,
runtimeGenerationId: operation.runtimeGenerationId,
generationFence: operation.generationFence,
};
const receipt = await importAdminSecretGeneration(
{
schemaVersion: "AdminSecretImportRequestV1",
operationId: operation.operationId,
target,
ciphertextDigest: operation.ciphertextDigest,
ciphertextBytes: operation.ciphertextBytes,
plaintextDigest: operation.plaintextDigest,
settingsDigest: operation.settingsDigest,
targetRoot,
expectedUid: process.getuid?.(),
expectedGid: process.getgid?.(),
},
{
reattestTarget: async () => target,
claimCiphertext: async () =>
fetchDeliveryCiphertext({
operation,
identity,
exchange,
claimId,
}),
decrypt: async (ciphertext) => {
const plaintext = sodium.crypto_box_seal_open(
ciphertext,
deliveryKey.publicKey,
deliveryKey.privateKey
);
if (!plaintext) throw new Error("admin_delivery_decrypt_failed");
const decoded = JSON.parse(Buffer.from(plaintext).toString("utf8"));
decodedManifest = decoded.manifest;
decodedFiles = Object.fromEntries(
Object.entries(decoded.files).map(([name, encoded]) => [
name,
Buffer.from(encoded, "base64"),
])
);
return Buffer.from(plaintext);
},
resealRecovery: async () => {
recovery = await publishRecovery({
operation,
manifest: decodedManifest,
files: decodedFiles,
identity,
exchange,
generation,
});
return {
keyId: recovery.recoveryKeyId,
ciphertextDigest: recovery.ciphertextDigest,
ciphertextBytes: recovery.ciphertextBytes,
locator: recovery.locator,
};
},
publishGeneration: async () => ({
ok: true,
fence: operation.generationFence,
}),
cleanupCiphertext: async () => undefined,
}
);
if (!recovery) {
const persisted = receipt?.recovery;
if (
typeof persisted?.keyId !== "string" ||
typeof persisted.ciphertextDigest !== "string" ||
typeof persisted.locator !== "string"
) {
throw new Error("admin_secret_import_recovery_receipt_rejected");
}
recovery = {
recoveryKeyId: persisted.keyId,
ciphertextDigest: persisted.ciphertextDigest,
ciphertextBytes: persisted.ciphertextBytes,
locator: persisted.locator,
};
}
return { receipt, claimId, recovery };
}
function settingsProof(generationRoot, settingsDigest) {
const names = [".env", "settings.json", "config-state.json"];
const root = lstatSync(generationRoot);
if (
root.isSymbolicLink() ||
!root.isDirectory() ||
(root.mode & 0o077) !== 0 ||
JSON.stringify(readdirSync(generationRoot).sort()) !==
JSON.stringify([...names].sort())
) {
throw new Error("admin_runtime_settings_generation_rejected");
}
const manifestFiles = {};
const files = names.map((name) => {
const path = join(generationRoot, name);
const stat = lstatSync(path);
if (
stat.isSymbolicLink() ||
!stat.isFile() ||
(stat.mode & 0o777) !== 0o600
) {
throw new Error("admin_runtime_settings_generation_rejected");
}
const bytes = readFileSync(path);
const digest = sha256(bytes);
manifestFiles[name] = {
present: true,
sha256: digest,
bytes: bytes.byteLength,
mode: "0600",
};
return { name, digest };
});
const observedDigest = sha256(
`${canonicalJson({
files: manifestFiles,
readiness: { core: true },
})}\n`
);
if (observedDigest !== settingsDigest) {
throw new Error("admin_runtime_settings_digest_rejected");
}
return { digest: settingsDigest, files };
}
function profileContractForClaim(claim) {
const template = claim.adminRuntimeGeneration?.tuple?.profileContractTemplate;
const profileId = claim.operation?.profileId;
if (
template?.schemaVersion !== "sellable-admin-profile-contract-template/v1" ||
typeof profileId !== "string" ||
!/^[A-Za-z0-9_-]{1,128}$/.test(profileId)
) {
throw new Error("admin_runtime_profile_contract_rejected");
}
const { contentDigest, ...templateContent } = template;
if (
!/^[a-f0-9]{64}$/.test(contentDigest ?? "") ||
flyRuntimeDigest(templateContent) !== contentDigest
) {
throw new Error("admin_runtime_profile_contract_rejected");
}
const unsigned = {
schemaVersion: "sellable-admin-profile-contract/v1",
profileHome: join(PROFILES_ROOT, profileId),
aliases: template.aliases,
packages: template.packages,
tools: template.tools,
contentDigest,
};
return { ...unsigned, digest: flyRuntimeDigest(unsigned) };
}
async function fetchRecoveryCiphertext({
sourceOperationId,
identity,
exchange,
generation,
}) {
const body = {
schemaVersion: "sellable-admin-recovery-fetch/v1",
runtimeId: RUNTIME_ID,
adminRuntimeGenerationId: generation.id,
adminRuntimeGenerationFence: generation.fence,
};
const authorization = signedWorkerAuthorization({
identity,
exchange,
verb: "admin-recovery-fetch",
body,
now: new Date(),
unbound: true,
});
const query = new URLSearchParams({
runtimeId: body.runtimeId,
adminRuntimeGenerationId: body.adminRuntimeGenerationId,
adminRuntimeGenerationFence: body.adminRuntimeGenerationFence,
});
const response = await fetch(
`${ORIGIN}/api/v3/sellable-agent/admin-secret-deliveries/${sourceOperationId}/recovery?${query}`,
{
headers: { authorization },
signal: AbortSignal.timeout(30_000),
}
);
if (!response.ok) {
const value = await response.json().catch(() => null);
throw new Error(value?.error ?? "admin_recovery_fetch_failed");
}
const ciphertext = new Uint8Array(await response.arrayBuffer());
if (
sha256(ciphertext) !== generation.recoveryCiphertextDigest ||
response.headers.get("x-sellable-admin-recovery-digest") !==
generation.recoveryCiphertextDigest
) {
throw new Error("admin_recovery_ciphertext_rejected");
}
return ciphertext;
}
async function restoreSuccessorSettings({ claim, identity, exchange }) {
const generation = claim.adminRuntimeGeneration;
if (
generation.state !== "PROVING" ||
!generation.recoverySourceOperationId ||
!generation.settingsDigest ||
!generation.recoveryKeyId ||
!generation.recoveryPublicKey ||
!generation.recoveryCiphertextDigest ||
!generation.recoveryLocator
) {
throw new Error("admin_runtime_recovery_authority_rejected");
}
const key = recoveryKey(generation);
const ciphertext = await fetchRecoveryCiphertext({
sourceOperationId: generation.recoverySourceOperationId,
identity,
exchange,
generation,
});
const plaintext = sodium.crypto_box_seal_open(
ciphertext,
key.publicKey,
key.privateKey
);
if (!plaintext) throw new Error("admin_recovery_decrypt_failed");
const targetRoot = join(RUNTIME_ROOT, "settings");
privateDirectory(targetRoot);
const receipt = restoreAdminSecretGeneration({
schemaVersion: "AdminRecoveryRestoreRequestV1",
recoverySourceOperationId: generation.recoverySourceOperationId,
workspaceId: claim.operation.workspaceId,
agentId: claim.operation.agentId,
successorGenerationId: generation.id,
successorGenerationFence: generation.fence,
settingsDigest: generation.settingsDigest,
targetRoot,
plaintext: Buffer.from(plaintext),
expectedUid: process.getuid?.(),
expectedGid: process.getgid?.(),
});
return {
receipt,
recovery: {
recoveryKeyId: generation.recoveryKeyId,
ciphertextDigest: generation.recoveryCiphertextDigest,
locator: generation.recoveryLocator,
},
};
}
async function acknowledgeDelivery({
operation,
claimId,
observedProfile,
profileProofDigest,
resultDigest,
identity,
exchange,
}) {
const receipt = {
schemaVersion: "sellable-admin-secret-delivery-receipt/v1",
claimId,
terminalStatus: "SUCCEEDED",
resultDigest,
observedProfileDigest: observedProfile.digest,
observedProfile,
profileProofDigest,
targetTransportCleaned: true,
failureCode: null,
};
const signedBody = {
operationId: operation.operationId,
workspaceId: operation.workspaceId,
agentId: operation.agentId,
...receipt,
};
const authorization = signedWorkerAuthorization({
identity,
exchange,
verb: "complete",
body: signedBody,
now: new Date(),
});
const response = await fetch(
`${ORIGIN}/api/v3/sellable-agent/admin-secret-deliveries/${operation.operationId}/receipt`,
{
method: "POST",
headers: { authorization, "content-type": "application/json" },
body: JSON.stringify(receipt),
signal: AbortSignal.timeout(30_000),
}
);
const value = await response.json();
if (!response.ok || value?.accepted !== true) {
throw new Error(value?.error ?? "admin_delivery_receipt_rejected");
}
}
function completionReceipt({
claim,
exchange,
operation,
profile,
settings,
recovery,
observedLifecycle,
memorySnapshot,
modelAuth,
activeProjection,
nativeDefaultProfileProof,
}) {
const identity = operationIdentity(claim, exchange);
const routeGenerationDigest = flyRuntimeDigest(
claim.revision?.manifest?.slack ?? null
);
const observedProfileDigest = sha256(
`${operation.profileContract.digest}\n${profile.observedSoulRawSha256}`
);
const adminRuntimeProof = {
schemaVersion: "sellable-agent-admin-runtime-proof/v1",
authenticatedKind: "ADMIN",
generationId: operation.runtimeGenerationId,
generationFence: operation.generationFence,
tupleDigest: claim.adminRuntimeGeneration.tupleDigest,
artifactContentDigest: claim.adminRuntimeGeneration.artifactContentDigest,
adminImageDigest: claim.adminRuntimeGeneration.adminImageDigest,
profile: {
factory: "stageAdminProfile",
digest: observedProfileDigest,
aliases: ["sellable", "sellable-admin"],
routeGenerationDigest,
},
settingsGeneration: settings,
recoveryGeneration: {
keyId: recovery.recoveryKeyId,
serviceSecretDigest:
claim.adminRuntimeGeneration.recoveryServiceSecretDigest,
ciphertextDigest: recovery.ciphertextDigest,
locatorDigest: sha256(recovery.locator),
},
modelAuth: {
activationId: modelAuth.activationId,
bundleGeneration: modelAuth.bundleGeneration,
policyHash: modelAuth.policyHash,
materializedStatus: modelAuth.materializedStatus,
},
servers: profile.servers,
traffic: {
slackAccepted: false,
customerRouting: false,
mutationCount: 0,
},
};
const runtimeHashes = {
package: sha256(JSON.stringify(packageSpecs())),
config: operation.profileContract.digest,
content: observedProfileDigest,
process: sha256(JSON.stringify({ servers: profile.servers, modelAuth })),
};
return {
...identity,
status: "SUCCEEDED",
secretDisposition: "INSTALLED",
observedLifecycle,
observedRevisionId: claim.operation.targetRevisionId,
...(activeProjection
? {
bundleVersion: activeProjection.bundleVersion,
bundleDigest: activeProjection.bundleDigest,
}
: {}),
signerGeneration: exchange.keyGeneration,
signedDigest: flyRuntimeDigest({
...identity,
observedLifecycle,
runtimeHashes,
adminRuntimeProof,
}),
runtimeHashes,
conditions: [
{ code: "ADMIN_PROFILE_MATERIALIZED", status: "TRUE" },
{ code: "DUAL_MCP_PROVEN", status: "TRUE" },
{ code: "SETTINGS_GENERATION_PROVEN", status: "TRUE" },
{ code: "RECOVERY_GENERATION_PROVEN", status: "TRUE" },
{
code:
modelAuth.materializedStatus === "current"
? "MODEL_AUTH_PROVEN"
: "MODEL_AUTH_DESIRED_PENDING_START",
status: "TRUE",
},
{ code: "SLACK_TRAFFIC_DISABLED", status: "TRUE" },
{ code: "NO_BOOT_INSTALL", status: "TRUE" },
{
code: "DEFAULT_PROFILE_NATIVE_PROOF",
status:
nativeDefaultProfileProof.status === "PROVEN" ? "TRUE" : "UNKNOWN",
},
],
errorClass: "NONE",
auditIds: [
`fly-admin-${claim.operation.id}`,
`soul-raw-${profile.observedSoulRawSha256}`,
`hermes-memory-${
memorySnapshot.ok ? memorySnapshot.status : memorySnapshot.code
}`.toLowerCase(),
`model-auth-${modelAuth.sourceClass}`,
],
adminRuntimeProof,
};
}
async function flushHermesSnapshot({
client,
profileId,
profileRoot,
operationIdentity,
forceDirty = false,
}) {
try {
if (forceDirty) markHermesSnapshotDirty(profileRoot);
return await reconcileHermesSnapshot({
profileId,
profileRoot,
hermesRoot: DATA_ROOT,
hermesVersion: "0.21.5",
debounceMs: 0,
operationIdentity,
upload: ({
archiveDigest,
archiveBytes,
sourceProfileId,
bytes,
ok: _ok,
status: _status,
hermesVersion: _hermesVersion,
...v2
}) =>
client.putMemorySnapshot(
{
archiveDigest,
archiveBytes,
hermesVersion: "0.21.5",
sourceProfileId,
...v2,
},
bytes
),
});
} catch {
return {
ok: false,
status: "PENDING_RETRY",
code: "hermes_snapshot_flush_failed",
};
}
}
function adminModelHealthReporter({ identity, exchange, installed, claim }) {
return createModelHealthReporter({
origin: ORIGIN,
identity: {
kind: "ADMIN",
runtimeGeneration: RUNTIME_GENERATION,
workloadKeyHash: exchange.publicKeyHash,
machineId: exchange.workerId,
adminRuntimeGenerationId: claim.adminRuntimeGeneration.id,
adminRuntimeGenerationFence: claim.adminRuntimeGeneration.fence,
},
observe: () =>
readModelAuthMaterialization({
authStorePath: installed.rootAuthPath,
markerPath: installed.markerPath,
}),
authorize: (body) =>
signedWorkerAuthorization({
identity,
exchange,
verb: "model-health",
body,
now: new Date(),
unbound: true,
}),
onError: (error) => {
process.stderr.write(
`${JSON.stringify({
ok: false,
code:
error instanceof Error
? error.message
: "model_health_report_failed",
})}\n`
);
},
allowHttpLoopback: process.env.SELLABLE_AGENT_ALLOW_TEST_OIDC === "1",
sequencePath: join(RUNTIME_ROOT, "model-health-sequence.json"),
});
}
function desiredLifecycleForClaim(claim) {
switch (claim.operation.action) {
case "INSTANTIATE":
case "DISABLE_GATEWAY":
case "ARCHIVE":
return "OFF";
case "ENABLE_GATEWAY":
case "RESTORE":
return "ON";
case "REFRESH":
if (
claim.operation.desiredLifecycle === "ON" ||
claim.operation.desiredLifecycle === "OFF"
) {
return claim.operation.desiredLifecycle;
}
throw new Error("admin_runtime_desired_lifecycle_missing");
default:
throw new Error("admin_runtime_action_rejected");
}
}
async function startNativeGateway({ profile, claim }) {
// `/run` is tmpfs. A Fly Machine restart can reuse the already-materialized
// durable profile without running its factory again, so recreate the Product
// MCP proxy roots at the actual gateway-launch boundary as well.
ensureProductMcpContextProxyRoots();
// buildNativeHermesLaunch replaces the environment instead of merging it, so
// the Admin gateway carries its own PATH for the generated bin/ wrappers.
// The escrowed .env is deliberately not sourced here: it carries
// SLACK_BOT_TOKEN, which the shared native-gateway policy rejects and which
// this runtime passes only through inherited file descriptors 8 and 9. Tool
// credentials stay in 0600 files and are loaded per invocation by
// bin/with-sellable-admin-env and by the generated CLI wrappers.
const launch = buildNativeHermesLaunch({
profileRoot: profile.profileHome,
secretsRoot: SECRET_ROOT,
profileId: profile.profile,
// The endpoint relay is runtime-local and uses a fresh tmpfs key. The key
// never crosses the control plane and Hermes still binds only loopback.
apiServerKeyRequired: true,
env: {
...adminToolEnv(adminHomeLayout(profile.profileHome)),
...adminAgentServiceEnvironment(claim),
// Keep model/tool credentials in the root-owned runtime directory, but
// place ordinary terminal output under the durable profile. Hermes
// denies /run as an attachment source, so using SECRET_ROOT as HOME made
// valid MEDIA directives degrade into text-only Slack messages.
HOME: profile.profileHome,
HERMES_HOME: DATA_ROOT,
HERMES_WRITE_SAFE_ROOT: DATA_ROOT,
SELLABLE_HERMES_PROFILE_ID: profile.profile,
CODEX_HOME: SECRET_ROOT,
NO_COLOR: "1",
},
});
const appFd = openSync(
launch.inheritedFileDescriptors.find(({ fd }) => fd === 8).path,
"r"
);
const botFd = openSync(
launch.inheritedFileDescriptors.find(({ fd }) => fd === 9).path,
"r"
);
const apiServerKey = readFileSync(launch.apiServerKeyFile, "utf8");
if (!/^[a-f0-9]{64}$/.test(apiServerKey)) {
closeSync(appFd);
closeSync(botFd);
throw new Error("admin_endpoint_api_key_rejected");
}
let child;
try {
child = spawn(HERMES, launch.args, {
cwd: launch.cwd,
env: { ...launch.env, API_SERVER_KEY: apiServerKey },
stdio: [
"ignore",
"ignore",
"ignore",
"ignore",
"ignore",
"ignore",
"ignore",
"ignore",
appFd,
botFd,
],
});
} finally {
closeSync(appFd);
closeSync(botFd);
}
const closed = new Promise((resolve) =>
child.once("close", (status, signal) => resolve({ status, signal }))
);
const early = await Promise.race([closed, sleep(5_000).then(() => null)]);
if (early) throw new Error("admin_native_gateway_start_rejected");
return { child, closed };
}
async function stopNativeGateway(gateway) {
if (!nativeGatewayRunning(gateway)) return;
gateway.child.kill("SIGTERM");
const stopped = await Promise.race([
gateway.closed.then(() => true),
sleep(10_000).then(() => false),
]);
if (!stopped && gateway.child.exitCode === null) {
gateway.child.kill("SIGKILL");
await gateway.closed;
}
}
async function recoverUnexpectedNativeGateway({
activeLaunch,
activeGateway,
activeModelHealthReporter,
identity,
exchange,
attempts,
}) {
const recovery = nextAdminNativeGatewayRecovery({ attempts });
await activeModelHealthReporter?.stop();
await stopNativeGateway(activeGateway);
if (recovery.action === "EXHAUSTED") {
process.stderr.write(
`${JSON.stringify({
ok: false,
code: "admin_native_gateway_restart_exhausted",
attempts: recovery.attempt,
manualRetryAvailable: true,
})}\n`
);
return {
activeGateway: null,
activeModelHealthReporter: null,
attempts: recovery.attempt,
exhausted: true,
};
}
let candidateGateway = null;
try {
process.stderr.write(
`${JSON.stringify({
ok: false,
code: "admin_native_gateway_exit_detected",
attempt: recovery.attempt,
})}\n`
);
candidateGateway = await startNativeGateway(activeLaunch);
if (!nativeGatewayRunning(candidateGateway)) {
throw new Error("admin_native_gateway_local_restart_failed");
}
const nextModelHealthReporter = adminModelHealthReporter({
identity,
exchange,
installed: activeLaunch.installed,
claim: activeLaunch.claim,
});
try {
await nextModelHealthReporter.start("gateway-recovered");
} catch (error) {
process.stderr.write(
`${JSON.stringify({
ok: false,
code:
error instanceof Error
? error.message
: "model_health_report_failed",
})}\n`
);
}
process.stderr.write(
`${JSON.stringify({
ok: true,
code: "admin_native_gateway_local_restart",
attempt: recovery.attempt,
processIdentity: `gateway-${candidateGateway.child.pid}-${activeLaunch.claim.operation.fence}`,
})}\n`
);
return {
activeGateway: candidateGateway,
activeModelHealthReporter: nextModelHealthReporter,
attempts: recovery.attempt,
exhausted: false,
};
} catch (error) {
await stopNativeGateway(candidateGateway);
process.stderr.write(
`${JSON.stringify({
ok: false,
code:
error instanceof Error
? error.message
: "admin_native_gateway_local_restart_failed",
attempt: recovery.attempt,
})}\n`
);
return {
activeGateway: null,
activeModelHealthReporter: null,
attempts: recovery.attempt,
exhausted: false,
};
}
}
export async function runAdminCanary() {
const step = process.env.SELLABLE_AGENT_ADMIN_CANARY_STEP;
const recoverySecretPath =
"/run/secrets/sellable-admin-recovery-private-key.pem";
if (
process.env.SELLABLE_AGENT_SERVER_AUTHENTICATED_KIND !== "ADMIN" ||
!["initial", "restore"].includes(step) ||
Object.prototype.hasOwnProperty.call(
process.env,
"SELLABLE_ADMIN_RECOVERY_PRIVATE_KEY_PEM"
) ||
!existsSync(recoverySecretPath)
)
throw new Error("admin_runtime_control_claim_required");
if (process.env.SELLABLE_AGENT_ADMIN_RUNTIME_MODE !== "canary")
throw new Error("admin_runtime_mode_rejected");
const recoverySecretBeforeDrop = lstatSync(recoverySecretPath);
if (
recoverySecretBeforeDrop.isSymbolicLink() ||
!recoverySecretBeforeDrop.isFile() ||
(recoverySecretBeforeDrop.mode & 0o777) !== 0o400
) {
throw new Error("admin_canary_recovery_secret_file_rejected");
}
if (process.getuid?.() === 0) {
dropFlyRuntimePrivileges({
user: "hermes",
group: "hermes",
beforeSetuid: ({ gid }) =>
prepareSoulLockRoot({ dataRoot: DATA_ROOT, gid }),
});
}
const runtimeUid = process.getuid?.() ?? 0;
const runtimeGid = process.getgid?.() ?? 0;
if (
runtimeUid < 1 ||
runtimeGid < 1 ||
recoverySecretBeforeDrop.uid !== runtimeUid ||
recoverySecretBeforeDrop.gid !== runtimeGid
) {
throw new Error("admin_canary_privilege_drop_rejected");
}
const packages = await proveInstalledPackages();
const appName = process.env.FLY_APP_NAME ?? "";
const machineId = process.env.FLY_MACHINE_ID ?? "";
if (
!/^slb-admin-[a-f0-9]{24}$/.test(appName) ||
!/^[a-f0-9]{14}$/.test(machineId)
) {
throw new Error("admin_canary_fly_identity_rejected");
}
privateDirectory(SECRET_ROOT);
const canaryRoot = join(DATA_ROOT, ".sellable-agent", "admin-canary");
const sourceRoot = join(SECRET_ROOT, "canary-source");
const initialSettingsRoot = join(canaryRoot, "settings-initial");
const restoredSettingsRoot = join(canaryRoot, "settings-restored");
const recoveryBundlePath = join(canaryRoot, "recovery.bundle");
const statePath = join(canaryRoot, "state.json");
privateDirectory(canaryRoot);
privateDirectory(sourceRoot);
for (const name of [".env", "settings.json", "config-state.json"]) {
rmSync(join(sourceRoot, name), { force: true });
}
const fixtureWorkspaceId = "phase158-canary-workspace";
const fixtureAgentId = "phase158-canary-admin-agent";
const fixtureFiles = {
".env": "PHASE158_ADMIN_CANARY=nonsecret\n",
"settings.json": `${JSON.stringify({
sellable: {
token: "skt_live_phase158_canary_nonsecret",
apiUrl: "https://example.invalid",
workspaceIds: [],
workspaceNames: {},
},
})}\n`,
"config-state.json": `${JSON.stringify({
sellable: { workspaceIds: [], workspaceNames: {} },
opportunitySequences: { notificationSlackUserId: "U00000000" },
clientRelationships: { shermeenSlackUserId: "U00000000" },
clients: {
"phase193-canary-client": {
status: "active",
workspaceId: fixtureWorkspaceId,
channels: { shared: "C00000000CANARY" },
},
},
})}\n`,
};
for (const [name, value] of Object.entries(fixtureFiles)) {
atomicPrivateFile(join(sourceRoot, name), value);
}
const rootAuthPath = join(DATA_ROOT, "auth.json");
const modelMarkerPath = join(canaryRoot, "model-auth-source.json");
const productConfigPath = join(SECRET_ROOT, "canary-product-config.json");
rmSync(rootAuthPath, { force: true });
rmSync(modelMarkerPath, { force: true });
rmSync(productConfigPath, { force: true });
const canaryExpiresAt = new Date(Date.now() + 60 * 60 * 1000);
const canaryJwt = (subject) =>
[
Buffer.from(JSON.stringify({ alg: "none", typ: "JWT" })).toString(
"base64url"
),
Buffer.from(
JSON.stringify({
sub: subject,
exp: Math.floor(canaryExpiresAt / 1000),
})
).toString("base64url"),
"phase158-canary-signature-placeholder",
].join(".");
const canaryModel = parseModelAuthRuntimeCredential(
JSON.stringify({
schemaVersion: "sellable-agent-model-auth-runtime/v2",
activationId: "phase158-canary-model-auth",
bundleGeneration: 1,
policyHash: "a".repeat(64),
credentialPoolStrategy: "fill_first",
runtime: { apiMaxRetries: 0, requestTimeoutSeconds: 60 },
codex: [
{
slotId: "dittto-primary",
priority: 1,
accessToken: canaryJwt("phase158-canary-dittto"),
accountId: "phase158-canary-dittto",
expiresAt: canaryExpiresAt.toISOString(),
grantFingerprint: "a".repeat(16),
},
{
slotId: "kaigo-secondary",
priority: 2,
accessToken: canaryJwt("phase158-canary-kaigo"),
accountId: "phase158-canary-kaigo",
expiresAt: canaryExpiresAt.toISOString(),
grantFingerprint: "b".repeat(16),
},
],
openrouter: {
apiKey: "phase158-canary-openrouter-placeholder",
keyFingerprint: "c".repeat(16),
models: [
"moonshotai/kimi-k3",
"~deepseek/deepseek-v4-flash-latest",
"z-ai/glm-5.2",
],
},
})
);
const canaryModelAuthState = reconcileHermesModelAuth({
authStorePath: rootAuthPath,
markerPath: modelMarkerPath,
credential: canaryModel,
lifecycle: "ON",
});
atomicPrivateFile(
productConfigPath,
`${JSON.stringify({
apiUrl: "https://example.invalid",
token: "skt_live_phase158_canary_nonsecret",
activeWorkspaceId: fixtureWorkspaceId,
workspaceId: fixtureWorkspaceId,
})}\n`
);
const canaryClaim = {
agentKind: "ADMIN",
adminRuntimeGeneration: {
id: "admin-generation-phase158-canary",
agentKind: "ADMIN",
workspaceId: fixtureWorkspaceId,
agentId: fixtureAgentId,
fence: "7",
},
operation: {
workspaceId: fixtureWorkspaceId,
agentId: fixtureAgentId,
profileId: "phase158-admin-canary",
fence: "phase158-canary-operation-fence-a",
},
runtimeContract: {
customerProgram: { timezone: "UTC" },
service: {
principalId: "phase158-canary-principal",
policyHash: "a".repeat(64),
toolInclude: ["get_workspace"],
credentialGeneration: 1,
},
slack: {
selectedBindings: [{ channelId: "C_PHASE158_CANARY" }],
},
},
};
const agentServiceEnvironment = adminAgentServiceEnvironment(canaryClaim);
await publishManagedAuth({
schemaVersion: "sellable-managed-auth/v1",
profileId: canaryClaim.operation.profileId,
runtimeId: RUNTIME_ID,
runtimeGeneration: RUNTIME_GENERATION,
agentKind: "ADMIN",
apiUrl: ORIGIN,
token: "sat_canary_phase158-nonsecret",
agent: {
workspaceId: fixtureWorkspaceId,
agentId: fixtureAgentId,
generation: canaryClaim.runtimeContract.service.credentialGeneration,
policyHash: canaryClaim.runtimeContract.service.policyHash,
selectedChannelIds:
canaryClaim.runtimeContract.slack.selectedBindings.map(
(binding) => binding.channelId
),
},
});
const template = JSON.parse(readFileSync(ADMIN_PROFILE_TEMPLATE_PATH));
const { contentDigest, ...templateContent } = template;
if (
template.schemaVersion !== "sellable-admin-profile-contract-template/v1" ||
!/^[a-f0-9]{64}$/.test(contentDigest ?? "") ||
flyRuntimeDigest(templateContent) !== contentDigest
) {
throw new Error("admin_canary_profile_template_rejected");
}
const unsignedProfileContract = {
schemaVersion: "sellable-admin-profile-contract/v1",
profileHome: join(PROFILES_ROOT, "phase158-admin-canary"),
aliases: template.aliases,
packages: template.packages,
tools: template.tools,
contentDigest,
};
const profileContract = {
...unsignedProfileContract,
digest: flyRuntimeDigest(unsignedProfileContract),
};
const sourceTarget = {
workspaceId: fixtureWorkspaceId,
agentId: fixtureAgentId,
agentKind: "ADMIN",
deliveryMode: "INITIAL_BOOTSTRAP",
flyAppName: appName,
machineId,
machineVersion: "phase158-canary-initial-v1",
workloadKeyHash: sha256(
`${appName}\0${machineId}\0phase158-canary-workload`
),
runtimeGenerationId: "phase158-canary-generation-a",
generationFence: "phase158-canary-fence-a",
};
let settingsRoot;
let transition;
if (step === "initial") {
privateDirectory(initialSettingsRoot);
const built = buildAdminSettingsManifest({
adminHome: sourceRoot,
target: sourceTarget,
});
const bundle = encodeAdminSettingsBundle(built);
const operationId = "phase158-canary-source-operation";
const imported = await importAdminSecretGeneration(
{
schemaVersion: "AdminSecretImportRequestV1",
operationId,
target: sourceTarget,
ciphertextDigest: sha256(bundle),
ciphertextBytes: bundle.byteLength,
plaintextDigest: sha256(bundle),
settingsDigest: built.manifest.settingsDigest,
targetRoot: initialSettingsRoot,
expectedUid: runtimeUid,
expectedGid: runtimeGid,
},
{
reattestTarget: async () => sourceTarget,
claimCiphertext: async () => bundle,
decrypt: async (ciphertext) => ciphertext,
resealRecovery: async () => {
atomicPrivateFile(recoveryBundlePath, bundle);
return {
keyId: "phase158-canary-recovery-key",
ciphertextDigest: sha256(bundle),
ciphertextBytes: bundle.byteLength,
locator: "private://phase158-admin-canary/recovery.bundle",
};
},
publishGeneration: async ({ settingsDigest, manifestDigest }) => ({
ok: true,
fence: "phase158-canary-published-fence-a",
settingsDigest,
manifestDigest,
}),
cleanupCiphertext: async () => undefined,
}
);
const state = {
schemaVersion: "phase158-admin-canary-state/v1",
sourceOperationId: operationId,
sourceTarget,
settingsDigest: built.manifest.settingsDigest,
manifestDigest: built.manifest.manifestDigest,
recoveryCiphertextDigest: sha256(bundle),
sourceMachineId: machineId,
};
atomicPrivateFile(statePath, `${JSON.stringify(state)}\n`);
settingsRoot = imported.generationRoot;
transition = {
kind: "initial_import",
sourceOperationId: operationId,
sourceMachineId: machineId,
successorMachineId: null,
idempotent: imported.idempotent,
settingsDigest: imported.settingsDigest,
manifestDigest: imported.manifestDigest,
recoveryCiphertextDigest: imported.recovery.ciphertextDigest,
};
} else {
if (!existsSync(statePath) || !existsSync(recoveryBundlePath)) {
throw new Error("admin_canary_recovery_source_missing");
}
const state = JSON.parse(readFileSync(statePath, "utf8"));
const bundle = readFileSync(recoveryBundlePath);
if (
state.schemaVersion !== "phase158-admin-canary-state/v1" ||
state.sourceTarget?.workspaceId !== fixtureWorkspaceId ||
state.sourceTarget?.agentId !== fixtureAgentId ||
state.recoveryCiphertextDigest !== sha256(bundle) ||
state.sourceMachineId === machineId
) {
throw new Error("admin_canary_recovery_source_rejected");
}
privateDirectory(restoredSettingsRoot);
const restored = restoreAdminSecretGeneration({
schemaVersion: "AdminRecoveryRestoreRequestV1",
recoverySourceOperationId: state.sourceOperationId,
workspaceId: fixtureWorkspaceId,
agentId: fixtureAgentId,
successorGenerationId: "phase158-canary-generation-b",
successorGenerationFence: "phase158-canary-fence-b",
settingsDigest: state.settingsDigest,
targetRoot: restoredSettingsRoot,
plaintext: bundle,
expectedUid: runtimeUid,
expectedGid: runtimeGid,
});
settingsRoot = restored.generationRoot;
transition = {
kind: "fresh_machine_restore",
sourceOperationId: state.sourceOperationId,
sourceMachineId: state.sourceMachineId,
successorMachineId: machineId,
idempotent: restored.idempotent,
settingsDigest: restored.settingsDigest,
manifestDigest: restored.manifestDigest,
recoveryCiphertextDigest: state.recoveryCiphertextDigest,
};
}
const soulBytes = Buffer.from(
"Phase 158 disposable Admin canary. No customer traffic. No Slack traffic.\n"
);
const profile = await materializeAdminProfile({
profileContract,
settingsRoot,
rootAuthPath,
modelConfig: modelAuthConfigOverlay(canaryModel),
productConfigPath,
soul: {
bytes: soulBytes,
sha256: sha256(soulBytes),
byteLength: soulBytes.byteLength,
},
agentServiceEnv: agentServiceEnvironment,
customerProgram: canaryClaim.runtimeContract.customerProgram,
});
const soulProof = observeProfileSoul(profile.profile, {
dataRoot: DATA_ROOT,
});
if (soulProof.ok !== true) {
throw new Error("admin_runtime_live_soul_loader_proof_rejected");
}
profile.observedSoulRawSha256 = soulProof.rawFileSha256;
const modelAuth = proveAdminModelAuth(
profile,
{
rootAuthPath,
markerPath: modelMarkerPath,
model: canaryModel,
modelAuthState: canaryModelAuthState,
},
"ON"
);
const attachmentHandoff = proveAdminAttachmentHandoff(profile);
const observedSettings = settingsProof(
settingsRoot,
transition.settingsDigest
);
const profileStat = lstatSync(profile.profileHome);
const profileConfigStat = lstatSync(join(profile.profileHome, "config.yaml"));
const recoverySecretAfterDrop = lstatSync(recoverySecretPath);
const receipt = {
schemaVersion: "phase158-admin-canary-runtime/v1",
kind: "phase158-admin-canary-runtime",
step,
appName,
machineId,
runtimeUid,
runtimeGid,
serverAuthenticatedKind: "ADMIN",
packages,
profileFactory: "stageAdminProfile",
profileDigest: profileContract.digest,
servers: profile.servers,
opportunitySequencesCron: profile.opportunitySequencesCron,
clientRelationshipsCrons: profile.clientRelationshipsCrons,
workspaceAuthority: {
agentKind: canaryClaim.agentKind,
workspaceLockPresent:
Object.prototype.hasOwnProperty.call(
agentServiceEnvironment,
"SELLABLE_LOCK_WORKSPACE_ID"
) ||
Object.prototype.hasOwnProperty.call(
agentServiceEnvironment,
"SELLABLE_REQUIRE_WORKSPACE_LOCK"
),
},
modelAuth,
attachmentHandoff,
settings: observedSettings,
adminHome: profile.adminHome,
transition,
permissions: {
profileMode: (profileStat.mode & 0o777).toString(8).padStart(4, "0"),
profileConfigMode: (profileConfigStat.mode & 0o777)
.toString(8)
.padStart(4, "0"),
recoverySecretMode: (recoverySecretAfterDrop.mode & 0o777)
.toString(8)
.padStart(4, "0"),
recoverySecretOwnerMatchesRuntime:
recoverySecretAfterDrop.uid === runtimeUid &&
recoverySecretAfterDrop.gid === runtimeGid,
recoverySecretEnvironmentPresent: false,
},
lockedTreeDigest:
process.env.SELLABLE_AGENT_ADMIN_LOCKED_TREE_DIGEST ?? null,
artifactDigest: process.env.SELLABLE_AGENT_ADMIN_ARTIFACT_DIGEST ?? null,
customerRouting: "disabled",
slackRouting: "disabled",
};
const receiptDigest = flyRuntimeDigest(receipt);
await mkdir(dirname(RECEIPT_PATH), { recursive: true });
await writeFile(
RECEIPT_PATH,
`${JSON.stringify({ ...receipt, receiptDigest }, null, 2)}\n`,
{ mode: 0o600 }
);
process.stdout.write(
`${JSON.stringify({
ok: true,
code: "admin_canary_ready",
step,
machineId,
receiptDigest,
})}\n`
);
if (process.env.SELLABLE_AGENT_ADMIN_CANARY_HOLD === "1") {
await new Promise((resolve) => {
const keepalive = setInterval(() => undefined, 60_000);
const stop = () => {
clearInterval(keepalive);
resolve();
};
process.once("SIGINT", stop);
process.once("SIGTERM", stop);
});
}
return { ...receipt, receiptDigest };
}
async function runAdminProduction() {
if (
Object.prototype.hasOwnProperty.call(
process.env,
"SELLABLE_ADMIN_RECOVERY_PRIVATE_KEY_PEM"
) ||
process.env.SELLABLE_AGENT_SERVER_AUTHENTICATED_KIND !== "ADMIN" ||
!/^fly-runtime_[a-f0-9]{32}$/.test(RUNTIME_ID) ||
!ORIGIN ||
!AUDIENCE ||
!Number.isSafeInteger(RUNTIME_GENERATION) ||
RUNTIME_GENERATION < 1
) {
throw new Error("admin_runtime_configuration_rejected");
}
const scratch = pruneAbandonedRuntimeScratch({
hermesRoot: DATA_ROOT,
profilesRoot: PROFILES_ROOT,
});
if (scratch.removed.length > 0) {
process.stderr.write(
`${JSON.stringify({
ok: true,
code: scratch.code,
removed: scratch.removed,
})}\n`
);
}
await sodium.ready;
await proveInstalledPackages();
let oidcToken = await obtainFlyOidcToken({ audience: AUDIENCE });
if (process.getuid?.() === 0) {
dropFlyRuntimePrivileges({
user: "hermes",
group: "hermes",
beforeSetuid: ({ gid }) => {
prepareSoulLockRoot({ dataRoot: DATA_ROOT, gid });
if (process.env.SELLABLE_AGENT_ALLOW_TEST_OIDC !== "1") {
handoffFlyOidcSocket({ gid });
}
},
});
} else if ((process.getuid?.() ?? 0) < 1 || (process.getgid?.() ?? 0) < 1) {
throw new Error("admin_runtime_privilege_rejected");
}
privateDirectory(RUNTIME_ROOT);
privateDirectory(SECRET_ROOT);
const identity = loadOrCreateFlyRuntimeIdentity({
identityRoot: join(RUNTIME_ROOT, "identity"),
persistent: true,
});
let exchange = await exchangeFlyAdminIdentity({
origin: ORIGIN,
runtimeId: RUNTIME_ID,
audience: AUDIENCE,
identity,
bootSessionId: BOOT_SESSION_ID,
oidcToken,
});
oidcToken = null;
if (
exchange.keyGeneration !== RUNTIME_GENERATION ||
exchange.adminDeliveryTarget.runtimeGeneration !== RUNTIME_GENERATION
) {
throw new Error("admin_runtime_generation_rejected");
}
const deliveryKey = loadOrCreateDeliveryKey();
await enrollDeliveryKey({ identity, exchange, deliveryKey });
let client = createFlyCustomerControlClient({
origin: ORIGIN,
identity,
exchange,
renewExchange: async () =>
(
await renewFlyAdminControlSession({
origin: ORIGIN,
runtimeId: RUNTIME_ID,
audience: AUDIENCE,
identity,
bootSessionId: BOOT_SESSION_ID,
runtimeGeneration: RUNTIME_GENERATION,
adminRuntimeGenerationId:
exchange.adminDeliveryTarget.adminRuntimeGenerationId,
adminRuntimeGenerationFence:
exchange.adminDeliveryTarget.adminRuntimeGenerationFence,
})
).exchange,
});
const controller = new AbortController();
let activeGateway = null;
let activeLaunch = null;
let activeModelHealthReporter = null;
let shutdownMemorySnapshot = null;
let customRunsServices = null;
try {
customRunsServices = await startConfiguredClaudeRunsApis({
diagnostic: (value) => process.stderr.write(`${JSON.stringify(value)}\n`),
toolLanes: {
turnDescriptorPath: join(MCP_TURN_CONTEXT_ROOT, "turn-context.json"),
server: () =>
activeLaunch
? profileSellableMcpServer(activeLaunch.profile.profileHome)
: null,
},
});
} catch {
process.stderr.write(
`${JSON.stringify({ ok: false, code: "custom_runs_api_start_failed" })}\n`
);
}
let publicRunsService = null;
try {
publicRunsService = await startPublicClaudeRunsApi({
diagnostic: (value) => process.stderr.write(`${JSON.stringify(value)}\n`),
});
} catch {
process.stderr.write(
`${JSON.stringify({ ok: false, code: "public_runs_api_start_failed" })}\n`
);
}
const endpointRelay = createCustomerEndpointRelay({
getControlClient: () => client,
getActiveRuntime: () =>
projectAdminEndpointRuntime({
activeLaunch,
activeGateway,
runtimeGeneration: RUNTIME_GENERATION,
trustRootPem: readMcpTurnTrustRoot(),
turnDescriptorPath: join(MCP_TURN_CONTEXT_ROOT, "turn-context.json"),
}),
runtimeGeneration: RUNTIME_GENERATION,
getApiKeyFile: () => join(SECRET_ROOT, "api-server-key"),
diagnostic: (value) => process.stderr.write(`${JSON.stringify(value)}\n`),
});
const endpointLoop = endpointRelay.run(controller.signal);
const beginShutdownSnapshot = () => {
if (!activeLaunch || shutdownMemorySnapshot) return;
const launch = activeLaunch;
const gateway = activeGateway;
shutdownMemorySnapshot = (async () => {
try {
await endpointRelay.fenceActive("RUNTIME_SHUTDOWN");
await stopNativeGateway(gateway);
if (activeGateway === gateway) activeGateway = null;
return await flushHermesSnapshot({
client,
profileId: launch.profile.profile,
profileRoot: launch.profile.profileHome,
});
} catch {
return {
ok: false,
status: "PENDING_RETRY",
code: "hermes_snapshot_shutdown_failed",
};
}
})();
};
const stop = () => {
controller.abort();
beginShutdownSnapshot();
};
process.once("SIGINT", stop);
process.once("SIGTERM", stop);
let operationHeartbeat = null;
let soulClaimTracker = null;
let gatewayRestartAttempts = 0;
let gatewayRestartExhaustionReported = false;
let failures = 0;
while (!controller.signal.aborted) {
let claimedOperation = null;
let claimSecretsConsumed = false;
try {
let claim;
try {
claim =
activeLaunch?.profile?.profile && soulClaimTracker
? await claimWithSoulObservation({
client,
tracker: soulClaimTracker,
readSoul: (request) =>
executeExternalSoulBridgeOperation(request, {
dataRoot: DATA_ROOT,
callerProfileId: activeLaunch.profile.profile,
}),
leaseSeconds: 60,
servingObservation: adminServingObservation(
activeLaunch,
activeGateway,
exchange
),
})
: await client.claim(60);
} catch (error) {
if (!isFlyCustomerEnrollmentRenewalError(error)) throw error;
const renewed = await renewFlyAdminControlSession({
origin: ORIGIN,
runtimeId: RUNTIME_ID,
audience: AUDIENCE,
identity,
bootSessionId: BOOT_SESSION_ID,
runtimeGeneration: RUNTIME_GENERATION,
adminRuntimeGenerationId:
exchange.adminDeliveryTarget.adminRuntimeGenerationId,
adminRuntimeGenerationFence:
exchange.adminDeliveryTarget.adminRuntimeGenerationFence,
});
exchange = renewed.exchange;
client = renewed.client;
await activeModelHealthReporter?.stop();
activeModelHealthReporter = null;
if (activeLaunch) {
activeModelHealthReporter = adminModelHealthReporter({
identity,
exchange,
installed: activeLaunch.installed,
claim: activeLaunch.claim,
});
activeModelHealthReporter.start("periodic").catch((reportError) => {
process.stderr.write(
`${JSON.stringify({
ok: false,
code:
reportError instanceof Error
? reportError.message
: "model_health_report_failed",
})}\n`
);
});
}
failures = 0;
process.stderr.write(
`${JSON.stringify({
ok: true,
code: "admin_enrollment_renewed",
keyGeneration: exchange.keyGeneration,
})}\n`
);
continue;
}
if (controller.signal.aborted) break;
if (!claim) {
if (activeLaunch) {
if (
!nativeGatewayRunning(activeGateway) &&
!gatewayRestartExhaustionReported
) {
await endpointRelay.fenceActive("GATEWAY_RESTART");
const recovery = await recoverUnexpectedNativeGateway({
activeLaunch,
activeGateway,
activeModelHealthReporter,
identity,
exchange,
attempts: gatewayRestartAttempts,
});
activeGateway = recovery.activeGateway;
activeModelHealthReporter = recovery.activeModelHealthReporter;
gatewayRestartAttempts = recovery.attempts;
gatewayRestartExhaustionReported = recovery.exhausted;
}
await reconcileHermesSnapshot({
profileId: activeLaunch.profile.profile,
profileRoot: activeLaunch.profile.profileHome,
hermesRoot: DATA_ROOT,
hermesVersion: "0.21.5",
upload: ({ archiveDigest, archiveBytes, sourceProfileId, bytes }) =>
client.putMemorySnapshot(
{
archiveDigest,
archiveBytes,
hermesVersion: "0.21.5",
sourceProfileId,
},
bytes
),
});
void sessionRetention.tick({
apiKeyFile: join(SECRET_ROOT, "api-server-key"),
statePath: join(
activeLaunch.profile.profileHome,
".sellable-agent",
"session-retention.json"
),
});
}
await sleep(1_000);
continue;
}
claimedOperation = claim;
if (claim.agentKind !== "ADMIN") {
throw new Error("admin_runtime_claim_rejected");
}
if (isOperationBoundBackupClaim(claim)) {
const snapshotIdentity = operationSnapshotIdentity(claim, exchange);
if (!snapshotIdentity) {
throw new Error("admin_operation_backup_claim_rejected");
}
operationHeartbeat = startOperationHeartbeat({
claim,
exchange,
client,
});
const profile = activeLaunch?.profile ?? profileContractForClaim(claim);
const backup = await flushHermesSnapshot({
client,
profileId: basename(profile.profileHome),
profileRoot: profile.profileHome,
operationIdentity: snapshotIdentity,
forceDirty: true,
});
if (!backup.ok || backup.status !== "UPLOADED") {
throw new Error(
backup.code ?? "admin_operation_backup_proof_unavailable"
);
}
// BACKUP advances the Agent authority generation without replacing the
// running gateway. Rebind its serving proof to the snapshot operation;
// otherwise idle claim ticks keep presenting the prior generation and
// Sellable correctly expires the otherwise-healthy Slack serving lease.
if (activeLaunch) activeLaunch = { ...activeLaunch, claim };
operationHeartbeat.assertHealthy();
await operationHeartbeat.stop();
operationHeartbeat = null;
claimedOperation = null;
failures = 0;
continue;
}
if (
claim.adminRuntimeGeneration?.id !==
exchange.adminDeliveryTarget.adminRuntimeGenerationId ||
claim.adminRuntimeGeneration.fence !==
exchange.adminDeliveryTarget.adminRuntimeGenerationFence
) {
throw new Error("admin_runtime_claim_rejected");
}
operationHeartbeat = startOperationHeartbeat({
claim,
exchange,
client,
});
const phaseTelemetry = createAdminRuntimePhaseTelemetry({
bootSessionId: BOOT_SESSION_ID,
operationId: claim.operation.id,
});
await activeModelHealthReporter?.stop();
activeModelHealthReporter = null;
await endpointRelay.fenceActive("LIFECYCLE_REFRESH");
await stopNativeGateway(activeGateway);
activeGateway = null;
activeLaunch = null;
await clearRuntimeSecrets();
let operation = await pollDelivery({ identity, exchange });
let imported;
let profileContract;
let recovery;
let claimId = null;
if (operation) {
imported = await importDelivery({
operation,
identity,
exchange,
deliveryKey,
generation: claim.adminRuntimeGeneration,
});
profileContract = operation.profileContract;
recovery = imported.recovery;
claimId = imported.claimId;
} else if (
claim.adminRuntimeGeneration.state === "ACTIVE" &&
claim.adminRuntimeGeneration.settingsDigest &&
claim.adminRuntimeGeneration.recoveryKeyId &&
claim.adminRuntimeGeneration.recoveryPublicKey &&
claim.adminRuntimeGeneration.recoveryCiphertextDigest &&
claim.adminRuntimeGeneration.recoveryLocator
) {
recoveryKey(claim.adminRuntimeGeneration);
const generationRoot = join(
RUNTIME_ROOT,
"settings",
"generations",
claim.adminRuntimeGeneration.settingsDigest
);
imported = {
receipt: {
generationRoot,
settingsDigest: claim.adminRuntimeGeneration.settingsDigest,
},
};
profileContract = profileContractForClaim(claim);
recovery = {
recoveryKeyId: claim.adminRuntimeGeneration.recoveryKeyId,
ciphertextDigest:
claim.adminRuntimeGeneration.recoveryCiphertextDigest,
locator: claim.adminRuntimeGeneration.recoveryLocator,
};
} else if (
claim.adminRuntimeGeneration.state === "PROVING" &&
claim.adminRuntimeGeneration.recoverySourceOperationId
) {
imported = await restoreSuccessorSettings({
claim,
identity,
exchange,
});
profileContract = profileContractForClaim(claim);
recovery = imported.recovery;
} else {
if (
claim.adminRuntimeGeneration.state === "PREPARING" &&
!claim.adminRuntimeGeneration.recoverySourceOperationId
) {
throw new Error("admin_initial_settings_delivery_required");
}
throw new Error("admin_runtime_settings_authority_rejected");
}
// Authority is fully decidable from the claim and durable settings
// metadata. Refuse it before consuming any lease secret so the failure
// can be reported as exact and retryable instead of uncertain.
const desiredLifecycle = desiredLifecycleForClaim(claim);
const { secrets, installed } = await phaseTelemetry.run(
"runtime_secrets",
async () => {
const secrets = await consumeRuntimeSecrets({
claim,
exchange,
client,
});
claimSecretsConsumed = true;
const installed = await installRuntimeSecrets({
claim,
secrets,
desiredLifecycle,
});
return { secrets, installed };
}
);
const proofOperation = operation ?? {
runtimeGenerationId: claim.adminRuntimeGeneration.id,
generationFence: claim.adminRuntimeGeneration.fence,
profileContract,
};
const memoryRestore = await phaseTelemetry.run(
"snapshot_restore",
async () =>
restoreLatestHermesSnapshot({
profileId: basename(profileContract.profileHome),
profileRoot: profileContract.profileHome,
hermesRoot: DATA_ROOT,
hermesVersion: "0.21.5",
fetchLatest: () => client.getMemorySnapshot(),
})
);
if (!memoryRestore.ok) throw new Error(memoryRestore.code);
await operationHeartbeat.reattest();
const legacyLightfieldVenv = quarantineLegacyLightfieldVenv({
profileId: basename(profileContract.profileHome),
profileRoot: profileContract.profileHome,
hermesRoot: DATA_ROOT,
});
if (!legacyLightfieldVenv.ok) {
throw new Error(legacyLightfieldVenv.code);
}
const legacyUserState = migrateLegacyHermesUserState({
profileId: basename(profileContract.profileHome),
profileRoot: profileContract.profileHome,
hermesRoot: DATA_ROOT,
});
if (!legacyUserState.ok) throw new Error(legacyUserState.code);
const priorMaterializerReceipt = adminMaterializerReceipt(
basename(profileContract.profileHome),
profileContract.profileHome
);
const profile = await materializeAdminProfile({
profileContract,
settingsRoot: imported.receipt.generationRoot,
rootAuthPath: installed.profileAuthPath,
modelConfig: installed.modelConfig,
productConfigPath: installed.productConfigPath,
soul: exactSoul(claim),
slack: nativeSlackProfileInput(claim, installed),
customerProgram: claim.runtimeContract.customerProgram,
agentServiceEnv: adminAgentServiceEnvironment(claim),
runPhase: phaseTelemetry.run,
});
await phaseTelemetry.run("product_mcp_readiness", async () =>
proveAdminProductMcpReadiness({ profile, claim, installed })
);
await operationHeartbeat.reattest();
const materializerReceipt = adminMaterializerReceipt(
profile.profile,
profile.profileHome
);
const activeProjection = buildDefaultProfileProjection({
kind: "ADMIN",
claim,
});
// Refresh image-owned public skills before default-bundle verification,
// including upgrades from profiles without receipts for the shared skills.
const publicSkills = reconcilePublicHermesSkills({
profileId: profile.profile,
profileRoot: profile.profileHome,
dataRoot: DATA_ROOT,
});
const reconciledDefaultProfile = await phaseTelemetry.run(
"default_profile_bundle",
async () =>
reconcileDefaultProfileBundle({
activeProjection,
materializerReceipt,
profileRoot: profile.profileHome,
bundleRoot: DEFAULT_PROFILE_BUNDLE_ROOT,
dataRoot: DATA_ROOT,
reattestAuthority: async () => {
await operationHeartbeat.reattest();
return activeProjection;
},
cronAdapter: adminDefaultProfileCronAdapter(
profile.profile,
profile.profileHome
),
})
);
if (!reconciledDefaultProfile.ok)
throw new Error(reconciledDefaultProfile.code);
const defaultProfile = reconciledDefaultProfile;
const soulProof = observeProfileSoul(profile.profile, {
dataRoot: DATA_ROOT,
});
if (soulProof.ok !== true) {
throw new Error("admin_runtime_live_soul_loader_proof_rejected");
}
profile.observedSoulRawSha256 = soulProof.rawFileSha256;
const modelAuth = proveAdminModelAuth(
profile,
installed,
desiredLifecycle
);
const materializerChanged =
canonicalJson(priorMaterializerReceipt) !==
canonicalJson(materializerReceipt) ||
profile.memoryCron.status !== "REUSED" ||
profile.opportunitySequencesCron.status !== "REUSED" ||
profile.clientRelationshipsCrons.status !== "REUSED" ||
profile.customerProgramCrons.status !== "REUSED";
if (materializerChanged || defaultProfile.changed || publicSkills.changed)
markHermesSnapshotDirty(profile.profileHome);
const memorySnapshot = await phaseTelemetry.run(
"memory_snapshot",
async () =>
flushHermesSnapshot({
client,
profileId: profile.profile,
profileRoot: profile.profileHome,
})
);
await operationHeartbeat.reattest();
const settings = settingsProof(
imported.receipt.generationRoot,
imported.receipt.settingsDigest
);
const profileProofDigest = flyRuntimeDigest({
profile: profileContract,
settings,
recovery,
servers: profile.servers,
});
if (operation) {
await acknowledgeDelivery({
operation,
claimId,
observedProfile: profileContract,
profileProofDigest,
resultDigest: flyRuntimeDigest(imported.receipt),
identity,
exchange,
});
}
operationHeartbeat.assertHealthy();
const slackProof = await phaseTelemetry.run(
"slack_serving_proof",
async () =>
desiredLifecycle === "ON"
? proveAdminSlackServing(claim, secrets)
: false
);
await operationHeartbeat.reattest();
const candidateLaunch =
desiredLifecycle === "ON"
? { profile, claim, installed, slackProof }
: null;
let candidateGateway = null;
try {
if (candidateLaunch) {
candidateGateway = await phaseTelemetry.run(
"native_gateway",
async () => startNativeGateway(candidateLaunch)
);
await operationHeartbeat.reattest();
}
const nativeDefaultProfileProof = candidateGateway
? proveDefaultProfileNativeSkills({
actions: defaultProfile.actions,
profileId: profile.profile,
dataRoot: DATA_ROOT,
})
: { status: "EMPTY", digest: sha256("[]") };
const receipt = completionReceipt({
claim,
exchange,
operation: proofOperation,
profile,
settings,
recovery,
observedLifecycle: desiredLifecycle,
memorySnapshot,
modelAuth,
activeProjection,
nativeDefaultProfileProof,
});
await phaseTelemetry.run("serving_observation", async () => {
if (claim.runtimeAuthority && candidateLaunch && candidateGateway) {
const servingObservation = adminServingObservation(
candidateLaunch,
candidateGateway,
exchange
);
if (!servingObservation) {
throw new Error("admin_serving_observation_unavailable");
}
const serving = await client.claimTick({
leaseSeconds: 60,
servingObservation,
});
if (serving?.servingObservationStatus !== "accepted") {
throw new Error(
`admin_serving_observation_rejected:${serving?.servingObservationCode ?? serving?.servingObservationStatus ?? "unknown"}`
);
}
}
});
const completed = await phaseTelemetry.run(
"operation_completion",
async () => client.complete(receipt)
);
if (completed?.accepted !== true) {
throw new Error("admin_runtime_completion_rejected");
}
claimedOperation = null;
claimSecretsConsumed = false;
if (candidateLaunch && candidateGateway) {
activeLaunch = candidateLaunch;
activeGateway = candidateGateway;
soulClaimTracker = createSoulClaimObservationTracker({
runtimeId: RUNTIME_ID,
profileId: candidateLaunch.profile.profile,
});
candidateGateway = null;
} else {
soulClaimTracker = null;
await clearRuntimeSecrets();
}
} finally {
await stopNativeGateway(candidateGateway);
}
if (desiredLifecycle === "ON" && !activeLaunch) {
throw new Error("admin_native_gateway_completion_rejected");
}
if (activeLaunch) {
activeModelHealthReporter = adminModelHealthReporter({
identity,
exchange,
installed: activeLaunch.installed,
claim: activeLaunch.claim,
});
try {
await activeModelHealthReporter.start("materialized");
} catch (error) {
process.stderr.write(
`${JSON.stringify({
ok: false,
code:
error instanceof Error
? error.message
: "model_health_report_failed",
})}\n`
);
}
}
await operationHeartbeat.stop();
operationHeartbeat = null;
gatewayRestartAttempts = 0;
gatewayRestartExhaustionReported = false;
failures = 0;
} catch (error) {
try {
await operationHeartbeat?.stop();
} catch (heartbeatError) {
error = heartbeatError;
} finally {
operationHeartbeat = null;
}
if (claimedOperation) {
try {
const failureReceipt = adminRuntimeFailureCompletionReceipt({
claim: claimedOperation,
exchange,
error,
secretsConsumed: claimSecretsConsumed,
});
const failure = await client.complete(failureReceipt);
if (failure?.accepted === true) {
process.stderr.write(
`${JSON.stringify({
ok: true,
code: "admin_runtime_failure_reported",
errorClass: failureReceipt.errorClass,
})}\n`
);
claimedOperation = null;
claimSecretsConsumed = false;
}
} catch (failureReportError) {
process.stderr.write(
`${JSON.stringify({
ok: false,
code:
failureReportError instanceof Error
? failureReportError.message
: "admin_runtime_failure_report_rejected",
})}\n`
);
}
}
if (!activeLaunch) await clearRuntimeSecrets();
failures = Math.min(failures + 1, 6);
process.stderr.write(
`${JSON.stringify({
ok: false,
code: error instanceof Error ? error.message : "admin_runtime_failed",
})}\n`
);
await sleep(Math.min(30_000, 500 * 2 ** failures));
}
}
beginShutdownSnapshot();
const finalMemorySnapshot = await shutdownMemorySnapshot;
if (finalMemorySnapshot?.ok === false) {
process.stderr.write(
`${JSON.stringify({
ok: false,
code: finalMemorySnapshot.code,
})}\n`
);
}
await endpointLoop;
await customRunsServices?.stop();
await publicRunsService?.stop();
await activeModelHealthReporter?.stop();
await stopNativeGateway(activeGateway);
await clearRuntimeSecrets();
}
if (import.meta.url === `file://${process.argv[1]}`) {
const mode = process.env.SELLABLE_AGENT_ADMIN_RUNTIME_MODE;
const entry =
mode === "canary"
? runAdminCanary
: mode === "production"
? runAdminProduction
: async () => {
throw new Error("admin_runtime_mode_rejected");
};
entry().catch((error) => {
process.stderr.write(
`${JSON.stringify({
ok: false,
code: error instanceof Error ? error.message : "admin_runtime_failed",
})}\n`
);
process.exit(78);
});
}