@sellable/install
Version:
One-command installer for Sellable MCP in Claude Code, Codex, and Hermes
4,589 lines • 155 kB
JavaScript
#!/usr/bin/env node
import { spawn } from "node:child_process";
import { createHash, randomUUID } from "node:crypto";
import {
chmodSync,
closeSync,
existsSync,
lstatSync,
mkdirSync,
openSync,
readFileSync,
renameSync,
unlinkSync,
writeFileSync,
} from "node:fs";
import { loadavg } from "node:os";
import { dirname, join } from "node:path";
import { pathToFileURL } from "node:url";
import { pruneAbandonedRuntimeScratch } from "../abandoned-runtime-scratch.mjs";
import { proveDefaultProfileNativeSkillsAsync } from "../default-profile-native-proof.mjs";
import {
bindDefaultProfileProbeCron,
buildDefaultProfileProjection,
reconcileDefaultProfileBundle,
} from "../default-profile-reconciler.mjs";
import {
buildNativeHermesLaunch,
buildRuntimeServingObservation,
clearCustomerRuntimeSecrets,
installCustomerRuntimeSecrets,
isOperationBoundBackupClaim,
} from "../fly-customer-worker.mjs";
import {
claimWithSoulObservation,
createFlyCustomerControlClient,
createSoulClaimObservationTracker,
dropFlyRuntimePrivileges,
exchangeFlyCustomerIdentity,
flyRuntimeDigest,
handoffFlyOidcSocket,
isFlyCustomerEnrollmentRenewalError,
loadOrCreateFlyRuntimeIdentity,
obtainFlyOidcToken,
renewFlyCustomerControlSession,
signedWorkerAuthorization,
} from "../fly-runtime-identity.mjs";
import {
executeExternalSoulBridgeOperation,
observeProfileSoul,
prepareSoulLockRoot,
} from "../fly-soul-bridge.mjs";
import { ensureHermesFirstWelcome } from "../hermes-first-welcome.mjs";
import {
ensureHermesCustomerProgramCrons,
ensureHermesFirstCampaignOnboardingCron,
ensureHermesRefillSendsCron,
ensureHermesSnapshotCron,
markHermesSnapshotDirty,
migrateLegacyHermesUserState,
reconcileHermesSnapshot,
restoreLatestHermesSnapshot,
} from "../hermes-memory-snapshot.mjs";
import {
clearManagedAuth,
managedAuthPath,
publishManagedAuth,
} from "../managed-auth-client.mjs";
import {
endpointHermesSessionId,
endpointMcpDescriptorPath,
installMcpTurnDescriptor,
replaceMcpTurnDescriptorAtomically,
} from "../mcp-context-proxy.mjs";
import {
evaluateModelAuthReadback,
modelAuthConfigOverlay,
modelAuthSelectionObservation,
parseModelAuthRuntimeCredential,
readModelAuthMaterialization,
reconcileHermesModelAuth,
} from "../model-auth-reconciler.mjs";
import { createModelHealthReporter } from "../model-health-reporter.mjs";
import {
materializeNativeSlackCustomerProfile,
SELLABLE_INTERNAL_AGENT_WORKSPACE_ID,
validateNativeSlackCustomerProfile,
} from "../profile-materializer.mjs";
import { reconcilePublicHermesSkills } from "../public-skills.mjs";
import { createCustomerRuntimePhaseTelemetry } from "./customer-runtime-phase-telemetry.mjs";
import {
endpointChainStepAdmissionDeadline,
endpointModelSelectionChain,
nextEndpointChainStep,
} from "./endpoint-model-selection.mjs";
import {
holdEndpointUatNativeCompletion,
validateEndpointUatCompletionControl,
} from "./endpoint-uat-completion-gate.mjs";
import {
createHermesSessionRetention,
HERMES_RUNS_FIXTURE_IDENTITY,
validateEndpointModelSelection,
validateOpenerOutputContract,
} from "./hermes-runs-client.mjs";
import {
createOpenerStageExecution,
usesOpenerDraftStrategy,
} from "./opener-stage-execution.mjs";
import { createConfiguredRunsClient } from "./runs-api-connections.mjs";
const DATA_ROOT = "/opt/data";
const PROFILES_ROOT = join(DATA_ROOT, "profiles");
const SECRET_ROOT = "/run/sellable-agent";
const sessionRetention = createHermesSessionRetention();
const TRUST_ROOT = "/usr/local/share/sellable-agent/mcp-turn-trust-root.pem";
const HERMES = "/opt/hermes/.venv/bin/hermes";
const HERMES_PYTHON = "/opt/hermes/.venv/bin/python3";
const MCP =
"/usr/local/lib/sellable-agent/external-runtime/mcp/bin/sellable-mcp";
const MCP_CONTEXT_PROXY = "/usr/local/lib/sellable-agent/mcp-context-proxy.mjs";
const RUNTIME_ID = process.env.SELLABLE_RUNTIME_ID ?? "";
const CUSTOMER_KEY = process.env.SELLABLE_AGENT_CUSTOMER_KEY ?? "";
const ORIGIN = process.env.SELLABLE_AGENT_CONTROL_ORIGIN ?? "";
const AUDIENCE = process.env.SELLABLE_WORKLOAD_AUDIENCE ?? "";
const RUNTIME_GENERATION = Number(
process.env.SELLABLE_AGENT_RUNTIME_GENERATION ??
process.env.SELLABLE_RUNTIME_GENERATION ??
"0"
);
const BOOT_SESSION_ID = randomUUID();
const HERMES_VERSION = process.env.SELLABLE_AGENT_HERMES_VERSION ?? "0.21.5";
const INSTALLER_PACKAGE =
process.env.SELLABLE_AGENT_INSTALLER_PACKAGE ?? "@sellable/install@0.1.804";
const MCP_PACKAGE =
process.env.SELLABLE_AGENT_MCP_PACKAGE ?? "@sellable/mcp@0.1.957";
export const ENDPOINT_MAX_CONCURRENT_RUNS = 10;
const DEFAULT_PROFILE_BUNDLE_ROOT =
"/usr/local/share/sellable-agent/default-profile-bundle";
const DEFAULT_PROFILE_CRON_PYTHON = [
"import json,sys",
"from cron.jobs import create_job, update_job",
"value=json.loads(sys.argv[1])",
"job_id=value.pop('jobId', None)",
"enabled=value.pop('enabled')",
"result=update_job(job_id, {**value, 'enabled': enabled}) if job_id else create_job(prompt='', **value)",
"print(json.dumps({'id': result.get('id') if result else None}))",
].join("\n");
const sha256 = (value) => createHash("sha256").update(value).digest("hex");
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
function canonicalizeEndpointJson(value) {
if (
value === null ||
typeof value === "string" ||
typeof value === "boolean"
) {
return value;
}
if (typeof value === "number") {
if (!Number.isFinite(value)) {
throw new Error("customer_endpoint_completion_output_rejected");
}
return value;
}
if (Array.isArray(value)) return value.map(canonicalizeEndpointJson);
if (value && typeof value === "object") {
return Object.fromEntries(
Object.entries(value)
.filter(([, item]) => item !== undefined)
.sort(([left], [right]) => left.localeCompare(right))
.map(([key, item]) => [key, canonicalizeEndpointJson(item)])
);
}
throw new Error("customer_endpoint_completion_output_rejected");
}
export function endpointCanonicalOutputDigest(output) {
return sha256(JSON.stringify(canonicalizeEndpointJson(output)));
}
function customerDefaultProfileCronAdapter(profileId, profileRoot) {
const mutate = async ({ desired, spec, existing }) => {
const child = await run(
HERMES_PYTHON,
[
"-c",
DEFAULT_PROFILE_CRON_PYTHON,
JSON.stringify({
name: desired.name,
schedule: spec.schedule,
script: desired.script,
no_agent: true,
deliver: "local",
enabled: true,
...(existing?.id ? { jobId: existing.id } : {}),
}),
],
{
env: {
...process.env,
HOME: DATA_ROOT,
HERMES_HOME: profileRoot,
HERMES_PROFILE: profileId,
NO_COLOR: "1",
},
timeoutMs: 120_000,
}
);
return {
...child,
exitCode: child.status,
signal: null,
outputLimitExceeded: false,
};
};
return {
create: (input) => mutate(input),
update: (input) => mutate(input),
bind: (input) =>
bindDefaultProfileProbeCron({ ...input, bootId: BOOT_SESSION_ID }),
};
}
function atomicFile(path, bytes, mode = 0o600) {
mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
const temporary = `${path}.${process.pid}.${randomUUID()}.tmp`;
writeFileSync(temporary, bytes, { flag: "wx", mode });
chmodSync(temporary, mode);
renameSync(temporary, path);
chmodSync(path, mode);
}
function atomicJson(path, value, mode = 0o600) {
atomicFile(path, `${JSON.stringify(value, null, 2)}\n`, mode);
}
async function run(
command,
args,
{ env = process.env, timeoutMs = 60_000 } = {}
) {
return new Promise((resolve) => {
const startedAt = Date.now();
const child = spawn(command, args, {
env,
stdio: ["ignore", "pipe", "pipe"],
});
const stdout = [];
const stderr = [];
let timedOut = false;
const timer = setTimeout(() => {
timedOut = true;
child.kill("SIGKILL");
}, timeoutMs);
child.stdout.on("data", (chunk) => stdout.push(chunk));
child.stderr.on("data", (chunk) => stderr.push(chunk));
child.once("error", (error) => {
clearTimeout(timer);
resolve({
status: null,
stdout: "",
stderr: "",
timedOut,
elapsedMs: Date.now() - startedAt,
error: error.message,
});
});
child.once("close", (status) => {
clearTimeout(timer);
resolve({
status,
stdout: Buffer.concat(stdout).toString("utf8"),
stderr: Buffer.concat(stderr).toString("utf8"),
timedOut,
elapsedMs: Date.now() - startedAt,
});
});
});
}
const EXACT_INSTALLER_PACKAGE =
/^@sellable\/install@\d+\.\d+\.\d+(?:[-.A-Za-z0-9]+)?$/;
export function assertCustomerRuntimeInstallerPackage({
expected,
actual = INSTALLER_PACKAGE,
}) {
if (
!EXACT_INSTALLER_PACKAGE.test(expected ?? "") ||
!EXACT_INSTALLER_PACKAGE.test(actual ?? "")
) {
throw new Error("customer_runtime_installer_package_invalid");
}
// Runtime releases are selected by immutable image digest. The backend and
// image are promoted in adjacent steps, so the signed claim can briefly name
// the next package while this Machine still runs the previously active
// digest (or vice versa). The image must materialize its own pinned package;
// rejecting the version skew wedges otherwise compatible Stop/Start actions.
return actual;
}
function doctorCheckName(output) {
const line = String(output)
.split(/\r?\n/)
.map((value) => value.trim())
.find((value) =>
/(?:✗|\b(?:fail|error|missing)\b|Lazy-installing|pip install|npm install)/i.test(
value
)
);
return (line ?? "unknown")
.replace(/^[^A-Za-z0-9.]+/, "")
.replace(/[^A-Za-z0-9.]+/g, "_")
.replace(/^_+|_+$/g, "")
.slice(0, 160);
}
export function customerRuntimeDoctorFailureCode(
doctor,
{ loadAverage = loadavg() } = {}
) {
if (doctor?.timedOut) {
const observedLoad = [...loadAverage]
.slice(0, 3)
.map((value) => Number(value).toFixed(2))
.join(",");
return `customer_runtime_doctor_timeout:elapsed_ms=${Math.max(0, Math.round(Number(doctor.elapsedMs) || 0))}:loadavg=${observedLoad}`;
}
const output = `${doctor?.stdout ?? ""}\n${doctor?.stderr ?? ""}`;
if (
doctor?.status !== 0 ||
/Lazy-installing|pip install|npm install/i.test(output)
) {
return `customer_runtime_doctor_rejected:check=${doctorCheckName(output)}`;
}
return null;
}
/**
* @param {string} action
* @param {{ runtimeAuthorityScope?: "ASSIGNED" | "CANDIDATE" | null, directReadinessProven?: boolean }} [options]
*/
export function customerRuntimeDoctorRequired(
action,
{ runtimeAuthorityScope = null, directReadinessProven = false } = {}
) {
// A successful provisioned proof checks the pinned binaries, MCP and model
// credentials; proveApplication then requires Slack identity/socket and the
// authenticated native API. Interactive doctor additionally requires a
// default-profile .env and a ~/.local/bin shim that these containers do not
// use. Those workstation checks must not reject an otherwise proven restart.
if (directReadinessProven) return false;
// START and candidate promotion already prove the binaries, model auth,
// Slack identities, gateway, private API, and native skills directly below.
// Keep the broad interactive diagnostic on assigned-runtime REFRESH only;
// putting its five-minute timeout inside the candidate's five-minute
// preparation fence can reject a healthy image before direct proof runs.
return action !== "ENABLE_GATEWAY" && runtimeAuthorityScope !== "CANDIDATE";
}
export const CUSTOMER_RUNTIME_DEFERRED_SNAPSHOT_CODE =
"hermes_snapshot_deferred_until_serving";
/**
* Upgrade's BACKING_UP stage already captured the durable pre-replacement
* profile. Reconciliation on the replacement can dirty that profile again,
* but compacting and uploading a large state.db is recovery work rather than
* serving proof. Keep the dirty marker for the normal post-serving reconciler
* instead of putting two independently bounded backups in front of candidate
* Slack readiness.
*
* @param {"ASSIGNED" | "CANDIDATE" | null | undefined} runtimeAuthorityScope
*/
export function customerRuntimeSnapshotRequiredBeforeServing(
runtimeAuthorityScope
) {
return runtimeAuthorityScope !== "CANDIDATE";
}
function deferredCustomerRuntimeSnapshot() {
return {
ok: false,
status: "PENDING_RETRY",
code: CUSTOMER_RUNTIME_DEFERRED_SNAPSHOT_CODE,
};
}
async function flushHermesSnapshotBeforeServing({
client,
profileId,
profileRoot,
runtimeAuthorityScope,
}) {
if (!customerRuntimeSnapshotRequiredBeforeServing(runtimeAuthorityScope)) {
return deferredCustomerRuntimeSnapshot();
}
return await flushHermesSnapshot({ client, profileId, profileRoot });
}
function exactArtifact(claim, type) {
const artifacts = claim?.revision?.artifacts?.filter(
(artifact) =>
artifact?.type === type && typeof artifact.safeContent === "string"
);
if (!Array.isArray(artifacts) || artifacts.length < 1) {
throw new Error(`customer_${type.toLowerCase()}_artifact_missing`);
}
for (const artifact of artifacts) {
if (
Buffer.byteLength(artifact.safeContent) !== artifact.sizeBytes ||
sha256(artifact.safeContent) !== artifact.contentHash
) {
throw new Error("customer_artifact_digest_rejected");
}
}
return artifacts
.sort((left, right) => left.logicalPath.localeCompare(right.logicalPath))
.map((artifact) => artifact.safeContent)
.join("\n");
}
function compileDesired(claim, trustRootPem) {
const operation = claim?.operation;
const slack = claim?.runtimeContract?.slack;
const service = claim?.runtimeContract?.service;
const model = claim?.runtimeContract?.model;
const apiServer = claim?.runtimeContract?.apiServer;
const installerPackage = claim?.runtimeContract?.installerPackage;
const customerProgram = claim?.runtimeContract?.customerProgram;
const selectedBindings = slack?.selectedBindings;
const toolInclude = service?.toolInclude;
if (
!operation ||
operation.targetRevisionId !== claim?.revision?.id ||
!slack ||
!service ||
!model ||
!apiServer ||
!installerPackage ||
!customerProgram ||
typeof customerProgram.timezone !== "string" ||
!customerProgram.timezone.trim() ||
apiServer.host !== "127.0.0.1" ||
apiServer.port !== 8642 ||
apiServer.cors !== false ||
!Array.isArray(toolInclude) ||
toolInclude.length < 1 ||
toolInclude.some((tool) => !/^[a-z][a-z0-9_]{0,127}$/.test(tool)) ||
JSON.stringify(toolInclude) !==
JSON.stringify([...new Set(toolInclude)].sort()) ||
!Array.isArray(selectedBindings) ||
selectedBindings.length < 1 ||
!Array.isArray(slack?.memberAllowlist) ||
!claim?.secretEnvelopes?.mcpService ||
!claim?.secretEnvelopes?.slackApp ||
!claim?.secretEnvelopes?.slackBot ||
!claim?.secretEnvelopes?.modelCodex ||
!claim?.secretEnvelopes?.hermesApiServer
) {
throw new Error("customer_claim_contract_rejected");
}
const imageInstallerPackage = assertCustomerRuntimeInstallerPackage({
expected: installerPackage,
});
const modelEnvelope = claim.secretEnvelopes.modelCodex;
if (
model.provider !== "codex" ||
!/^[A-Za-z0-9_.:-]{1,256}$/.test(model.activationId ?? "") ||
!Number.isSafeInteger(model.bundleGeneration) ||
model.bundleGeneration < 1 ||
!/^[a-f0-9]{64}$/.test(model.policyHash ?? "") ||
!/^[a-f0-9]{16}$/.test(model.credentialFingerprint ?? "") ||
modelEnvelope.generation !== model.bundleGeneration ||
modelEnvelope.fingerprint !== model.credentialFingerprint ||
modelEnvelope.activationId !== model.activationId ||
modelEnvelope.bundleGeneration !== model.bundleGeneration ||
modelEnvelope.policyHash !== model.policyHash
) {
throw new Error("customer_model_auth_identity_rejected");
}
const routeGenerations = [
...new Set(selectedBindings.map((binding) => binding.routeGeneration)),
];
const membershipGenerations = [
...new Set(selectedBindings.map((binding) => binding.membershipGeneration)),
];
if (
routeGenerations.length !== 1 ||
membershipGenerations.length !== 1 ||
!Number.isSafeInteger(routeGenerations[0]) ||
!Number.isSafeInteger(membershipGenerations[0])
) {
throw new Error("customer_route_generation_rejected");
}
const defaultBinding = selectedBindings.find(
(binding) => binding.id === slack.defaultBindingId
);
if (!defaultBinding) throw new Error("customer_default_route_rejected");
const selectedChannelIds = [
...new Set(selectedBindings.map((binding) => binding.channelId)),
].sort();
const accessPolicyHash = flyRuntimeDigest({
contract: "selected-channel-any-human/v1",
allowAllUsers: true,
allowBots: "none",
allowDirectMessages: false,
requireMention: true,
strictMention: false,
installationId: slack.installationId,
teamId: slack.teamId,
botUserId: slack.botUserId,
selectedChannelIds,
defaultChannelId: defaultBinding.channelId,
});
const runtimeDesiredHash = flyRuntimeDigest({
contract: "customer-runtime-desired/v1",
installerPackage,
accessPolicyHash,
});
if (
slack.hashes?.accessPolicy !== accessPolicyHash ||
slack.hashes?.customerRuntimeDesired !== runtimeDesiredHash
) {
throw new Error("customer_access_policy_identity_rejected");
}
return Object.freeze({
toolInclude: Object.freeze([...toolInclude]),
desired: Object.freeze({
runtimeMode: "native_slack_customer",
workspaceId: operation.workspaceId,
agentId: operation.agentId,
runtimeId: RUNTIME_ID,
profileId: operation.profileId,
revisionId: operation.targetRevisionId,
revisionSequence: operation.lifecycleGeneration,
installationId: slack.installationId,
teamId: slack.teamId,
botUserId: slack.botUserId,
slackAppGeneration: slack.appCredentialGeneration,
slackGeneration: slack.generation,
routeGeneration: routeGenerations[0],
membershipGeneration: membershipGenerations[0],
serviceCredentialGeneration: service.credentialGeneration,
modelCredentialGeneration: model.bundleGeneration,
modelActivationId: model.activationId,
modelBundleGeneration: model.bundleGeneration,
modelPolicyHash: model.policyHash,
apiServerKeyGeneration: apiServer.keyGeneration,
apiServerKeyFingerprint: apiServer.keyFingerprint,
runtimeGeneration: RUNTIME_GENERATION,
selectedChannelIds,
defaultChannelId: defaultBinding.channelId,
memberAllowlist: [...new Set(slack.memberAllowlist)].sort(),
accessPolicyHash,
runtimeDesiredHash,
hermesVersion: HERMES_VERSION,
installerPackage: imageInstallerPackage,
mcpPackage: MCP_PACKAGE,
policyHash: service.policyHash,
customerProgramTimezone: customerProgram.timezone,
soul: exactArtifact(claim, "SOUL"),
skill: exactArtifact(claim, "SKILL"),
}),
});
}
function operationIdentity(claim, exchange) {
return {
operationId: claim.operation.id,
workspaceId: claim.operation.workspaceId,
agentId: claim.operation.agentId,
hostId: exchange.hostId,
workerEnrollmentId: exchange.enrollmentId,
workerId: exchange.workerId,
fence: claim.operation.fence,
targetRevisionId: claim.operation.targetRevisionId,
};
}
function customerRuntimeFailureErrorClass(error) {
const raw =
error instanceof Error && error.message
? error.message
: "customer_runtime_failed";
const normalized = raw
.toUpperCase()
.replace(/[^A-Z0-9_]/g, "_")
.replace(/_+/g, "_")
.slice(0, 64);
return /^[A-Z]/.test(normalized)
? normalized
: `CUSTOMER_${normalized}`.slice(0, 64);
}
export function customerRuntimeBackupFailureCompletionReceipt({
claim,
exchange,
error,
}) {
const unsigned = {
...operationIdentity(claim, exchange),
status: "FAILED",
secretDisposition: "NOT_PRESENT",
observedLifecycle: "DEGRADED",
observedRevisionId: null,
signerGeneration: exchange.keyGeneration,
runtimeHashes: {
package: null,
config: null,
content: null,
process: null,
},
conditions: [{ code: "PROVISIONING_FAILED", status: "FALSE" }],
errorClass: customerRuntimeFailureErrorClass(error),
auditIds: [`fly-customer-${claim.operation.id}`],
};
return { ...unsigned, signedDigest: flyRuntimeDigest(unsigned) };
}
function operationSnapshotIdentity(claim, exchange) {
const authority = claim?.runtimeAuthority;
if (
!isOperationBoundBackupClaim(claim) ||
authority?.scope !== "ASSIGNED" ||
authority.runtimeArtifactId !== authority.assignedRuntimeArtifactId
) {
return undefined;
}
return {
operationId: claim.operation.id,
authorityGeneration: authority.authorityGeneration,
runtimeArtifactId: authority.assignedRuntimeArtifactId,
sourceMachineId: exchange.workerId,
sourceMachineVersion: authority.machineVersion,
sourceBootSessionId: BOOT_SESSION_ID,
profileSchemaVersion: "customer-native-profile/v1",
};
}
export function customerServingObservation(activeRuntime, exchange) {
try {
const authority = activeRuntime?.claim?.runtimeAuthority;
const gateway = activeRuntime?.gateway;
const running = gateway?.child?.exitCode === null && !gateway.child.killed;
if (
!authority ||
!running ||
!activeRuntime.proof?.gatewayReadback?.running
)
return null;
const observedAt = new Date();
return buildRuntimeServingObservation({
scope: authority.scope,
kind: "CUSTOMER",
operationId: activeRuntime.claim.operation.id,
workspaceId: activeRuntime.claim.operation.workspaceId,
agentId: activeRuntime.claim.operation.agentId,
runtimeId: authority.runtimeId,
authorityGeneration: authority.authorityGeneration,
runtimeArtifactId: authority.runtimeArtifactId,
machineId: exchange.workerId,
machineVersion: authority.machineVersion,
bootSessionId: BOOT_SESSION_ID,
profileIdentity: activeRuntime.desired.profileId,
processIdentity: `gateway-${gateway.child.pid}-${activeRuntime.claim.operation.fence}`,
observedAt: observedAt.toISOString(),
expiresAt: new Date(observedAt.getTime() + 120_000).toISOString(),
slack: {
authenticated: true,
socketModeConnected: true,
apiReachable: true,
},
productMcp: authority.productMcp,
adminMcp: null,
...(authority.bundleVersion === undefined
? {}
: {
bundleVersion: authority.bundleVersion,
bundleDigest: authority.bundleDigest,
}),
nativeGateway: true,
customerRouting: true,
});
} catch {
return null;
}
}
function startOperationHeartbeat({ claim, exchange, client }) {
const identity = operationIdentity(claim, exchange);
let stopped = false;
let failure = null;
let inFlight = Promise.resolve();
const beat = () => {
if (stopped) return;
inFlight = client
.heartbeat({ ...identity, leaseSeconds: 60 })
.then((value) => {
if (value?.renewed !== true) {
throw new Error("customer_runtime_heartbeat_rejected");
}
})
.catch((error) => {
failure =
error instanceof Error
? error
: new Error("customer_runtime_heartbeat_failed");
});
};
const timer = setInterval(beat, 20_000);
beat();
return {
assertHealthy() {
if (failure) throw failure;
},
async reattest() {
beat();
await inFlight;
if (failure) throw failure;
},
async stop() {
stopped = true;
clearInterval(timer);
await inFlight;
if (failure) throw failure;
},
};
}
export async function proveCustomerRuntimeDefaultSkills({
claim,
exchange,
client,
gateway,
actions,
profileId,
reattestAuthority,
proveNativeSkills = proveDefaultProfileNativeSkillsAsync,
stopGateway = stopNativeGateway,
}) {
try {
for (let attempt = 0; attempt < 2; attempt += 1) {
await reattestAuthority();
try {
return await proveNativeSkills({
actions,
profileId,
dataRoot: DATA_ROOT,
});
} catch (error) {
if (
attempt === 1 ||
!["skills_native_call_failed", "skills_native_call_timeout"].includes(
error?.message
)
)
throw error;
}
}
} catch (error) {
await stopGateway(gateway);
// This boundary is reached only after all five credentials were installed
// and the gateway started. Report that truth, never NOT_CONSUMED or success.
await reattestAuthority();
const { signedDigest: _discarded, ...base } =
customerRuntimeBackupFailureCompletionReceipt({ claim, exchange, error });
const unsigned = {
...base,
status: "UNCERTAIN",
secretDisposition: "REDELIVERABLE",
...(claim.runtimeAuthority?.bundleVersion === undefined
? {}
: {
bundleVersion: claim.runtimeAuthority.bundleVersion,
bundleDigest: claim.runtimeAuthority.bundleDigest,
}),
};
const receipt = { ...unsigned, signedDigest: flyRuntimeDigest(unsigned) };
const reported = await client.complete(receipt);
if (reported?.accepted !== true)
throw new Error("customer_runtime_failure_report_rejected");
throw error;
}
}
const ENDPOINT_ID = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/;
const ENDPOINT_DIGEST = /^[a-f0-9]{64}$/;
function stableEndpointJson(value) {
if (Array.isArray(value)) {
return `[${value.map(stableEndpointJson).join(",")}]`;
}
if (value && typeof value === "object") {
return `{${Object.keys(value)
.sort()
.map((key) => `${JSON.stringify(key)}:${stableEndpointJson(value[key])}`)
.join(",")}}`;
}
return JSON.stringify(value);
}
function validEndpointJson(value) {
if (
value === null ||
typeof value === "string" ||
typeof value === "boolean"
) {
return true;
}
if (typeof value === "number") return Number.isFinite(value);
if (Array.isArray(value)) return value.every(validEndpointJson);
if (!value || typeof value !== "object") return false;
return Object.entries(value).every(
([key, member]) => typeof key === "string" && validEndpointJson(member)
);
}
function endpointConversationHistory(value) {
if (
!value ||
typeof value !== "object" ||
Array.isArray(value) ||
JSON.stringify(Object.keys(value).sort()) !==
JSON.stringify(
[
"digest",
"includedDigest",
"includedItems",
"items",
"omittedItems",
"schemaVersion",
"strategy",
"totalItems",
"truncated",
].sort()
) ||
value.schemaVersion !== "sellable-agent-endpoint-history/v1" ||
value.strategy !== "latest-items-byte-budget/v1" ||
!Number.isSafeInteger(value.totalItems) ||
!Number.isSafeInteger(value.includedItems) ||
!Number.isSafeInteger(value.omittedItems) ||
value.totalItems < 0 ||
value.includedItems < 0 ||
value.omittedItems < 0 ||
value.totalItems !== value.includedItems + value.omittedItems ||
value.truncated !== value.omittedItems > 0 ||
!ENDPOINT_DIGEST.test(value.digest ?? "") ||
!ENDPOINT_DIGEST.test(value.includedDigest ?? "") ||
!Array.isArray(value.items) ||
value.items.length !== value.includedItems ||
value.items.length > 32 ||
value.items.reduce(
(total, item) => total + Buffer.byteLength(stableEndpointJson(item)),
0
) > 49_152 ||
flyRuntimeDigest(value.items) !== value.includedDigest ||
(!value.truncated && value.digest !== value.includedDigest)
) {
throw new Error("customer_endpoint_history_rejected");
}
let previous = null;
const rank = { TURN_INPUT: 0, TOOL_RECEIPT: 1, TURN_OUTPUT: 2 };
const native = value.items.map((item) => {
const expectedKeys =
item?.kind === "TOOL_RECEIPT"
? ["content", "digest", "kind", "runId", "sequence", "turnNumber"]
: ["content", "digest", "kind", "runId", "turnNumber"];
if (
!item ||
typeof item !== "object" ||
Array.isArray(item) ||
JSON.stringify(Object.keys(item).sort()) !==
JSON.stringify(expectedKeys.sort()) ||
!Object.hasOwn(rank, item.kind) ||
!ENDPOINT_ID.test(item.runId ?? "") ||
!Number.isSafeInteger(item.turnNumber) ||
item.turnNumber < 1 ||
!validEndpointJson(item.content) ||
flyRuntimeDigest(item.content) !== item.digest ||
(item.kind === "TOOL_RECEIPT" &&
(typeof item.sequence !== "string" ||
!/^[1-9][0-9]{0,19}$/.test(item.sequence)))
) {
throw new Error("customer_endpoint_history_rejected");
}
const order = [
item.turnNumber,
rank[item.kind],
item.kind === "TOOL_RECEIPT" ? BigInt(item.sequence) : 0n,
];
if (
previous &&
(order[0] < previous[0] ||
(order[0] === previous[0] && order[1] < previous[1]) ||
(order[0] === previous[0] &&
order[1] === previous[1] &&
order[2] <= previous[2]))
) {
throw new Error("customer_endpoint_history_rejected");
}
previous = order;
return Object.freeze({
role: item.kind === "TURN_INPUT" ? "user" : "assistant",
content: stableEndpointJson(item.content),
});
});
return Object.freeze(native);
}
function endpointClaimIdentity(claim) {
const attempt = claim?.attempt;
const runtime = claim?.runtime;
const revision = claim?.revision;
const runClaim = claim?.run;
const stringMembers = [
attempt?.id,
attempt?.claimId,
attempt?.workspaceId,
attempt?.agentId,
attempt?.endpointId,
attempt?.revisionId,
attempt?.conversationId,
attempt?.runId,
attempt?.hostId,
attempt?.workerEnrollmentId,
attempt?.workerId,
runtime?.runtimeId,
runtime?.profileId,
];
if (
!attempt ||
!runtime ||
!revision ||
!runClaim ||
stringMembers.some((value) => !ENDPOINT_ID.test(value ?? "")) ||
revision.id !== attempt.revisionId ||
!Number.isSafeInteger(attempt.attemptNumber) ||
attempt.attemptNumber < 1 ||
!/^[1-9][0-9]{0,19}$/.test(attempt.fence ?? "") ||
!Number.isSafeInteger(runtime.runtimeGeneration) ||
runtime.runtimeGeneration < 1 ||
!Number.isSafeInteger(runtime.profileGeneration) ||
runtime.profileGeneration < 1 ||
!Number.isSafeInteger(runtime.policyGeneration) ||
runtime.policyGeneration < 1 ||
!Number.isSafeInteger(revision.toolPolicyGeneration) ||
revision.toolPolicyGeneration !== runtime.policyGeneration ||
!ENDPOINT_DIGEST.test(revision.resolvedToolDigest ?? "") ||
!ENDPOINT_DIGEST.test(runClaim.inputHash ?? "") ||
!Number.isSafeInteger(revision.timeoutSeconds) ||
revision.timeoutSeconds < 1 ||
revision.timeoutSeconds > 86_400 ||
typeof revision.prompt !== "string" ||
Buffer.byteLength(revision.prompt) > 128 * 1024 ||
!Array.isArray(revision.resolvedToolNames) ||
revision.resolvedToolNames.some(
(tool) => !/^[a-z][a-z0-9_]{0,127}$/.test(tool)
) ||
Date.parse(attempt.deadlineAt) !== Date.parse(attempt.deadlineAt) ||
Date.parse(attempt.leaseExpiresAt) !== Date.parse(attempt.leaseExpiresAt)
) {
throw new Error("customer_endpoint_claim_rejected");
}
endpointConversationHistory(runClaim.conversationHistory);
validateEndpointUatCompletionControl(claim.uatCompletionControl, claim);
return Object.freeze({
operationId: attempt.id,
workspaceId: attempt.workspaceId,
agentId: attempt.agentId,
hostId: attempt.hostId,
workerEnrollmentId: attempt.workerEnrollmentId,
workerId: attempt.workerId,
fence: attempt.fence,
targetRevisionId: attempt.revisionId,
endpointId: attempt.endpointId,
revisionId: attempt.revisionId,
conversationId: attempt.conversationId,
runId: attempt.runId,
attemptId: attempt.id,
claimId: attempt.claimId,
runtimeId: runtime.runtimeId,
profileId: runtime.profileId,
runtimeGeneration: runtime.runtimeGeneration,
profileGeneration: runtime.profileGeneration,
policyGeneration: runtime.policyGeneration,
});
}
function endpointMcpDescriptorBinding(claim, activeRuntime) {
const descriptor = claim?.mcpTurnDescriptor;
const body = descriptor?.body;
const expectedNames = claim?.revision?.resolvedToolNames;
if (
!descriptor ||
!body ||
body.schemaVersion !== "sellable-agent-mcp-endpoint-run/v1" ||
body.principalKind !== "endpoint_run" ||
stableEndpointJson(body.routes) !== stableEndpointJson(["sellable-mcp"]) ||
body.workspaceId !== claim.attempt.workspaceId ||
body.agentId !== claim.attempt.agentId ||
body.endpointId !== claim.attempt.endpointId ||
body.revisionId !== claim.attempt.revisionId ||
body.conversationId !== claim.attempt.conversationId ||
body.runId !== claim.attempt.runId ||
body.attemptId !== claim.attempt.id ||
body.claimId !== claim.attempt.claimId ||
body.runtimeId !== claim.runtime.runtimeId ||
body.profileId !== claim.runtime.profileId ||
body.runtimeGeneration !== claim.runtime.runtimeGeneration ||
body.profileGeneration !== claim.runtime.profileGeneration ||
body.serviceCredentialGeneration !==
activeRuntime?.desired?.serviceCredentialGeneration ||
body.toolPolicyGeneration !== claim.runtime.policyGeneration ||
body.allowedToolDigest !== claim.revision.resolvedToolDigest ||
stableEndpointJson(body.allowedToolNames) !==
stableEndpointJson(expectedNames) ||
body.fence !== claim.attempt.fence ||
body.turnId !== claim.attempt.id ||
body.bodyDigest !== claim.run.inputHash
) {
throw new Error("customer_endpoint_mcp_descriptor_binding_rejected");
}
return {
descriptor,
expected: Object.fromEntries(
[
"workspaceId",
"agentId",
"endpointId",
"revisionId",
"conversationId",
"runId",
"attemptId",
"claimId",
"runtimeId",
"profileId",
"runtimeGeneration",
"profileGeneration",
"serviceCredentialGeneration",
"toolPolicyGeneration",
"allowedTools",
"allowedToolNames",
"allowedToolDigest",
"fence",
"turnId",
"bodyDigest",
].map((key) => [key, body[key]])
),
};
}
function endpointRuntimeCurrent(claim, activeRuntime, runtimeGeneration) {
return Boolean(
activeRuntime?.gateway?.readback?.().running &&
activeRuntime.desired?.runtimeId === claim.runtime.runtimeId &&
activeRuntime.desired?.profileId === claim.runtime.profileId &&
claim.runtime.runtimeGeneration === runtimeGeneration &&
activeRuntime.desired?.runtimeGeneration === runtimeGeneration
);
}
function endpointEventEnvelope(event, ordinal) {
const redactedData =
event.kind === "LATE_COMPLETION_NOTICE"
? {
observedAttemptStatus: "ABANDONED",
observedRunStatus: "SUPERSEDED",
observedOutputDigest: event.outputDigest,
}
: {
classification: event.classification ?? null,
kind: event.kind ?? null,
timestamp: event.timestamp ?? null,
tool: event.tool ?? null,
choice: event.choice ?? null,
positiveCancellation: event.positiveCancellation === true,
outputDigest: event.outputDigest ?? null,
errorDigest: event.errorDigest ?? null,
evidenceDigest: event.evidence?.digest ?? null,
// Which model of an endpoint's fallback chain the event belongs to.
...(Number.isSafeInteger(event.chainStep)
? { chainStep: event.chainStep }
: {}),
...(event.servedRuntime || event.usage
? {
servedProvider: event.servedRuntime?.provider ?? null,
servedModel: event.servedRuntime?.model ?? null,
inputTokens: event.usage?.inputTokens ?? null,
outputTokens: event.usage?.outputTokens ?? null,
cacheReadTokens: event.usage?.cacheReadTokens ?? null,
cacheWriteTokens: event.usage?.cacheWriteTokens ?? null,
}
: {}),
...(/^writer-[123]$|^editor$/.test(event.stage ?? "")
? {
stage: event.stage,
strategy: event.strategy ?? "three-draft-editor/v1",
nativeRunIdDigest: event.nativeRunIdDigest ?? null,
originalInputDigest: event.originalInputDigest ?? null,
elapsedMs: event.elapsedMs ?? null,
schemaValid: event.schemaValid ?? null,
}
: {}),
};
const payloadDigest =
event.outputDigest ??
event.errorDigest ??
event.requestDigest ??
event.evidence?.digest ??
sha256(JSON.stringify(redactedData));
const nativeEventId =
event.nativeEventIdentity ??
sha256(`${ordinal}\n${payloadDigest}\n${JSON.stringify(redactedData)}`);
// RUN_COMPLETED is owned by the control plane's canonical completion
// record. Preserve the native terminal witness under a distinct name so
// its persist-before-complete fence cannot collide with that reserved
// adapter event type.
const nativeType =
event.kind === "RUN_COMPLETED"
? "NATIVE_RUN_COMPLETED"
: (event.kind ?? event.classification ?? "NATIVE_EVENT_UNKNOWN");
const type = String(nativeType)
.toUpperCase()
.replace(/[^A-Z0-9_]/g, "_")
.slice(0, 64);
if (
(event.kind === "LATE_COMPLETION_NOTICE" &&
!ENDPOINT_DIGEST.test(event.outputDigest ?? "")) ||
!ENDPOINT_DIGEST.test(nativeEventId) ||
!ENDPOINT_DIGEST.test(payloadDigest) ||
!/^[A-Z][A-Z0-9_]{0,63}$/.test(type) ||
Buffer.byteLength(JSON.stringify(redactedData)) > 8 * 1024
) {
throw new Error("customer_endpoint_event_rejected");
}
return Object.freeze({ nativeEventId, type, payloadDigest, redactedData });
}
function endpointCompletionBody({
identity,
attemptStatus,
status,
output = null,
productContractDigest = null,
productProjectedOutput = null,
productProjectedOutputDigest = null,
positiveCancellation = false,
hasEffectIntent = false,
now,
}) {
// The control plane verifies semanticRequestHash(canonicalOutput). Hermes
// preserves model key insertion order, so hashing the raw object caused a
// valid structured completion (production run 1e501766-120c-4ada-af07-01357aef516d)
// to be retried as UNCERTAIN. Keep this serializer byte-equivalent to the
// server's recursively key-sorted JSON contract.
const canonicalOutputDigest =
output === null ? null : endpointCanonicalOutputDigest(output);
return {
...identity,
schemaVersion: "sellable-agent-endpoint-complete/v1",
attemptStatus,
status,
canonicalOutput: output,
canonicalOutputDigest,
...(productContractDigest === null
? {}
: {
productContractDigest,
productProjectedOutput,
productProjectedOutputDigest,
}),
positiveCancellation,
hasEffectIntent,
completedAt: now().toISOString(),
};
}
export function endpointProductProjection(productExecution, output) {
if (productExecution == null) return { ok: true, completion: {} };
if (
!productExecution ||
typeof productExecution !== "object" ||
Array.isArray(productExecution) ||
!ENDPOINT_DIGEST.test(productExecution.contractDigest ?? "") ||
!["OPENER", "REPLY", "FOLLOW_UP"].includes(productExecution.workKind) ||
typeof productExecution.subjectRequired !== "boolean"
) {
return { ok: false };
}
let proposal = output;
if (typeof output === "string") {
try {
proposal = JSON.parse(output);
} catch {
return { ok: false };
}
}
if (
productExecution.nativeProposalSchemaVersion ===
"sellable-product-generation-semantic-proposal/v1"
) {
if (
!proposal ||
typeof proposal !== "object" ||
Array.isArray(proposal) ||
Object.keys(proposal).length !== 1 ||
!Object.hasOwn(proposal, "proposal")
) {
return { ok: false };
}
const semantic = proposal.proposal;
if (
!semantic ||
typeof semantic !== "object" ||
Array.isArray(semantic) ||
semantic.schemaVersion !== productExecution.nativeProposalSchemaVersion ||
!["DRAFT", "SUPPRESS"].includes(semantic.outcome) ||
!Array.isArray(semantic.evidence) ||
semantic.evidence.length > 64 ||
!semantic.evidence.every(
(row) =>
row &&
typeof row === "object" &&
!Array.isArray(row) &&
Object.keys(row).length === 1 &&
typeof row.summary === "string" &&
row.summary.length >= 1 &&
row.summary.length <= 4_096
) ||
typeof semantic.reasoning !== "string" ||
semantic.reasoning.length < 1 ||
semantic.reasoning.length > 4_096
) {
return { ok: false };
}
if (semantic.outcome === "SUPPRESS") {
if (
Object.keys(semantic).sort().join(",") !==
"evidence,outcome,reasoning,schemaVersion"
) {
return { ok: false };
}
return {
ok: true,
completion: {
productContractDigest: productExecution.contractDigest,
productProjectedOutput: null,
productProjectedOutputDigest: null,
},
};
}
if (
Object.keys(semantic).sort().join(",") !==
"evidence,message,outcome,reasoning,schemaVersion"
) {
return { ok: false };
}
const message = semantic.message;
const candidate = message?.candidate;
const subject = message?.subject;
const hasSubject = typeof subject === "string";
const expectedMessageKeys = productExecution.subjectRequired
? "candidate,subject"
: "candidate";
if (
!message ||
typeof message !== "object" ||
Array.isArray(message) ||
Object.keys(message).sort().join(",") !== expectedMessageKeys ||
typeof candidate !== "string" ||
candidate.length < 1 ||
candidate.length > 32_768 ||
hasSubject !== productExecution.subjectRequired ||
(hasSubject && (subject.length < 1 || subject.length > 512)) ||
/\[SUPPRESS\]|\bSUPPRESS\b|\b(?:no|nothing)\s+truthful\b|\b(?:do|should)\s+not\s+send\b/i.test(
`${subject ?? ""}\n${candidate}`
)
) {
return { ok: false };
}
const projectedOutput = productExecution.subjectRequired
? { subject, body: candidate }
: { body: candidate };
return {
ok: true,
completion: {
productContractDigest: productExecution.contractDigest,
productProjectedOutput: projectedOutput,
productProjectedOutputDigest:
endpointCanonicalOutputDigest(projectedOutput),
},
};
}
if (
!proposal ||
typeof proposal !== "object" ||
Array.isArray(proposal) ||
proposal.schemaVersion !== productExecution?.nativeProposalSchemaVersion
) {
return { ok: false };
}
if (proposal.schemaVersion === "reply-agent-native-proposal.v2") {
const allowedKeys = new Set([
"schemaVersion",
"outcome",
"evidence",
"reasoning",
"actions",
"message",
]);
if (
Object.keys(proposal).some((key) => !allowedKeys.has(key)) ||
!Array.isArray(proposal.evidence) ||
typeof proposal.reasoning !== "string" ||
proposal.reasoning.length < 1 ||
proposal.reasoning.length > 4_096
) {
return { ok: false };
}
if (proposal.outcome === "SUPPRESS") {
if (
Object.hasOwn(proposal, "actions") ||
Object.hasOwn(proposal, "message")
) {
return { ok: false };
}
return {
ok: true,
completion: {
productContractDigest: productExecution.contractDigest,
productProjectedOutput: null,
productProjectedOutputDigest: null,
},
};
}
const message = proposal.message;
const candidate = message?.candidate;
const subject = message?.subject;
const hasSubject = typeof subject === "string";
const actions = proposal.actions;
const action = Array.isArray(actions) ? actions[0] : null;
const projectedArguments = productExecution.subjectRequired
? { subject, body: candidate }
: { body: candidate };
if (
proposal.outcome !== "DRAFT" ||
!message ||
typeof message !== "object" ||
Array.isArray(message) ||
typeof candidate !== "string" ||
candidate.length < 1 ||
hasSubject !== productExecution.subjectRequired ||
!Array.isArray(actions) ||
actions.length !== 1 ||
!action ||
action.actionKind !== "SEND_MESSAGE" ||
action.status !== "PROPOSED" ||
action.argumentHash !==
endpointCanonicalOutputDigest(projectedArguments) ||
/\[SUPPRESS\]|\bSUPPRESS\b|\b(?:no|nothing)\s+truthful\b|\b(?:do|should)\s+not\s+send\b/i.test(
`${subject ?? ""}\n${candidate}`
)
) {
return { ok: false };
}
return {
ok: true,
completion: {
productContractDigest: productExecution.contractDigest,
productProjectedOutput: projectedArguments,
productProjectedOutputDigest:
endpointCanonicalOutputDigest(projectedArguments),
},
};
}
if (proposal.schemaVersion !== "reply-agent-native-proposal.v1") {
return { ok: false };
}
const message = proposal?.message;
const candidate = message?.candidate;
const subject = message?.subject;
const hasSubject = typeof subject === "string";
if (
typeof candidate !== "string" ||
candidate.length < 1 ||
hasSubject !== productExecution.subjectRequired
) {
return { ok: false };
}
const projectedOutput = productExecution.subjectRequired
? { subject, body: candidate }
: { body: candidate };
return {
ok: true,
completion: {
productContractDigest: productExecution.contractDigest,
productProjectedOutput: projectedOutput,
productProjectedOutputDigest:
endpointCanonicalOutputDigest(projectedOutput),
},
};
}
export function endpointOpenerOutputContract(claim) {
if (!usesOpenerDraftStrategy(claim)) return undefined;
return openerOutputContractFor(claim);
}
// A single-run opener on a caller-named model (Opus on OpenRouter) is held to
// the same strict schema, which the bridge sends as the provider's json_schema
// so the model cannot return malformed JSON. The machine's default model has
// no strict transport, so it gets no contract. Neither does a Codex step: the
// bridge enforces the strict format only on OpenRouter. In a fallback chain
// the server attaches the schema for its custom Runs steps, so a Hermes step
// (Codex or OpenRouter) runs without it, as it would on its own.
export function endpointStrictOutputContract(
claim,
modelSelection,
{ chained = false } = {}
) {
if (
!modelSelection ||
(!modelSelection.runsApiBaseUrl &&
(chained || modelSelection.provider === "openai-codex")) ||
usesOpenerDraftStrategy(claim) ||
claim?.run?.productExecution?.workKind !== "OPENER" ||
!claim.run.productExecution.providerOutputSchema
)
return undefined;
return openerOutputContractFor(claim);
}
function openerOutputContractFor(claim) {
const execution = claim.run.productExecution;
if (
execution.nativeProposalSchemaVersion !==
"sellable-product-generation-semantic-proposal/v1"
)
return undefined;
const contract = validateOpenerOutputContract({
schemaVersion: "sellable-opener-output-contract/v1",
schema: execution.providerOutputSchema,
nativeProposalSchemaDigest: execution.nativeProposalSchemaDigest,
});
const variants = contract.schema.properties.proposal.anyOf;
const draft = variants?.find(
(variant) => variant.properties?.outcome?.enum?.[0] === "DRAFT"
);
const expected = execution.subjectRequired
? ["candidate", "subject"]
: ["candidate"];
if (
typeof execution.subjectRequired !== "boolean" ||
JSON.stringify(draft?.properties?.message?.required?.slice().sort()) !==
JSON.stringify(expected)
) {
throw new Error("hermes_opener_output_contract_rejected");
}
return contract;
}
export function endpointRunInstructions(claim) {
const productExecution = claim.run.productExecution;
if (productExecution == null) return claim.revision.prompt;
if (
!productExecution ||
typeof productExecution !== "object" ||
Array.isArray(productExecution) ||
!ENDPOINT_DIGEST.test(productExecution.contractDigest ?? "") ||
!["OPENER", "REPLY", "FOLLOW_UP"].includes(productExecution.workKind) ||
typeof productExecution.subjectRequired !== "boolean"
) {
throw new Error("customer_endpoint_product_contract_rejected");
}
const subjectRule = productExecution.subjectRequired
? "subjectRequired=true: include message.subject as a non-empty string."
: "subjectRequired=false: omit message.subject.";
const semanticRequirement =
productExecution.nativeProposalSchemaVersion ===
"sellable-product-generation-semantic-proposal/v1"
? [
"Return one semantic-only JSON envelope with exactly one top-level field: proposal.",
"proposal must contain schemaVersion=sellable-product-generation-semantic-proposal/v1, outcome=DRAFT|SUPPRESS, evidence=[{summary}], and reasoning.",
"For DRAFT include message.candidate and follow the subject rule below. For SUPPRESS omit message.",
"The provider is semantic-only: never include actions, IDs, hashes, or idempotency fields. The server derives all integrity and authority fields after validation.",
]
: [];
const openerJsonShapes =
usesOpenerDraftStrategy(claim) &&
productExecution.nativeProposalSchemaVersion ===
"sellable-product-generation-semantic-proposal/v1"
? [
"Choose one JSON shape below and replace angle-bracket placeholders with your content. Return only JSON, with no code fence, commentary, or alternate wrapper.",
"DRAFT JSON shape:",
JSON.stringify({
proposal: {
schemaVersion: productExecution.nativeProposalSchemaVersion,
outcome: "DRAFT",
evidence: [{ summary: "<supplied fact used>" }],
reasoning: "<brief decision summary>",
message: productExecution.subjectRequired
? {
subject: "<short subject>",
candidate: "<complete opener>",
}
: { candidate: "<complete opener>" },
},
}),
"SUPPRESS JSON shape:",
JSON.stringify({
proposal: {
schemaVersion: productExecution.nativeProposalSchemaVersion,
outcome: "SUPPRESS",
evidence: [],
reasoning: "<why no truthful draft is available>",
},
}),
]
: [];
return [
claim.revision.prompt,
"",
"## Server-owned product output requirement",
`workKind=${productExecution.workKind}`,
...semanticRequirement,
subjectRule,
...openerJsonShapes,
"This requirement is authoritative for the final message object.",
].join("\n");
}
function endpointPendingControl(value) {
if (value === null) return null;
const approvalKeys = [
"kind",
"nativeProposalDigest",
"proposalFingerprint",
"proposalId",
];
const stopKeys = ["kind", "stopRequestFingerprint"];
if (
!value ||
typeof value !== "object" ||
Array.isArray(value) ||
(value.kind === "APPROVAL_RELAY_PENDING" &&
(JSON.stringify(Object.keys(value).sort()) !==
JSON.stringify(approvalKeys.sort()) ||
!ENDPOINT_ID.test(value.proposalId ?? "") ||
!ENDPOINT_DIGEST.test(value.proposalFingerprint ?? "") ||
!ENDPOINT_DIGEST.test(value.nativeProposalDigest ?? ""))) ||
(value.kind === "STOP_PENDING" &&
(JSON.stringify(Object.keys(value).sort()) !==
JSON.stringify(stopKeys.sort()) ||
!ENDPOINT_DIGEST.test(value.stopRequestFingerprint ?? ""))) ||
!["APPROVAL_RELAY_PENDING", "STOP_PENDING"].includes(value.kind)
) {
throw new Error("customer_endpoint_control_rejected");
}
return Object.freeze({ ...value });
}
function endpointHeartbeatResponse(value, expectedFence, claim, activeRuntime) {
if (
!value ||
typeof value !== "object" ||
Array.isArray(value) ||
JSON.stringify(Object.keys(value).sort()) !==
JSON.stringify([
"control",
"fence",
"leaseExpiresAt",
"mcpTurnDescriptor",
"renewed",
]) ||
value.renewed !== true ||
value.fence !== expectedFence ||
typeof value.leaseExpiresAt !== "string" ||
!Number.isFinite(Date.parse(value.leaseExpiresAt))
) {
throw new Error("customer_endpoint_heartbeat_rejected");
}
const descriptorBinding = endpointMcpDescriptorBinding(
{ ...claim, mcpTurnDescriptor: value.mcpTurnDescriptor },
activeRuntime
);
return {
control: endpointPendingControl(value.control),
descriptorBinding,
};
}
function nativeControlReceipt(value) {
return flyRuntimeDigest({
classification:
typeof value?.classification === "string"
? value.classification.slice(0, 64)
: null,
ok: value?.ok === true,
positiveCancellation: value?.positiveCancellation === true,
resolved: value?.resolved === true,
status:
typeof value?.status === "string" ? value.status.slice(0, 64) : null,
});
}
async function abortablePause(ms, signal, sleepImpl) {
if (signal.aborted) return;
await Promise.race([
sleepImpl(ms),
new Promise((resolve) =>
signal.addEventListener("abort", resolve, { once: true })
),
]);
}
/** @param {any} options */
export function createCustomerEndpointRelay(options = {}) {
const {
getControlClient,
getActiveRuntime,
runtimeGeneration,
getApiKeyFile = ({ claim }) =>
join(SECRET_ROOT, claim.runtime.profileId, "api-server-key"),
createRunsClient = createConfiguredRunsClient,
now = () => new Date(),
sleepImpl = sleep,
diagnostic = () => {},
installDescriptor = installMcpTurnDescriptor,
replaceDescriptor = replaceMcpTurnDescriptorAtomically,
} = options;
if (
typeof getControlClient !== "function" ||
typeof getActiveRuntime !== "function" ||
typeof getApiKeyFile !== "function" ||
typeof createRunsClient !== "function" ||
typeof now !== "function" ||
typeof sleepImpl !== "function" ||
typeof diagnostic !== "function" ||
typeof installDescriptor !== "function" ||
typeof replaceDescriptor !== "function" ||
!Number.isSafeInteger(runtimeGeneration) ||
runtimeGeneration < 1
) {
throw new Error("customer_endpoint_relay_configuration_rejected");
}
const activeAttempts = new Set();
let failures = 0;
const maximumConcurrentRuns = ENDPOINT_MAX_CONCURRENT_RUNS;
// Each redacted event is bounded to 8 KiB and the signed request is bounded
// to 64 KiB. Six events leave room for identity/signature envelope overhead
// while replacing the previous one-HTTP-request-per-event handoff.
const endpointEventBatchSize = 6;
const appendMany = async (claim, identity, indexedEvents) => {
if (!endpointRuntimeCurrent(claim, getActiveRuntime(), runtimeGeneration)) {
throw new Error("customer_endpoint_generation_fenced");
}
const envelopes = indexedEvents.map(({ event, ordinal }) =>
endpointEventEnvelope(event, ordinal)
);
const value = await getControlClient().appendEndpointEvents({
...identity,
schemaVersion: "sellable-agent-endpoint-events/v1",
events: envelopes,
});
if (
value?.accepted !== true ||
!Array.isArray(value.events) ||
value.events.length !== envelopes.length ||
value.events.some(
(persisted, index) =>
persisted?.nativeEventId !== envelopes[index].nativeEventId ||
typeof persisted?.sequence !== "string"
)
) {
throw new Error(
indexedEvents.some(
({ event }) => event.kind === "LATE_COMPLETION_NOTICE"
)
? "customer_endpoint_late_notice_rejected"
: "customer_endpoint_event_persist_rejected"
);
}
return value.events.map((event) => event.sequence);
};
const append = async (claim, identity, event, ordinal) => {
const sequences = await appendMany(claim, identity, [{ event, ordinal }]);
return sequences[0];
};
const complete = async (claim, identity, completion) => {
if (!endpointRuntimeCurrent(claim, getActiveRuntime(), runtimeGeneration)) {
return { accepted: false, code: "GENERATION_FENCED" };
}
return getControlClient().completeEndpoint(
endpointCompletionBody({ identity, now, ...completion })
);
};
const acknowledgeControl = async (held, key, body, { final = true } = {}) => {
const response = await getControlClient().acknowledgeEndpointControl({
...held.identity,
schemaVersion: "sellable-agent-endpoint-control/v1",
...body,
});
if (
!response ||
typeof response !== "object" ||
Array.isArray(response) ||
JSON.stringify(Object.keys(response).sort()) !==
JSON.stringify(["accepted", "code"]) ||
typeof response.accepted !== "boolean" ||
typeof response.code !== "string" ||
response.code.length < 1 ||
response.code.length > 128
) {
throw new Error("customer_endpoint_control_response_rejected");
}
if (response.accepted === true && final) held.processedControls.add(key);
return response;
};
const processApprovalControl = async (held, control) => {
if (held.openerStages)
throw new Error("OPENER_STAGE_TOOL_POLICY_VIOLATION");
const key = `approval:${control.proposalFingerprint}`;
if (held.processedControls.has(key)) return;
const pendingAck = held.controlReceipts.get(key);
if (pendingAck) {
await acknowledgeControl(held, key, pendingAck);
return;
}
if (!held.nativeRunId) return;
const status = await held.runs.status(held.nativeRunId, {
signal: held.controller.signal,
});
if (status?.ok !== true || status.classification !== "AWAITING_APPROVAL") {
// Do not authorize or persist a choice while Hermes is not currently at
// the exact pending approval boundary. A later heartbeat re-reads it.
return;
}
let authorization = held.approvalAuthorizations.get(key);
if (!authorization) {
const probed = await getControlClient().acknowledgeEndpointControl({
...held.identity,
schemaVersion: "sellable-agent-endpoint-control/v1",
kind: "APPROVAL_RELAY_PROBE",
proposalId: control.proposalId,
proposalFingerprint: control.proposalFingerprint,
nativeProposalDigest: control.nativeProposalDigest,
});
const probeKeys = Object.keys(probed ?? {}).sort();
if (
!probed ||
typeof probed !== "object" ||
Array.isArray(probed) ||
typeof probed.accepted !== "boolean" ||
typeof probed.code !== "string" ||
probed.code.length < 1 ||
probed.code.length > 128 ||
(probed.accepted === false &&
JSON.stringify(probeKeys) !== JSON.stringify(["accepted", "code"])) ||
(probed.accepted === true &&
![
JSON.stringify(["accepted", "code"]),
JSON.stringify(["accepted", "code", "relayFingerprint"]),
JSON.stringify([
"accepted",
"code",
"relayChoice",
"relayFingerprint",
]),
].includes(JSON.stringify(probeKeys)))
) {
throw new Error("customer_endpoint_approval_probe_rejected");
}
if (probed?.accepted !== true) {
held.processedControls.add(key);
return;
}
if (
!["once", "deny"].includes(probed.relayChoice) ||
!ENDPOINT_DIGEST.test(probed.relayFingerprint ?? "")
) {
// An idempotent duplicate may omit its choice. It is usable only when
// this process already cached the original authorization.
return;
}
authorization = Object.freeze({
relayChoice: probed.relayChoice,
relayFingerprint: probed.relayFingerprint,
});
held.approvalAuthorizations.set(key, authorization);
}
let native;
try {
native = await held.runs.approve(
held.nativeRunId,
authorization.relayChoice,
{ signal: held.controller.signal }
);
} catch (error) {
native = {
ok: false,
classification: "NATIVE_APPROVAL_UNCERTAIN",
errorType: error instanceof Error ? error.name : typeof error,
};
}
const outcome =
native?.ok === true && native.resolved === true
? "APPLIED"
: native?.classification === "NATIVE_APPROVAL_REFUSED"
? "REFUSED"
: "UNCERTAIN";
const ack = Object.freeze({
kind: "APPROVAL_RELAY_ACK",
proposalId: control.proposalId,
proposalFingerprint: control.proposalFingerprint,
relayFingerprint: authorization.relayFingerprint,
relayChoice: authorization.relayChoice,
outcome,
nativeReceiptDigest: nativeControlReceipt(native),
});
// Record the exact ACK before transport. If ACK delivery is ambiguous, a
// later heartbeat retries this body and never calls native approve twice.
held.controlReceipts.set(key, ack);
await acknowledgeControl(held, key, ack);
};
const processStopControl = async (held, control) => {
const key = `stop:${control.stopRequestFingerprint}`;
if (held.processedControls.has(key)) return;
const finalAck = held.controlReceipts.get(key);
if (finalAck) {
await acknowledgeControl(held, key, finalAck);
return;
}
const state = held.stopStates.get(key) ?? {
stopIssued: false,
intermediateAck: null,
};
held.stopStates.set(key, state);
if (state.intermediateAck) {
const response = await acknowledgeControl(
held,
key,
state.intermediateAck,
{
final: false,
}
);
if (response.accepted === true) state.intermediateAck = null;
return;
}
if (!held.nativeRunId && !held.openerStages) return;
const controlledRuns = held.openerStages ?? held.runs;
let native = await controlledRuns.status(held.nativeRunId, {
signal: held.controller.signal,
});
let outcome;
if (
native?.classification === "CANCELLED_POSITIVE" &&
native.positiveCancellation === true
) {
outcome = "CANCELLED";
} else if (native?.classification === "NATIVE_HANDLE_LOST") {
outcome = "RESTART_LOST";
} else if (native?.terminal === true) {
outcome = "NOT_FOUND";
} else if (!state.stopIssued) {
try {
native = await controlledRuns.stop(held.nativeRunId, {
signal: held.controller.signal,
});
state.stopIssued = true;
} catch (error) {
native = {
ok: false,
classification: "STOP_UNCERTAIN",
errorType: error instanceof Error ? error.name : typeof error,
};
}
outcome =
native?.positiveCancellation === true
? "CANCELLED"
: native?.classification === "STOPPING_UNCERTAIN"
? "STOPPING"
: native?.classification === "STOP_RACE_UNCERTAIN"
? "NOT_FOUND"
: native?.classification === "NATIVE_HANDLE_LOST"
? "RESTART_LOST"
: "TRANSPORT_AMBIGUOUS";
} else if (
native?.classification === "STOPPING_UNCERTAIN" ||
native?.classification === "ACTIVE" ||
native?.status === "stopping" ||
native?.status === "running" ||
native?.status === "queued"
) {
// STOPPING is a receipt, not terminal cancellation. Keep polling this
// exact fingerprint without issuing another native stop.
return;
} else {
outcome = "TRANSPORT_AMBIGUOUS";
}
const ack = Object.freeze({
kind: "STOP_ACK",
stopRequestFingerprint: control.stopRequestFingerprint,
outcome,
nativeReceiptDigest: nativeControlReceipt(native),
});
if (outcome === "STOPPING") {
state.intermediateAck = ack;
const response = await acknowledgeControl(held, key, ack, {
final: false,
});
if (response.accepted === true) state.intermediateAck = null;
return;
}
held.controlReceipts.set(key, ack);
await acknowledgeControl(held, key, ack);
};
const processPendingControl = async (held, control) => {
if (!control || held.controller.signal.aborted) return;
if (control.kind === "STOP_PENDING") {
await processStopControl(held, control);
return;
}
await processApprovalControl(held, control);
};
const runClaim = async (claim, outerSignal) => {
const identity = endpointClaimIdentity(claim);
// A caller-named model rides on the run; absent means the machine default.
const modelSelection =
claim?.run?.modelSelection === undefined
? null
: validateEndpointModelSelection(claim.run.modelSelection);
const activeRuntime = getActiveRuntime();
if (!endpointRuntimeCurrent(claim, activeRuntime, runtimeGeneration)) {
return { ok: false, code: "customer_endpoint_generation_fenced" };
}
const descriptorBinding = endpointMcpDescriptorBinding(
claim,
activeRuntime
);
const hermesSessionId = endpointHermesSessionId(
descriptorBinding.descriptor.body
);
const descriptorPath = endpointMcpDescriptorPath(
activeRuntime.runtimeConfig.turnDescriptorPath,
hermesSessionId
);
if (!descriptorPath) {
throw new Error("customer_endpoint_hermes_session_rejected");
}
const installedDescriptor = installDescriptor({
descriptorPath,
descriptorRoot: dirname(descriptorPath),
descriptor: descriptorBinding.descriptor,
trustRootPublicKeyPem: activeRuntime.trustRootPem,
trustRootFingerprint: sha256(activeRuntime.trustRootPem),
expected: descriptorBinding.expected,
now: now(),
});
if (installedDescriptor.ok !== true) {
throw new Error("customer_endpoint_mcp_descriptor_rejected");
}
const apiKeyFile = getApiKeyFile({ claim, activeRuntime });
// The models this run tries, in order. Each step opens its own client from
// a claim carrying exactly that step's selection, so the trusted
// connection checks are the same as for a single-model run.
const chain = endpointModelSelectionChain(modelSelection);
const chained = chain.length > 1;
// A one-model run is a chain of one; no selection is the Agent default.
const steps = chain.length ? chain : [null];
// No selection is the Agent default, built here as before (the opener
// stages use it). A chain opens each step inside the run loop instead.
const runs = chain.length
? null
: await createRunsClient({ apiKeyFile, claim, activeRuntime });
// A workspace connection step's key, released to this attempt only. Its
// literal values are scrubbed from this attempt's error code: the
// diagnostic patterns miss a hex key and an srt_ grant.
const released = [];
const fetchConnectionKey = async (connectionId, chainStep) => {
const value = await getControlClient().fetchEndpointConnectionKey({
...identity,
schemaVersion: "sellable-agent-endpoint-connection-key/v1",
connectionId,
chainStep,
});
for (const secret of [value?.apiKey, value?.tools?.authorization])
if (typeof secret === "string" && secret) released.push(secret);
return value;
};
// A step whose client cannot be opened sent nothing for this step; it
// counts as a clean failure (STEP_UNAVAILABLE).
const openStep = async (index) => {
try {
return await createRunsClient({
apiKeyFile,
claim: {
...claim,
run: { ...claim.run, modelSelection: chain[index] },
},
activeRuntime,
chainStep: index,
fetchConnectionKey,
});
} catch {
return null;
}
};
const controller = new AbortController();
const abortFromOuter = () => controller.abort(outerSignal.reason);
outerSignal.addEventListener("abort", abortFromOuter, { once: true });
const deadlineMs = Math.min(
Date.parse(claim.attempt.deadlineAt),
now().getTime() + claim.revision.timeoutSeconds * 1_000
);
const uatCompletionControl = validateEndpointUatCompletionControl(
claim.uatCompletionControl,
claim
);
let deadlineExpired = now().getTime() >= deadlineMs;
const deadlineTimer = setTimeout(
() => {
// Durable server state owns the timeout transition. Keep observing the
// already-submitted native attempt so a later terminal can be appended
// only as LATE_COMPLETION_NOTICE; never abort/resubmit at the boundary.
deadlineExpired = true;
},
Math.max(0, deadlineMs - now().getTime())
);
let heartbeatFailure = null;
let heartbeatStopped = false;
const held = {
claim,
identity,
runs,
controller,
nativeRunId: null,
openerStages: null,
stopRequested: false,
pendingControl: null,
processedControls: new Set(),
approvalAuthorizations: new Map(),
controlReceipts: new Map(),
stopStates: new Map(),
controlChain: Promise.resolve(),
};
const queueControl = async (control) => {
held.controlChain = held.controlChain
.then(() => processPendingControl(held, control))
.catch((error) => {
diagnostic({
ok: false,
code: "customer_endpoint_control_retry_pending",
errorDigest: sha256(
error instanceof Error ? error.name : typeof error
),
});
});
await held.controlChain;
};
activeAttempts.add(held);
const heartbeat = (async () => {
while (!controller.signal.aborted && !heartbeatStopped) {
if (deadlineExpired || now().getTime() >= deadlineMs) {
// The durable server owns timeout at the deadline and refuses lease
// renewal afterward. Stop heartbeating without aborting the already
// submitted native stream so a late terminal remains notice-only.
heartbeatStopped = true;
break;
}
try {
const value = await getControlClient().heartbeatEndpoint({
...identity,
schemaVersion: "sellable-agent-endpoint-heartbeat/v1",
leaseSeconds: 60,
});
const heartbeatResponse = endpointHeartbeatResponse(
value,
identity.fence,
claim,
activeRuntime
);
const refreshedDescriptor = replaceDescriptor({
descriptorPath,
descriptorRoot: dirname(descriptorPath),
descriptor: heartbeatResponse.descriptorBinding.descriptor,
trustRootPublicKeyPem: activeRuntime.trustRootPem,
trustRootFingerprint: sha256(activeRuntime.trustRootPem),
expected: heartbeatResponse.descriptorBinding.expected,
now: now(),
});
if (refreshedDescriptor.ok !== true) {
throw new Error(
"customer_endpoint_mcp_descriptor_refresh_rejected"
);
}
const control = heartbeatResponse.control;
held.pendingControl = control;
if (held.nativeRunId && control) {
await queueControl(control);
}
} catch (error) {
heartbeatFailure = error;
controller.abort(error);
break;
}
await abortablePause(20_000, controller.signal, sleepImpl);
}
})();
let ordinal = 0;
const pendingStreamEvents = [];
const flushStreamEvents = async () => {
if (pendingStreamEvents.length === 0) return;
const batch = pendingStreamEvents.slice(0, endpointEventBatchSize);
const sequences = await appendMany(claim, identity, batch);
pendingStreamEvents.splice(0, batch.length);
held.committedSequence = sequences.at(-1);
};
const done = (result) => ({ done: true, result });
// Runs one model of the chain. A final outcome has already completed the
// attempt (`done`); a clean pre-output failure returns its class and a
// `finish` that completes the attempt as a single model would have.
const runNativeStep = async (stepIndex, stepRuns) => {
const selection = chain[stepIndex] ?? null;
// Events of a chained run name the model they belong to.
const tag = (event) =>
chained ? { ...event, chainStep: stepIndex } : event;
if (!stepRuns) {
return {
failure: "STEP_UNAVAILABLE",
finish: async () => {
await complete(claim, identity, {
attemptStatus: "FAILED",
status: "FAILED",
});
return { ok: false, code: "FAILED" };
},
};
}
held.runs = stepRuns;
held.nativeRunId = null;
let submitted;
let terminalEvent = null;
const submitRequest = {
input: claim.run.input,
instructions: endpointRunInstructions(claim),
conversationHistory: endpointConversationHistory(
claim.run.conversationHistory
),
correlation: `${claim.attempt.conversationId}:${claim.attempt.runId}:${claim.attempt.id}`,
...(selection ? { modelSelection: selection } : {}),
};
const strictOutputContract = endpointStrictOutputContract(
claim,
selection,
{ chained }
);
if (strictOutputContract)
submitRequest.outputContract = strictOutputContract;
// A busy step stops waiting at its own budget so the next model still
// has time; the last step (or a single model) waits until the deadline.
const admissionDeadlineMs = endpointChainStepAdmissionDeadline({
startedAtMs: now().getTime(),
deadlineMs,
stepIndex,
stepCount: steps.length,
});
while (!submitted) {
try {
submitted = await stepRuns.submit(submitRequest, {
signal: controller.signal,
});
} catch (error) {
if (error?.classification === "SUBMIT_AMBIGUOUS") {
await complete(claim, identity, {
attemptStatus: "UNCERTAIN",
status: "UNCERTAIN",
});
return done({ ok: false, code: "SUBMIT_AMBIGUOUS" });
}
if (error?.classification !== "SUBMIT_NOT_ADMITTED") throw error;
const notAdmittedTimeout = async () => {
await complete(claim, identity, {
attemptStatus: "TIMED_OUT",
status: "TIMED_OUT",
});
return { ok: false, code: "SUBMIT_NOT_ADMITTED_TIMEOUT" };
};
const remainingMs = deadlineMs - now().getTime();
if (deadlineExpired || remainingMs <= 0)
return done(await notAdmittedTimeout());
const admissionRemainingMs = admissionDeadlineMs - now().getTime();
if (admissionRemainingMs <= 0)
return { failure: "NOT_ADMITTED", finish: notAdmittedTimeout };
const retryAfterMs = Number.isFinite(error.retryAfterMs)
? Math.max(100, Math.min(5_000, error.retryAfterMs))
: 1_000;
await abortablePause(
Math.min(retryAfterMs, admissionRemainingMs),
controller.signal,
sleepImpl
);
if (heartbeatFailure) throw heartbeatFailure;
if (controller.signal.aborted) {
throw controller.signal.reason ?? new Error("endpoint_run_aborted");
}
}
}
held.nativeRunId = submitted.runId;
await append(
claim,
identity,
tag({
kind: "NATIVE_SUBMIT_ACCEPTED",
requestDigest: submitted.requestDigest,
nativeRunIdDigest: sha256(submitted.runId),
}),
ordinal++
);
if (held.pendingControl) await queueControl(held.pendingControl);
let streamed;
try {
streamed = await stepRuns.streamEvents(submitted.runId, {
signal: controller.signal,
onEvent: async (event) => {
const observedEvent = await holdEndpointUatNativeCompletion({
control: uatCompletionControl,
claim,
event,
now,
sleep: sleepImpl,
signal: controller.signal,
});
const late = deadlineExpired || now().getTime() >= deadlineMs;
const lateCompletion =
late && observedEvent.kind === "RUN_COMPLETED";
// Late notices use a distinct post-deadline authorization path and
// cannot share a signed batch with ordinary pre-deadline events.
if (lateCompletion) {
while (pendingStreamEvents.length > 0) {
await flushStreamEvents();
}
}
const durableEvent = lateCompletion
? {
kind: "LATE_COMPLETION_NOTICE",
outputDigest: observedEvent.outputDigest,
timestamp: observedEvent.timestamp,
nativeEventIdentity: observedEvent.nativeEventIdentity,
}
: observedEvent;
pendingStreamEvents.push({
event: tag(durableEvent),
ordinal: ordinal++,
});
const approvalBoundary =
observedEvent.classification === "AWAITING_APPROVAL" ||
String(observedEvent.kind ?? "").includes("APPROVAL");
if (
pendingStreamEvents.length >= endpointEventBatchSize ||
observedEvent.terminal ||
approvalBoundary
) {
await flushStreamEvents();
}
// Terminal state is assigned only after its durable event batch.
if (observedEvent.terminal)
terminalEvent = { ...observedEvent, late };
},
});
} finally {
while (pendingStreamEvents.length > 0) {
await flushStreamEvents();
}
}
if (!terminalEvent && !controller.signal.aborted) {
const status = await stepRuns.status(submitted.runId, {
signal: controller.signal,
});
if (status.terminal) {
const observedStatus = await holdEndpointUatNativeCompletion({
control: uatCompletionControl,
claim,
event: status,
now,
sleep: sleepImpl,
signal: controller.signal,
});
terminalEvent = {
...observedStatus,
late: deadlineExpired || now().getTime() >= deadlineMs,
};
} else if (status.ok === false) {
await append(claim, identity, tag(status), ordinal++);
}
}
if (
terminalEvent?.classification === "COMPLETED" ||
terminalEvent?.kind === "RUN_COMPLETED"
) {
const late =
terminalEvent.late ||
deadlineExpired ||
now().getTime() >= deadlineMs;
if (late) {
// Server timeout is already canonical. The committed digest-only
// late notice above is the sole mutation; never replace timeout with
// a second completion or expose the late output as canonical.
return done({ ok: true, code: "LATE_NOTICE_ONLY" });
}
const productProjection = endpointProductProjection(
claim.run.productExecution,
terminalEvent.output
);
if (!productProjection.ok) {
// The output was never used, so another model may still write.
return {
failure: "OUTPUT_CONTRACT_FAILED",
finish: async () => {
await complete(claim, identity, {
attemptStatus: "SUCCEEDED",
status: "OUTPUT_CONTRACT_FAILED",
});
return { ok: false, code: "OUTPUT_CONTRACT_FAILED" };
},
};
}
await complete(claim, identity, {
attemptStatus: "SUCCEEDED",
status: "SUCCEEDED",
output: terminalEvent.output,
...productProjection.completion,
});
return done({ ok: true, code: "SUCCEEDED" });
}
if (
terminalEvent?.classification === "CANCELLED_POSITIVE" ||
terminalEvent?.kind === "RUN_CANCELLED"
) {
await complete(claim, identity, {
attemptStatus: "CANCELLED",
status: "CANCELLED_PRE_EFFECT",
positiveCancellation: true,
});
return done({ ok: true, code: "CANCELLED" });
}
if (terminalEvent) {
// A native failure carries no output (Claude Code usage limit and
// sign-in refusals arrive this way too).
return {
failure: "NATIVE_FAILED",
finish: async () => {
await complete(claim, identity, {
attemptStatus: "FAILED",
status: "FAILED",
});
return { ok: false, code: "FAILED" };
},
};
}
const deadline = deadlineExpired || now().getTime() >= deadlineMs;
await complete(claim, identity, {
attemptStatus: deadline ? "TIMED_OUT_UNCERTAIN" : "UNCERTAIN",
status: deadline ? "TIMED_OUT_UNCERTAIN" : "UNCERTAIN",
});
return done({
ok: false,
code:
streamed?.classification ??
(deadline ? "TIMED_OUT_UNCERTAIN" : "NATIVE_LOST_UNCERTAIN"),
});
};
try {
if (usesOpenerDraftStrategy(claim) && modelSelection) {
// The three-draft stages are pinned to the Codex output-format path in
// the Hermes bridge; refuse rather than silently serve the default.
await complete(claim, identity, {
attemptStatus: "FAILED",
status: "FAILED",
});
return { ok: false, code: "OPENER_STAGE_MODEL_SELECTION_UNSUPPORTED" };
}
if (usesOpenerDraftStrategy(claim)) {
// Phase 206: writers are private native sessions within this one fenced
// claim. Only the editor's validated result reaches durable completion.
let stageEventChain = Promise.resolve();
const appendStageEvent = (event) => {
stageEventChain = stageEventChain.then(() =>
append(claim, identity, event, ordinal++)
);
return stageEventChain;
};
held.openerStages = createOpenerStageExecution({
runs,
input: claim.run.input,
instructions: endpointRunInstructions(claim),
outputContract: endpointOpenerOutputContract(claim),
correlation: `${claim.attempt.conversationId}:${claim.attempt.runId}:${claim.attempt.id}:${randomUUID()}`,
deadlineAt: new Date(deadlineMs),
signal: controller.signal,
now,
validateOutput: (output) =>
endpointProductProjection(claim.run.productExecution, output).ok,
onSubmitted: async ({ stage, runId, requestDigest }) => {
const first = held.nativeRunId === null;
held.nativeRunId ??= runId;
await appendStageEvent({
kind: first
? "NATIVE_SUBMIT_ACCEPTED"
: "OPENER_STAGE_SUBMIT_ACCEPTED",
stage,
requestDigest,
nativeRunIdDigest: sha256(runId),
});
if (held.pendingControl) await queueControl(held.pendingControl);
},
onReceipt: async (receipt) => {
await appendStageEvent({
kind: "OPENER_STAGE_COMPLETED",
...receipt,
});
},
});
const selected = await held.openerStages.run();
const observed = await holdEndpointUatNativeCompletion({
control: uatCompletionControl,
claim,
event: {
kind: "RUN_COMPLETED",
terminal: true,
output: selected.output,
outputDigest: selected.outputDigest,
timestamp: selected.timestamp,
nativeEventIdentity: selected.nativeEventIdentity,
},
now,
sleep: sleepImpl,
signal: controller.signal,
});
const late = deadlineExpired || now().getTime() >= deadlineMs;
await append(
claim,
identity,
late
? {
kind: "LATE_COMPLETION_NOTICE",
outputDigest: observed.outputDigest,
timestamp: observed.timestamp,
nativeEventIdentity: observed.nativeEventIdentity,
}
: observed,
ordinal++
);
if (late) return { ok: true, code: "LATE_NOTICE_ONLY" };
const projection = endpointProductProjection(
claim.run.productExecution,
selected.output
);
if (!projection.ok) throw new Error("OPENER_STAGE_OUTPUT_INVALID");
const completion = await complete(claim, identity, {
attemptStatus: "SUCCEEDED",
status: "SUCCEEDED",
output: selected.output,
...projection.completion,
});
return completion?.accepted === true
? { ok: true, code: "SUCCEEDED" }
: { ok: false, code: "OPENER_STAGE_COMPLETION_REJECTED" };
}
// A step that failed cleanly before producing any output hands the run
// to the next model; every other outcome completes this attempt once.
// Only when the failed step cannot have changed anything: a product draft
// run has no effects by construction, and an endpoint without tools has
// nothing to call. Any other run may already have used a tool, so another
// model must not repeat the turn.
const chainMayAdvance =
claim.run.productExecution != null ||
(descriptorBinding.expected.allowedToolNames ?? []).length === 0;
const history = [];
let stepIndex = 0;
let stepRuns = chain.length ? await openStep(0) : runs;
for (;;) {
const outcome = await runNativeStep(stepIndex, stepRuns);
if (outcome.done) return outcome.result;
// A stop, a gateway fence or a lost lease (which aborts the
// controller) ends the run as a single model would have ended it.
const stopRequested =
held.stopRequested ||
held.pendingControl?.kind === "STOP_PENDING" ||
controller.signal.aborted;
const decision =
stopRequested || !chainMayAdvance
? { advance: false }
: nextEndpointChainStep(
steps,
history,
outcome.failure,
deadlineMs - now().getTime()
);
if (!decision.advance) return await outcome.finish();
await append(
claim,
identity,
{
kind: "CHAIN_STEP_ADVANCED",
classification: outcome.failure,
chainStep: stepIndex,
},
ordinal++
);
history.push(outcome.failure);
stepIndex = decision.stepIndex;
stepRuns = await openStep(stepIndex);
}
} catch (error) {
if (heartbeatFailure) throw heartbeatFailure;
if (
error?.message === "customer_endpoint_generation_fenced" ||
error?.message === "customer_endpoint_late_notice_rejected"
) {
return { ok: false, code: error.message };
}
if (
held.openerStages &&
!deadlineExpired &&
now().getTime() < deadlineMs
) {
const cancelled =
error?.code === "OPENER_STAGE_STOPPED" &&
(await held.openerStages.status()).positiveCancellation === true;
const invalid = error?.code === "OPENER_STAGE_OUTPUT_INVALID";
const failed = error?.code === "OPENER_STAGE_NATIVE_FAILED";
await complete(claim, identity, {
attemptStatus: cancelled
? "CANCELLED"
: invalid
? "SUCCEEDED"
: failed
? "FAILED"
: "UNCERTAIN",
status: cancelled
? "CANCELLED_PRE_EFFECT"
: invalid
? "OUTPUT_CONTRACT_FAILED"
: failed
? "FAILED"
: "UNCERTAIN",
positiveCancellation: cancelled,
});
return { ok: false, code: error?.code ?? "OPENER_STAGE_FAILED" };
}
await complete(claim, identity, {
attemptStatus:
deadlineExpired || now().getTime() >= deadlineMs
? "TIMED_OUT_UNCERTAIN"
: "UNCERTAIN",
status:
deadlineExpired || now().getTime() >= deadlineMs
? "TIMED_OUT_UNCERTAIN"
: "UNCERTAIN",
});
return {
ok: false,
code:
error instanceof Error
? redactRuntimeDiagnostic(
released.reduce(
(text, secret) => text.replaceAll(secret, "[redacted]"),
error.message
)
)
: "customer_endpoint_relay_failed",
};
} finally {
released.length = 0;
clearTimeout(deadlineTimer);
outerSignal.removeEventListener("abort", abortFromOuter);
heartbeatStopped = true;
controller.abort();
await heartbeat;
await held.controlChain;
if (existsSync(descriptorPath)) unlinkSync(descriptorPath);
activeAttempts.delete(held);
}
};
return Object.freeze({
readiness(activeRuntime = getActiveRuntime()) {
if (!activeRuntime?.desired) return null;
return Object.freeze({
capability: "fixture_bound_runs_relay_v1",
fixtureDigest: HERMES_RUNS_FIXTURE_IDENTITY.fixtureDigest,
host: "127.0.0.1",
port: 8642,
publicListener: false,
authRequired: true,
runtimeGeneration,
profileIdDigest: sha256(activeRuntime.desired.profileId ?? ""),
});
},
async fenceActive(reason = "GATEWAY_RESTART") {
const heldAttempts = [...activeAttempts].filter(
(held) => !held.stopRequested
);
if (heldAttempts.length === 0)
return { ok: true, code: "NO_ACTIVE_ATTEMPT" };
const results = await Promise.all(
heldAttempts.map(async (held) => {
held.stopRequested = true;
let stopResult = {
ok: false,
classification: "RESTART_LOST",
certainty: "UNCERTAIN",
};
if (held.nativeRunId || held.openerStages) {
try {
stopResult = await (held.openerStages ?? held.runs).stop(
held.nativeRunId,
{
signal: AbortSignal.timeout(5_000),
}
);
} catch {
stopResult = {
ok: false,
classification: "STOP_UNCERTAIN",
certainty: "UNCERTAIN",
};
}
}
try {
await append(
held.claim,
held.identity,
{
kind: "GATEWAY_FENCE_REQUESTED",
classification: stopResult.classification ?? "STOP_UNCERTAIN",
requestDigest: sha256(reason),
},
Number.MAX_SAFE_INTEGER
);
} catch {
// A newer runtime generation is expected to reject this old fence.
}
held.controller.abort(new Error("customer_endpoint_gateway_fenced"));
return stopResult;
})
);
const uncertain = results.find(
(result) => result.positiveCancellation !== true
);
return {
ok: !uncertain,
code: uncertain
? (uncertain.classification ?? "STOP_UNCERTAIN")
: "CANCELLED",
};
},
async run(signal) {
if (!(signal instanceof AbortSignal)) {
throw new Error("customer_endpoint_relay_signal_rejected");
}
const inFlight = new Set();
const launch = (claim) => {
let task;
task = (async () => {
try {
const result = await runClaim(claim, signal);
diagnostic({
ok: result.ok,
code: result.code,
attemptIdDigest: sha256(claim.attempt?.id ?? ""),
});
} catch (error) {
diagnostic({
ok: false,
code:
error instanceof Error
? redactRuntimeDiagnostic(error.message)
: "customer_endpoint_run_failed",
attemptIdDigest: sha256(claim.attempt?.id ?? ""),
});
} finally {
inFlight.delete(task);
}
})();
inFlight.add(task);
};
while (!signal.aborted) {
try {
if (inFlight.size >= maximumConcurrentRuns) {
await Promise.race(inFlight);
continue;
}
const activeRuntime = getActiveRuntime();
if (!activeRuntime?.gateway?.readback?.().running) {
await abortablePause(500, signal, sleepImpl);
continue;
}
const claim = await getControlClient().claimEndpoint(60);
if (!claim) {
failures = 0;
await abortablePause(1_000, signal, sleepImpl);
continue;
}
launch(claim);
failures = 0;
} catch (error) {
failures = Math.min(failures + 1, 6);
diagnostic({
ok: false,
code:
error instanceof Error
? redactRuntimeDiagnostic(error.message)
: "customer_endpoint_loop_failed",
});
await abortablePause(
Math.min(30_000, 500 * 2 ** failures),
signal,
sleepImpl
);
}
}
await Promise.allSettled(inFlight);
},
});
}
async function consumeSecrets({ claim, exchange, client, desired }) {
const identity = operationIdentity(claim, exchange);
const consume = async (descriptor) => {
const value = await client.consumeSecret({
...identity,
envelopeId: descriptor.id,
claimId: randomUUID(),
});
if (
value.envelopeId !== descriptor.id ||
value.generation !== descriptor.generation ||
typeof value.secret !== "string" ||
value.secret.length < 1 ||
sha256(value.secret).slice(0, 16) !== descriptor.fingerprint
) {
throw new Error("customer_secret_delivery_rejected");
}
return value.secret;
};
const [mcpService, slackApp, slackBot, modelCodex, hermesApiServer] =
await Promise.all([
consume(claim.secretEnvelopes.mcpService),
consume(claim.secretEnvelopes.slackApp),
consume(claim.secretEnvelopes.slackBot),
consume(claim.secretEnvelopes.modelCodex),
consume(claim.secretEnvelopes.hermesApiServer),
]);
const expected = { ...desired, fence: claim.operation.fence };
const envelope = (purpose, provider, secret, generation) => ({
purpose,
provider,
secret,
fingerprint: sha256(secret).slice(0, 16),
generation,
binding: {
workspaceId: expected.workspaceId,
agentId: expected.agentId,
runtimeId: expected.runtimeId,
profileId: expected.profileId,
revisionId: expected.revisionId,
fence: expected.fence,
purpose,
},
});
const installed = installCustomerRuntimeSecrets({
expected,
secretsRoot: SECRET_ROOT,
envelopes: {
slackApp: envelope(
"slack-app",
"slack",
slackApp,
desired.slackAppGeneration
),
slackBot: envelope(
"slack-bot",
"slack",
slackBot,
desired.slackGeneration
),
mcpService: envelope(
"service-credential",
"sellable",
mcpService,
desired.serviceCredentialGeneration
),
modelCodex: envelope(
"model-codex-auth",
"codex",
modelCodex,
desired.modelCredentialGeneration
),
hermesApiServer: envelope(
"hermes-api-server",
"sellable",
hermesApiServer,
desired.apiServerKeyGeneration
),
},
});
if (!installed.ok) throw new Error(installed.code);
return {
expected,
mcpService,
slackApp,
slackBot,
modelCodex,
hermesApiServer,
};
}
function writeRuntimeHermesConfig({
desired,
toolInclude,
profileRoot,
secrets,
claim,
trustRootPem,
}) {
const secretRoot = join(SECRET_ROOT, desired.profileId);
// Hermes resolves `--profile <id>` beneath `$HERMES_HOME/profiles/<id>`.
// Keep the effective runtime bundle in that canonical profile directory so
// the named-profile switch cannot silently fall back to an empty config.
const runtimeConfigRoot = profileRoot;
mkdirSync(runtimeConfigRoot, { recursive: true, mode: 0o700 });
chmodSync(runtimeConfigRoot, 0o700);
const localHermesAuthStorePath = join(runtimeConfigRoot, "auth.json");
const hermesRootAuthStorePath = join(DATA_ROOT, "auth.json");
const hermesRootAuthMarkerPath = join(
DATA_ROOT,
".sellable-agent",
"model-auth-source.json"
);
if (existsSync(localHermesAuthStorePath)) {
const localAuthStore = lstatSync(localHermesAuthStorePath);
if (localAuthStore.isSymbolicLink() || !localAuthStore.isFile()) {
throw new Error("customer_model_auth_store_rejected");
}
unlinkSync(localHermesAuthStorePath);
}
const mcpConfig = JSON.parse(
readFileSync(join(profileRoot, "sellable", "mcp.json"), "utf8")
);
const serviceFingerprint = sha256(secrets.mcpService).slice(0, 16);
const runtimeMcpPath = join(runtimeConfigRoot, "sellable-mcp.json");
const turnDescriptorPath = join(secretRoot, "turn-context.json");
const turnConsumedRoot = join(secretRoot, "turn-consumed");
mkdirSync(turnConsumedRoot, { recursive: true, mode: 0o700 });
chmodSync(turnConsumedRoot, 0o700);
atomicJson(runtimeMcpPath, {
...mcpConfig,
apiUrl: ORIGIN,
tools: { include: [...toolInclude] },
activeWorkspaceId: desired.workspaceId,
workspaceId: desired.workspaceId,
serviceCredential: {
...mcpConfig.serviceCredential,
reference: `secret://agent/service/g${desired.serviceCredentialGeneration}`,
fingerprint: serviceFingerprint,
generation: desired.serviceCredentialGeneration,
},
});
const baseConfig = JSON.parse(
readFileSync(join(profileRoot, "config.yaml"), "utf8")
);
const sellable = baseConfig.mcp_servers?.sellable;
if (!sellable?.env) throw new Error("customer_mcp_config_rejected");
const adminServer = baseConfig.mcp_servers?.["sellable-admin"];
if (adminServer?.command && existsSync(adminServer.command)) {
// The admin MCP installer has shipped this wrapper without an execute
// bit; repair it here so a config refresh never revives a dead admin
// server entry.
const wrapperMode = lstatSync(adminServer.command).mode;
if ((wrapperMode & 0o100) === 0) {
chmodSync(adminServer.command, wrapperMode | 0o100);
}
}
// The named-profile materializer owns the one canonical SOUL file and seeds
// it only when absent. Runtime config refreshes must never rewrite live SOUL
// or create a root-level second copy.
atomicFile(
join(runtimeConfigRoot, "skills", "sellable", "SKILL.md"),
desired.skill,
0o444
);
const runtimeConfigPath = join(runtimeConfigRoot, "config.yaml");
const testModelOrigin = allowTestProviders()
? process.env.SELLABLE_AGENT_TEST_MODEL_ORIGIN
: null;
const modelCredential = parseModelAuthRuntimeCredential(secrets.modelCodex);
if (
modelCredential.activationId !== desired.modelActivationId ||
modelCredential.bundleGeneration !== desired.modelBundleGeneration ||
modelCredential.policyHash !== desired.modelPolicyHash
) {
throw new Error("customer_model_auth_identity_rejected");
}
const modelBundlePath = join(
secretRoot,
modelCredential.value.schemaVersion.endsWith("/v3")
? "model-auth-runtime-v3.json"
: "model-auth-runtime-v2.json"
);
atomicFile(modelBundlePath, modelCredential.authJson, 0o600);
const modelLifecycle =
claim.operation.action === "INSTANTIATE" ? "OFF" : "ON";
const modelAuthState = reconcileHermesModelAuth({
authStorePath: hermesRootAuthStorePath,
markerPath: hermesRootAuthMarkerPath,
credential: modelCredential,
lifecycle: modelLifecycle,
});
const modelConfig = modelAuthConfigOverlay(modelCredential);
atomicJson(runtimeConfigPath, {
...baseConfig,
...(testModelOrigin
? {
model: {
default: "whole-image-model",
provider: "custom:whole-image-local",
base_url: `${testModelOrigin}/v1`,
context_length: 65_536,
max_tokens: 256,
},
custom_providers: [
{
name: "whole-image-local",
base_url: `${testModelOrigin}/v1`,
key_env: "SELLABLE_AGENT_TEST_MODEL_KEY",
model: "whole-image-model",
api_mode: "chat_completions",
},
],
}
: {
...modelConfig,
}),
provider_routing: {
// OpenRouter provider selection defaults to highest throughput; an
// explicit profile value always wins.
sort: "throughput",
...(baseConfig.provider_routing ?? {}),
},
auxiliary: {
...(baseConfig.auxiliary ?? {}),
vision: {
// Non-vision main models (e.g. deepseek) need a working vision
// auxiliary; the Nous portal has no credentials on customer
// machines, so image input silently fails without this default.
provider: "openrouter",
model: "google/gemini-3.5-flash",
...(baseConfig.auxiliary?.vision ?? {}),
},
},
mcp_servers: {
...baseConfig.mcp_servers,
sellable: {
...sellable,
command: process.execPath,
args: [MCP_CONTEXT_PROXY],
env: {
...sellable.env,
SELLABLE_CONFIG_PATH: runtimeMcpPath,
SELLABLE_AGENT_MANAGED_AUTH_PATH: managedAuthPath(desired.profileId),
SELLABLE_AGENT_RUNTIME_GENERATION: String(RUNTIME_GENERATION),
SELLABLE_RUNTIME_ID: RUNTIME_ID,
SELLABLE_AGENT_SERVER_AUTHENTICATED_KIND: "CUSTOMER",
// Hermes forwards only explicit MCP env entries. Preserve the
// opt-in local fixture origin without relaxing production HTTPS.
...(allowTestProviders()
? { SELLABLE_AGENT_ALLOW_TEST_OIDC: "1" }
: {}),
SELLABLE_AGENT_PROFILE_ID: desired.profileId,
SELLABLE_AGENT_RUNTIME_SECRET_ROOT: SECRET_ROOT,
SELLABLE_AGENT_WORKSPACE_ID: desired.workspaceId,
SELLABLE_AGENT_ID: desired.agentId,
SELLABLE_AGENT_CREDENTIAL_GENERATION: String(
desired.serviceCredentialGeneration
),
SELLABLE_AGENT_POLICY_HASH: desired.policyHash,
SELLABLE_AGENT_SELECTED_CHANNEL_IDS: JSON.stringify(
desired.selectedChannelIds
),
SELLABLE_LOCK_WORKSPACE_ID: desired.workspaceId,
SELLABLE_REQUIRE_WORKSPACE_LOCK: "1",
SELLABLE_AGENT_TURN_DESCRIPTOR_PATH: turnDescriptorPath,
SELLABLE_AGENT_TURN_CONSUMED_ROOT: turnConsumedRoot,
SELLABLE_AGENT_MCP_RUNTIME_BINDING_PATH: join(
runtimeConfigRoot,
"sellable",
"runtime-binding.json"
),
SELLABLE_AGENT_MCP_TRUST_ROOT_PATH: TRUST_ROOT,
SELLABLE_AGENT_MCP_TRUST_ROOT_FINGERPRINT: sha256(trustRootPem),
SELLABLE_AGENT_MCP_DOWNSTREAM_COMMAND: MCP,
SELLABLE_AGENT_MCP_DOWNSTREAM_ARGS_JSON: "[]",
},
},
},
platforms: {
...baseConfig.platforms,
api_server: {
enabled: true,
extra: {
host: "127.0.0.1",
port: 8642,
cors_origins: [],
},
},
},
});
atomicJson(join(runtimeConfigRoot, "runtime-identity.json"), {
runtimeId: RUNTIME_ID,
profileId: desired.profileId,
revisionId: desired.revisionId,
fence: claim.operation.fence,
slackAppGeneration: desired.slackAppGeneration,
slackGeneration: desired.slackGeneration,
routeGeneration: desired.routeGeneration,
membershipGeneration: desired.membershipGeneration,
serviceCredentialGeneration: desired.serviceCredentialGeneration,
modelCredentialGeneration: desired.modelCredentialGeneration,
modelActivationId: desired.modelActivationId,
modelBundleGeneration: desired.modelBundleGeneration,
modelPolicyHash: desired.modelPolicyHash,
apiServerKeyGeneration: desired.apiServerKeyGeneration,
});
return {
runtimeHome: runtimeConfigRoot,
runtimeConfigPath,
runtimeMcpPath,
modelBundlePath,
hermesRootAuthStorePath,
hermesRootAuthMarkerPath,
modelAuthSourceFingerprint: modelAuthState.sourceFingerprint,
modelAuthState,
modelCredential,
turnDescriptorPath,
turnConsumedRoot,
};
}
const INTERNAL_ADMIN_SETTINGS_FILES = Object.freeze([
".env",
"settings.json",
"config-state.json",
]);
export function installInternalAdminRuntimeSettings({
desired,
profileRoot,
secretsRoot = SECRET_ROOT,
}) {
if (desired.workspaceId !== SELLABLE_INTERNAL_AGENT_WORKSPACE_ID) {
return { ok: true, enabled: false };
}
const sourceRoot = join(profileRoot, ".sellable-admin");
const targetRoot = join(secretsRoot, desired.profileId, "admin-settings");
try {
const source = lstatSync(sourceRoot);
if (
source.isSymbolicLink() ||
!source.isDirectory() ||
(source.mode & 0o077) !== 0
) {
throw new Error("customer_internal_admin_settings_root_rejected");
}
mkdirSync(targetRoot, { recursive: true, mode: 0o700 });
chmodSync(targetRoot, 0o700);
for (const name of INTERNAL_ADMIN_SETTINGS_FILES) {
const pathValue = join(sourceRoot, name);
const entry = lstatSync(pathValue);
if (
entry.isSymbolicLink() ||
!entry.isFile() ||
(entry.mode & 0o077) !== 0 ||
entry.size < 1 ||
entry.size > 1024 * 1024
) {
throw new Error("customer_internal_admin_settings_file_rejected");
}
atomicFile(join(targetRoot, name), readFileSync(pathValue), 0o600);
}
return { ok: true, enabled: true, settingsRoot: targetRoot };
} catch (error) {
throw new Error(
error instanceof Error &&
error.message.startsWith("customer_internal_admin_settings_")
? error.message
: "customer_internal_admin_settings_missing"
);
}
}
function allowTestProviders() {
return (
process.env.SELLABLE_AGENT_ALLOW_TEST_PROVIDERS === "1" ||
process.env.SELLABLE_AGENT_ALLOW_TEST_OIDC === "1"
);
}
function slackApiOrigin() {
const configured = process.env.SELLABLE_AGENT_SLACK_API_ORIGIN;
if (!configured) return "https://slack.com";
const parsed = new URL(configured);
const testHttp =
process.env.SELLABLE_AGENT_ALLOW_TEST_OIDC === "1" &&
parsed.protocol === "http:" &&
["127.0.0.1", "host.docker.internal"].includes(parsed.hostname);
if (
(parsed.protocol !== "https:" && !testHttp) ||
parsed.username ||
parsed.password ||
parsed.pathname !== "/" ||
parsed.search ||
parsed.hash
) {
throw new Error("customer_slack_origin_rejected");
}
return parsed.origin;
}
export function applicationRuntimeEnv({ desired, runtimeConfig, secrets }) {
const env = {
...process.env,
HOME: DATA_ROOT,
HERMES_HOME: DATA_ROOT,
SELLABLE_HERMES_PROFILE_ID: desired.profileId,
HERMES_CONFIG_PATH: runtimeConfig.runtimeConfigPath,
CODEX_HOME: dirname(runtimeConfig.modelBundlePath),
SELLABLE_AGENT_RUNTIME: "1",
SELLABLE_AGENT_MANAGED_AUTH_PATH: managedAuthPath(desired.profileId),
SELLABLE_AGENT_RUNTIME_GENERATION: String(RUNTIME_GENERATION),
SELLABLE_RUNTIME_ID: RUNTIME_ID,
SELLABLE_AGENT_SERVER_AUTHENTICATED_KIND: "CUSTOMER",
SELLABLE_AGENT_WORKSPACE_ID: desired.workspaceId,
SELLABLE_AGENT_ID: desired.agentId,
SELLABLE_AGENT_CREDENTIAL_GENERATION: String(
desired.serviceCredentialGeneration
),
SELLABLE_AGENT_POLICY_HASH: desired.policyHash,
SELLABLE_AGENT_SELECTED_CHANNEL_IDS: JSON.stringify(
desired.selectedChannelIds
),
SELLABLE_LOCK_WORKSPACE_ID: desired.workspaceId,
SELLABLE_REQUIRE_WORKSPACE_LOCK: "1",
SELLABLE_AGENT_TURN_DESCRIPTOR_PATH: runtimeConfig.turnDescriptorPath,
SLACK_HOME_CHANNEL: desired.defaultChannelId,
SLACK_ALLOWED_CHANNELS: desired.selectedChannelIds.join(","),
SLACK_ALLOW_ALL_USERS: "true",
SLACK_REQUIRE_MENTION: "true",
SLACK_STRICT_MENTION: "false",
SLACK_ALLOW_BOTS: "none",
...(allowTestProviders()
? {
SELLABLE_AGENT_TEST_MODEL_KEY:
runtimeConfig.modelCredential.testAccessToken,
}
: {}),
NO_COLOR: "1",
};
delete env.OPENAI_BASE_URL;
delete env.OPENAI_API_KEY;
delete env.OPENROUTER_API_KEY;
delete env.SLACK_APP_TOKEN;
delete env.SLACK_BOT_TOKEN;
delete env.API_SERVER_KEY;
delete env.API_SERVER_HOST;
delete env.API_SERVER_PORT;
delete env.API_SERVER_CORS_ORIGINS;
return env;
}
export function redactRuntimeDiagnostic(value) {
return String(value)
.replace(
/(?:xoxb|xapp)-[A-Za-z0-9-]+|sat_[A-Za-z0-9_-]+|sk-[A-Za-z0-9_-]+|or-[A-Za-z0-9_-]{16,}|codex-(?:access|refresh|id)-token-[A-Za-z0-9_-]+|[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{8,}/g,
"[redacted]"
)
.slice(-8_000);
}
async function stopNativeGateway(gateway) {
if (!gateway?.child || gateway.child.exitCode !== null) return;
gateway.child.kill("SIGTERM");
const stopped = await Promise.race([
gateway.closed.then(() => true),
sleep(10_000).then(() => false),
]);
if (!stopped && gateway.child.exitCode === null) {
gateway.child.kill("SIGKILL");
await gateway.closed;
}
}
async function startNativeGateway({ desired, runtimeConfig, secrets }) {
const secretRoot = join(SECRET_ROOT, desired.profileId);
const launch = buildNativeHermesLaunch({
profileRoot: runtimeConfig.runtimeHome,
secretsRoot: secretRoot,
profileId: desired.profileId,
env: applicationRuntimeEnv({ desired, runtimeConfig, secrets }),
});
const appDescriptor = openSync(
launch.inheritedFileDescriptors.find((item) => item.fd === 8).path,
"r"
);
const botDescriptor = openSync(
launch.inheritedFileDescriptors.find((item) => item.fd === 9).path,
"r"
);
const apiServerKey = readFileSync(launch.apiServerKeyFile, "utf8");
if (
sha256(apiServerKey).slice(0, 16) !== desired.apiServerKeyFingerprint ||
apiServerKey === secrets.mcpService ||
apiServerKey === secrets.slackApp ||
apiServerKey === secrets.slackBot ||
apiServerKey === secrets.modelCodex
) {
closeSync(appDescriptor);
closeSync(botDescriptor);
throw new Error("customer_api_server_key_binding_rejected");
}
const stdout = [];
const stderr = [];
let child;
try {
child = spawn(HERMES, launch.args, {
cwd: launch.cwd,
env: { ...launch.env, API_SERVER_KEY: apiServerKey },
stdio: [
"ignore",
"pipe",
"pipe",
"ignore",
"ignore",
"ignore",
"ignore",
"ignore",
appDescriptor,
botDescriptor,
],
});
} finally {
closeSync(appDescriptor);
closeSync(botDescriptor);
}
let traceTimer = null;
let traceDigest = null;
const traceEnabled =
allowTestProviders() && process.env.SELLABLE_AGENT_TRACE_GATEWAY === "1";
const emitTrace = () => {
traceTimer = null;
const output = redactRuntimeDiagnostic(
`${Buffer.concat(stdout).toString("utf8")}\n${Buffer.concat(stderr).toString("utf8")}`
);
const digest = output ? sha256(output) : null;
if (!output || digest === traceDigest) return;
traceDigest = digest;
process.stderr.write(
`${JSON.stringify({
ok: true,
code: "customer_native_gateway_trace",
output,
})}\n`
);
};
const scheduleTrace = () => {
if (!traceEnabled) return;
if (traceTimer) clearTimeout(traceTimer);
traceTimer = setTimeout(emitTrace, 250);
traceTimer.unref();
};
const append = (target, chunk) => {
target.push(Buffer.from(chunk));
while (
target.length > 1 &&
target.reduce((total, value) => total + value.length, 0) > 128_000
) {
target.shift();
}
scheduleTrace();
};
child.stdout.on("data", (chunk) => append(stdout, chunk));
child.stderr.on("data", (chunk) => append(stderr, chunk));
const closed = new Promise((resolve) => {
child.once("close", (status, signal) => {
if (traceTimer) clearTimeout(traceTimer);
if (traceEnabled) emitTrace();
resolve({ status, signal });
});
});
const failed = new Promise((resolve) => {
child.once("error", (error) =>
resolve({ status: null, signal: null, error: error.message })
);
});
const early = await Promise.race([
closed,
failed,
sleep(5_000).then(() => null),
]);
if (early) {
throw new Error(
`customer_native_gateway_start_rejected:${redactRuntimeDiagnostic(
`${Buffer.concat(stdout).toString("utf8")}\n${Buffer.concat(stderr).toString("utf8")}`
)}`
);
}
return {
child,
closed,
readback() {
return {
pid: child.pid,
running: child.exitCode === null && !child.killed,
output: redactRuntimeDiagnostic(
`${Buffer.concat(stdout).toString("utf8")}\n${Buffer.concat(stderr).toString("utf8")}`
),
};
},
};
}
export const API_SERVER_READINESS_DEADLINE_MS = 240_000;
const API_SERVER_READINESS_RETRY_MS = 1_000;
export function nativeGatewayRuntimeStatusAccepted(
status,
{ expectedPid, expectedHome }
) {
return Boolean(
status &&
Number.isSafeInteger(expectedPid) &&
expectedPid > 1 &&
status.pid === expectedPid &&
status.kind === "hermes-gateway" &&
Array.isArray(status.argv) &&
status.argv.includes("gateway") &&
status.argv.includes("run") &&
Number.isSafeInteger(status.start_time) &&
status.start_time > 0 &&
status.hermes_home === expectedHome &&
status.gateway_state === "running" &&
status.platforms?.slack?.state === "connected" &&
status.platforms?.api_server?.state === "connected"
);
}
function readNativeGatewayRuntimeStatus(runtimeHome) {
try {
const statusPath = join(runtimeHome, "gateway_state.json");
const entry = lstatSync(statusPath);
if (
entry.isSymbolicLink() ||
!entry.isFile() ||
(entry.mode & 0o022) !== 0 ||
entry.size < 2 ||
entry.size > 64 * 1024
) {
return null;
}
return JSON.parse(readFileSync(statusPath, "utf8"));
} catch {
return null;
}
}
async function proveApiServerReadiness({
desired,
apiServerKey,
gateway,
runtimeConfig,
}) {
const origin = "http://127.0.0.1:8642";
const request = (authorization) =>
fetch(`${origin}/v1/capabilities`, {
headers: authorization ? { authorization } : {},
signal: AbortSignal.timeout(5_000),
});
const deadlineAt = Date.now() + API_SERVER_READINESS_DEADLINE_MS;
while (Date.now() < deadlineAt) {
const gatewayReadback = gateway?.readback();
if (!gatewayReadback?.running) {
throw new Error(
`customer_native_gateway_start_rejected:${redactRuntimeDiagnostic(
gatewayReadback?.output ?? "candidate exited before readiness"
)}`
);
}
const candidateStatus = readNativeGatewayRuntimeStatus(
runtimeConfig.runtimeHome
);
if (
!nativeGatewayRuntimeStatusAccepted(candidateStatus, {
expectedPid: gateway.child.pid,
expectedHome: runtimeConfig.runtimeHome,
})
) {
await sleep(API_SERVER_READINESS_RETRY_MS);
continue;
}
try {
const [health, missing, wrong, correct] = await Promise.all([
fetch(`${origin}/health`, { signal: AbortSignal.timeout(5_000) }),
request(null),
request("Bearer phase166-readiness-intentionally-wrong"),
request(`Bearer ${apiServerKey}`),
]);
const capabilities = correct.ok ? await correct.json() : null;
if (
health.status === 200 &&
missing.status === 401 &&
wrong.status === 401 &&
correct.status === 200 &&
nativeGatewayRuntimeStatusAccepted(
readNativeGatewayRuntimeStatus(runtimeConfig.runtimeHome),
{
expectedPid: gateway.child.pid,
expectedHome: runtimeConfig.runtimeHome,
}
) &&
capabilities?.auth?.required === true &&
capabilities?.features?.run_submission === true &&
capabilities?.features?.run_events_sse === true
) {
return Object.freeze({
host: "127.0.0.1",
port: 8642,
cors: false,
keyGeneration: desired.apiServerKeyGeneration,
keyFingerprint: desired.apiServerKeyFingerprint,
authStatusDigest: sha256(
JSON.stringify({
health: health.status,
missing: missing.status,
wrong: wrong.status,
correct: correct.status,
})
),
capabilitiesDigest: sha256(JSON.stringify(capabilities)),
fixtureDigest: HERMES_RUNS_FIXTURE_IDENTITY.fixtureDigest,
relayCapability: "fixture_bound_runs_relay_v1",
runtimeGeneration: desired.runtimeGeneration,
gatewayPid: gateway.child.pid,
});
}
} catch {
// Hermes can hold its singleton lock before the local HTTP listener is
// accepting connections. Keep the lifecycle claim fenced while the
// bounded readiness window converges instead of replacing it again.
}
await sleep(API_SERVER_READINESS_RETRY_MS);
}
throw new Error("customer_api_server_readiness_rejected");
}
export async function startProvenNativeGateway({
desired,
runtimeConfig,
secrets,
startGateway = startNativeGateway,
proveReadiness = proveApiServerReadiness,
stopGateway = stopNativeGateway,
}) {
let gateway = null;
try {
gateway = await startGateway({ desired, runtimeConfig, secrets });
const gatewayReadback = gateway.readback();
if (!gatewayReadback.running) {
throw new Error("customer_native_gateway_readback_rejected");
}
const apiServerReadiness = await proveReadiness({
desired,
apiServerKey: secrets.hermesApiServer,
gateway,
runtimeConfig,
});
return { gateway, gatewayReadback, apiServerReadiness };
} catch (error) {
if (gateway) {
try {
await stopGateway(gateway);
} catch (cleanupError) {
process.stderr.write(
`${JSON.stringify({
ok: false,
code:
cleanupError instanceof Error
? cleanupError.message
: "customer_native_gateway_cleanup_failed",
})}\n`
);
}
}
throw error;
}
}
async function proveProvisionedApplication({
desired,
runtimeConfig,
secrets,
profileDigest,
}) {
const runtimeEnv = applicationRuntimeEnv({ desired, runtimeConfig, secrets });
const materializedModelAuth = runtimeConfig.modelAuthState.materialized;
const [hermesVersion, mcpVersion, hermesModelAuth] = await Promise.all([
run(HERMES, ["--version"], { env: runtimeEnv, timeoutMs: 180_000 }),
run(MCP, ["--version"], { env: runtimeEnv, timeoutMs: 180_000 }),
materializedModelAuth
? run(
HERMES_PYTHON,
[
"-c",
[
"import json",
"from agent.credential_pool import load_pool",
'pool = load_pool("openai-codex")',
"entry = pool.select() if pool else None",
'print(json.dumps({"provider": "openai-codex", "source": "credential_pool", "credential_present": bool(entry and entry.runtime_api_key), "credential_id": getattr(entry, "id", None), "runtime_refresh_owned": bool(getattr(entry, "refresh_token", None))}))',
].join(";"),
],
{
env: { ...runtimeEnv, HERMES_HOME: runtimeConfig.runtimeHome },
timeoutMs: 180_000,
}
)
: Promise.resolve({
status: 0,
stdout: JSON.stringify({ materialization_pending_start: true }),
stderr: "",
}),
]);
let hermesModelAuthReadback = null;
try {
hermesModelAuthReadback = JSON.parse(hermesModelAuth.stdout);
} catch {
hermesModelAuthReadback = null;
}
const modelAuthReadback = materializedModelAuth
? readModelAuthMaterialization({
authStorePath: runtimeConfig.hermesRootAuthStorePath,
markerPath: runtimeConfig.hermesRootAuthMarkerPath,
})
: {
schemaVersion:
runtimeConfig.modelCredential.value.schemaVersion.endsWith("/v3")
? "sellable-agent-model-auth-readback/v3"
: "sellable-agent-model-auth-readback/v2",
activationId: desired.modelActivationId,
bundleGeneration: desired.modelBundleGeneration,
policyHash: desired.modelPolicyHash,
provider: "openai-codex",
model: runtimeConfig.modelCredential.model,
fallbackDepth:
runtimeConfig.modelCredential.value.openrouter.models.length,
desiredStatus: "converged",
materializedStatus: "pending_start",
slots: runtimeConfig.modelCredential.identity.slots.map((slot) => ({
slotId: slot.slotId,
grantFingerprint: slot.grantFingerprint,
expiresAt: new Date(slot.expiresAt).toISOString(),
state: "ok",
lastErrorReason: null,
lastErrorResetAt: null,
})),
openrouter: {
keyFingerprint:
runtimeConfig.modelCredential.identity.openrouterKeyFingerprint,
state: "ok",
},
};
const modelAuthVerdict = materializedModelAuth
? evaluateModelAuthReadback(modelAuthReadback, {
activationId: desired.modelActivationId,
bundleGeneration: desired.modelBundleGeneration,
policyHash: desired.modelPolicyHash,
})
: { accepted: true, rejection: null };
if (
materializedModelAuth &&
(hermesModelAuth.status !== 0 ||
hermesModelAuthReadback?.provider !== "openai-codex" ||
hermesModelAuthReadback?.source !== "credential_pool" ||
hermesModelAuthReadback?.credential_present !== true ||
hermesModelAuthReadback?.runtime_refresh_owned !== false ||
![
"dittto-primary",
"kaigo-secondary",
"sellable-primary",
"dittto-secondary",
"kaigo-tertiary",
].includes(hermesModelAuthReadback?.credential_id))
) {
throw new Error("access_only_snapshot_infeasible");
}
if (!modelAuthVerdict.accepted) {
throw new Error(
`customer_model_auth_readback_rejected:${modelAuthVerdict.rejection ?? "unknown"}`
);
}
if (
hermesVersion.status !== 0 ||
mcpVersion.status !== 0 ||
/Lazy-installing|pip install|npm install/i.test(
`${hermesVersion.stdout}\n${hermesVersion.stderr}\n${mcpVersion.stdout}\n${mcpVersion.stderr}\n${hermesModelAuth.stdout}\n${hermesModelAuth.stderr}`
)
) {
throw new Error("customer_runtime_binary_rejected");
}
return {
profileDigest,
hermesOutput: hermesVersion.stdout,
mcpOutput: mcpVersion.stdout,
modelAuthOutput: `${hermesModelAuth.stdout}\n${JSON.stringify(
modelAuthReadback
)}`,
modelAuthReadback,
modelAuthSelection: materializedModelAuth
? modelAuthSelectionObservation(modelAuthReadback)
: null,
};
}
async function proveApplication({
action,
desired,
runtimeConfig,
secrets,
profileDigest,
runtimeAuthorityScope,
}) {
const provisioned = await proveProvisionedApplication({
desired,
runtimeConfig,
secrets,
profileDigest,
});
const runtimeEnv = applicationRuntimeEnv({ desired, runtimeConfig, secrets });
const doctor = customerRuntimeDoctorRequired(action, {
runtimeAuthorityScope,
directReadinessProven: true,
})
? await run(HERMES, ["doctor"], {
env: runtimeEnv,
timeoutMs: 300_000,
})
: {
status: 0,
stdout: "Hermes START readiness uses direct runtime checks.",
stderr: "",
timedOut: false,
elapsedMs: 0,
};
const doctorFailure = customerRuntimeDoctorFailureCode(doctor);
if (doctorFailure) throw new Error(doctorFailure);
const slackResponse = await fetch(`${slackApiOrigin()}/api/auth.test`, {
method: "POST",
headers: {
authorization: `Bearer ${secrets.slackBot}`,
"content-type": "application/x-www-form-urlencoded",
},
body: "",
signal: AbortSignal.timeout(15_000),
});
const slack = await slackResponse.json();
if (
!slackResponse.ok ||
slack?.ok !== true ||
slack.team_id !== desired.teamId ||
slack.user_id !== desired.botUserId
) {
throw new Error("customer_slack_identity_rejected");
}
const socketResponse = await fetch(
`${slackApiOrigin()}/api/apps.connections.open`,
{
method: "POST",
headers: {
authorization: `Bearer ${secrets.slackApp}`,
"content-type": "application/x-www-form-urlencoded",
},
body: "",
signal: AbortSignal.timeout(15_000),
}
);
const socket = await socketResponse.json();
if (
!socketResponse.ok ||
socket?.ok !== true ||
typeof socket.url !== "string" ||
!/^wss?:\/\//.test(socket.url)
) {
throw new Error("customer_slack_socket_identity_rejected");
}
const { gateway, gatewayReadback, apiServerReadiness } =
await startProvenNativeGateway({
desired,
runtimeConfig,
secrets,
});
return {
...provisioned,
hermesOutput: `${provisioned.hermesOutput}\n${doctor.stdout}`,
gateway,
gatewayReadback,
apiServerReadiness,
};
}
function completionReceipt({
claim,
exchange,
desired,
proof,
memorySnapshot,
observedSoulRawSha256,
activeProjection,
nativeDefaultProfileProof,
}) {
const identity = operationIdentity(claim, exchange);
const observedLifecycle =
claim.operation.action === "INSTANTIATE" ? "OFF" : "ON";
const runtimeHashes = {
package: sha256(`${INSTALLER_PACKAGE}\n${MCP_PACKAGE}\n${HERMES_VERSION}`),
config: proof.profileDigest,
content: sha256(`${proof.profileDigest}\n${observedSoulRawSha256}`),
process: sha256(
`${proof.hermesOutput}\n${proof.mcpOutput}\n${proof.modelAuthOutput}\n${
proof.apiServerReadiness?.capabilitiesDigest ?? "api-server-held"
}`
),
};
return {
...identity,
status: "SUCCEEDED",
secretDisposition: "INSTALLED",
observedLifecycle,
observedRevisionId: desired.revisionId,
...(activeProjection
? {
bundleVersion: activeProjection.bundleVersion,
bundleDigest: activeProjection.bundleDigest,
}
: {}),
signerGeneration: exchange.keyGeneration,
signedDigest: flyRuntimeDigest({
...identity,
observedLifecycle,
revisionId: desired.revisionId,
runtimeHashes,
}),
runtimeHashes,
conditions: [
{ code: "PROFILE_MATERIALIZED", status: "TRUE" },
...(claim.operation.action === "INSTANTIATE"
? []
: [
{ code: "HERMES_DOCTOR", status: "TRUE" },
{ code: "SLACK_AUTH_TEST", status: "TRUE" },
{ code: "SLACK_SOCKET_OPEN", status: "TRUE" },
{ code: "NATIVE_SLACK_GATEWAY", status: "TRUE" },
{ code: "HERMES_API_SERVER_READINESS", status: "TRUE" },
]),
{ code: "MCP_PREBAKED", status: "TRUE" },
{
code:
claim.operation.action === "INSTANTIATE"
? "MODEL_AUTH_DESIRED_PENDING_START"
: "CODEX_OAUTH_READBACK",
status: "TRUE",
},
{ code: "NO_BOOT_INSTALL", status: "TRUE" },
{ code: "DEDICATED_SLACK_APP", status: "TRUE" },
{
code: "DEFAULT_PROFILE_NATIVE_PROOF",
status:
nativeDefaultProfileProof.status === "PROVEN" ? "TRUE" : "UNKNOWN",
},
],
errorClass: "NONE",
auditIds: [
`fly-${claim.operation.id}`,
`soul-raw-${observedSoulRawSha256}`,
`hermes-memory-${
memorySnapshot.ok ? memorySnapshot.status : memorySnapshot.code
}`.toLowerCase(),
],
};
}
async function flushHermesSnapshot({
client,
profileId,
profileRoot,
operationIdentity,
forceDirty = false,
}) {
try {
if (forceDirty) markHermesSnapshotDirty(profileRoot);
return await reconcileHermesSnapshot({
profileId,
profileRoot,
hermesRoot: DATA_ROOT,
hermesVersion: HERMES_VERSION,
debounceMs: 0,
operationIdentity,
upload: ({ archiveDigest, archiveBytes, sourceProfileId, bytes }) =>
client.putMemorySnapshot(
{
archiveDigest,
archiveBytes,
hermesVersion: HERMES_VERSION,
sourceProfileId,
...(operationIdentity ?? {}),
},
bytes
),
});
} catch {
return {
ok: false,
status: "PENDING_RETRY",
code: "hermes_snapshot_flush_failed",
};
}
}
function runtimeModelHealthReporter({ identity, exchange, runtimeConfig }) {
return createModelHealthReporter({
origin: ORIGIN,
identity: {
kind: "CUSTOMER",
runtimeGeneration: RUNTIME_GENERATION,
workloadKeyHash: exchange.publicKeyHash,
machineId: exchange.workerId,
adminRuntimeGenerationId: null,
adminRuntimeGenerationFence: null,
},
observe: () =>
readModelAuthMaterialization({
authStorePath: runtimeConfig.hermesRootAuthStorePath,
markerPath: runtimeConfig.hermesRootAuthMarkerPath,
}),
authorize: (body) =>
signedWorkerAuthorization({
identity,
exchange,
verb: "model-health",
body,
now: new Date(),
unbound: true,
}),
onError: (error) => {
process.stderr.write(
`${JSON.stringify({
ok: false,
code:
error instanceof Error
? error.message
: "model_health_report_failed",
})}\n`
);
},
allowHttpLoopback: allowTestProviders(),
sequencePath: join(
DATA_ROOT,
".sellable-agent",
"model-health-sequence.json"
),
});
}
async function processOperation({
claim,
exchange,
client,
identity,
reattestAuthority,
}) {
const phaseTelemetry = createCustomerRuntimePhaseTelemetry({
bootSessionId: BOOT_SESSION_ID,
operationId: claim.operation.id,
});
const runtimeAuthorityScope = claim.runtimeAuthority?.scope ?? null;
const trustRootPem = readFileSync(TRUST_ROOT, "utf8");
const { desired, toolInclude } = compileDesired(claim, trustRootPem);
mkdirSync(PROFILES_ROOT, { recursive: true, mode: 0o700 });
chmodSync(PROFILES_ROOT, 0o700);
const secrets = await phaseTelemetry.run("runtime_secrets", async () => {
const consumed = await consumeSecrets({
claim,
exchange,
client,
desired,
});
await publishManagedAuth({
schemaVersion: "sellable-managed-auth/v1",
profileId: desired.profileId,
runtimeId: RUNTIME_ID,
runtimeGeneration: RUNTIME_GENERATION,
agentKind: "CUSTOMER",
apiUrl: ORIGIN,
token: consumed.mcpService,
agent: {
workspaceId: desired.workspaceId,
agentId: desired.agentId,
generation: desired.serviceCredentialGeneration,
policyHash: desired.policyHash,
selectedChannelIds: desired.selectedChannelIds,
},
});
return consumed;
});
const profileRoot = join(PROFILES_ROOT, desired.profileId);
const memoryRestore = await phaseTelemetry.run("snapshot_restore", () =>
restoreLatestHermesSnapshot({
profileId: desired.profileId,
profileRoot,
hermesRoot: DATA_ROOT,
hermesVersion: HERMES_VERSION,
fetchLatest: () => client.getMemorySnapshot(),
})
);
if (!memoryRestore.ok) throw new Error(memoryRestore.code);
const legacyUserState = await phaseTelemetry.run("legacy_migration", () =>
migrateLegacyHermesUserState({
profileId: desired.profileId,
profileRoot,
hermesRoot: DATA_ROOT,
})
);
if (!legacyUserState.ok) throw new Error(legacyUserState.code);
const { materialized, pristineProfile } = await phaseTelemetry.run(
"profile_materialization",
() => {
const materialized = materializeNativeSlackCustomerProfile({
desired,
profilesRoot: PROFILES_ROOT,
});
if (!materialized.ok) {
const detail =
materialized.code === "native_slack_profile_insufficient_disk"
? `:available_bytes=${materialized.availableBytes}:required_bytes=${materialized.requiredBytes}`
: materialized.causeCode
? `:cause=${materialized.causeCode}`
: "";
throw new Error(`${materialized.code}${detail}`);
}
const pristineProfile = validateNativeSlackCustomerProfile({
desired,
profileRoot,
});
if (!pristineProfile.ok) throw new Error(pristineProfile.code);
return { materialized, pristineProfile };
}
);
const activeProjection = buildDefaultProfileProjection({
kind: "CUSTOMER",
claim,
});
// Public skill files are image-owned. Update them before the default bundle
// verifies their bytes; older images may have no default-bundle receipt yet.
await reattestAuthority();
const publicSkills = reconcilePublicHermesSkills({
profileId: desired.profileId,
profileRoot,
dataRoot: DATA_ROOT,
});
const reconciledDefaultProfile = await phaseTelemetry.run(
"default_profile_bundle",
() =>
reconcileDefaultProfileBundle({
activeProjection,
materializerReceipt: pristineProfile.manifest,
profileRoot,
bundleRoot: DEFAULT_PROFILE_BUNDLE_ROOT,
dataRoot: DATA_ROOT,
reattestAuthority: async () => {
await reattestAuthority();
return activeProjection;
},
cronAdapter: customerDefaultProfileCronAdapter(
desired.profileId,
profileRoot
),
})
);
if (!reconciledDefaultProfile.ok)
throw new Error(reconciledDefaultProfile.code);
const defaultProfile = reconciledDefaultProfile;
const { runtimeConfig, soulProof } = await phaseTelemetry.run(
"profile_configuration",
async () => {
installInternalAdminRuntimeSettings({ desired, profileRoot });
const runtimeConfig = writeRuntimeHermesConfig({
desired,
toolInclude,
profileRoot,
secrets,
claim,
trustRootPem,
});
const soulProof = observeProfileSoul(desired.profileId, {
dataRoot: DATA_ROOT,
});
if (soulProof.ok !== true) {
throw new Error("customer_live_soul_loader_proof_rejected");
}
return { runtimeConfig, soulProof };
}
);
const observedSoulRawSha256 = soulProof.rawFileSha256;
const memoryCron = await phaseTelemetry.run("memory_cron", () =>
ensureHermesSnapshotCron({
profileId: desired.profileId,
profileRoot,
hermesRoot: DATA_ROOT,
hermesCli: HERMES,
})
);
if (!memoryCron.ok) throw new Error(memoryCron.code);
const materializerChanged =
materialized.status !== "REUSED" || memoryCron.status !== "REUSED";
if (materializerChanged || defaultProfile.changed || publicSkills.changed)
markHermesSnapshotDirty(profileRoot);
const memorySnapshot = await phaseTelemetry.run("pre_serving_snapshot", () =>
flushHermesSnapshotBeforeServing({
client,
profileId: desired.profileId,
profileRoot,
runtimeAuthorityScope,
})
);
if (!memorySnapshot.ok) markHermesSnapshotDirty(profileRoot);
const prove =
claim.operation.action === "INSTANTIATE"
? proveProvisionedApplication
: proveApplication;
// Persist the one-time welcome intent before the legacy onboarding job is
// reconciled. Existing profiles stay ineligible; a new profile is armed only
// after the runtime has completed successfully below.
const welcomeInput = {
profileId: desired.profileId,
profileRoot,
hermesRoot: DATA_ROOT,
workspaceId: desired.workspaceId,
channelId: desired.defaultChannelId,
botUserId: desired.botUserId,
newAgent: claim.operation.action === "INSTANTIATE",
};
await ensureHermesFirstWelcome(welcomeInput);
const proof = await phaseTelemetry.run("slack_serving_proof", () =>
prove({
action: claim.operation.action,
desired,
runtimeConfig,
secrets,
profileDigest: pristineProfile.manifest.profileDigest,
runtimeAuthorityScope,
})
);
await phaseTelemetry.run("customer_program_crons", async () => {
const customerProgramCrons = await ensureHermesCustomerProgramCrons({
profileId: desired.profileId,
profileRoot,
hermesRoot: DATA_ROOT,
workspaceId: desired.workspaceId,
channelId: desired.defaultChannelId,
timezone: desired.customerProgramTimezone,
apiUrl: ORIGIN,
authorityPath: managedAuthPath(desired.profileId),
});
if (!customerProgramCrons.ok) throw new Error(customerProgramCrons.code);
const onboardingCron = await ensureHermesFirstCampaignOnboardingCron({
profileId: desired.profileId,
profileRoot,
hermesRoot: DATA_ROOT,
workspaceId: desired.workspaceId,
channelId: desired.defaultChannelId,
});
if (!onboardingCron.ok) throw new Error(onboardingCron.code);
// NON-FATAL: a missed refill tick is harmless (refill no-ops when queues
// are full and the next weekday run continues), so it never blocks serving.
await ensureHermesRefillSendsCron({
profileId: desired.profileId,
profileRoot,
hermesRoot: DATA_ROOT,
workspaceId: desired.workspaceId,
});
});
markHermesSnapshotDirty(profileRoot);
const onboardingMemorySnapshot = await phaseTelemetry.run(
"post_cron_snapshot",
() =>
flushHermesSnapshotBeforeServing({
client,
profileId: desired.profileId,
profileRoot,
runtimeAuthorityScope,
})
);
const nativeDefaultProfileProof = proof.gatewayReadback?.running
? await phaseTelemetry.run("native_skill_proof", () =>
proveCustomerRuntimeDefaultSkills({
claim,
exchange,
client,
gateway: proof.gateway,
reattestAuthority,
actions: defaultProfile.actions,
profileId: desired.profileId,
})
)
: { status: "EMPTY", digest: sha256("[]") };
const receipt = completionReceipt({
claim,
exchange,
desired,
proof,
memorySnapshot: onboardingMemorySnapshot,
observedSoulRawSha256,
activeProjection,
nativeDefaultProfileProof,
});
const candidateRuntime = {
claim,
desired,
expected: secrets.expected,
secrets,
profileRoot,
secretsRoot: join(SECRET_ROOT, desired.profileId),
runtimeConfig,
trustRootPem,
action: claim.operation.action,
gateway: proof.gateway ?? null,
proof,
modelHealthReporter: null,
};
if (claim.runtimeAuthority && receipt.observedLifecycle === "ON") {
const servingObservation = customerServingObservation(
candidateRuntime,
exchange
);
if (!servingObservation)
throw new Error("customer_serving_observation_unavailable");
const serving = await phaseTelemetry.run("serving_observation", () =>
client.claimTick({
leaseSeconds: 60,
servingObservation,
})
);
if (serving?.servingObservationStatus !== "accepted") {
throw new Error(
`customer_serving_observation_rejected:${serving?.servingObservationCode ?? serving?.servingObservationStatus ?? "unknown"}`
);
}
}
const completed = await phaseTelemetry.run("operation_completion", () =>
client.complete(receipt)
);
if (completed?.accepted !== true)
throw new Error("customer_completion_rejected");
// Native delivery owns the welcome after readiness. A delivery failure must
// not turn a healthy, accepted runtime into a failed onboarding operation.
if (proof.gatewayReadback?.running && receipt.observedLifecycle === "ON") {
const welcome = await ensureHermesFirstWelcome({
...welcomeInput,
ready: true,
});
markHermesSnapshotDirty(profileRoot);
process.stdout.write(
`${JSON.stringify({ event: "customer_first_welcome", ...welcome })}\n`
);
}
let modelHealthReporter = null;
if (claim.operation.action !== "INSTANTIATE") {
modelHealthReporter = runtimeModelHealthReporter({
identity,
exchange,
runtimeConfig,
});
try {
await modelHealthReporter.start("materialized");
} catch (error) {
process.stderr.write(
`${JSON.stringify({
ok: false,
code:
error instanceof Error
? error.message
: "model_health_report_failed",
})}\n`
);
}
}
atomicJson(join(DATA_ROOT, "sellable-agent-runtime-proof.json"), {
ok: true,
runtimeId: RUNTIME_ID,
customerKey: CUSTOMER_KEY,
operationId: claim.operation.id,
action: claim.operation.action,
profileId: desired.profileId,
revisionId: desired.revisionId,
observedLifecycle: receipt.observedLifecycle,
profileDigest: proof.profileDigest,
modelAuth: proof.modelAuthReadback,
observedSoulRawSha256,
runtimeUid: process.getuid?.() ?? null,
runtimeGid: process.getgid?.() ?? null,
profileMode: (lstatSync(profileRoot).mode & 0o777)
.toString(8)
.padStart(4, "0"),
secretMode: (lstatSync(join(SECRET_ROOT, desired.profileId)).mode & 0o777)
.toString(8)
.padStart(4, "0"),
descriptorDeleted: !existsSync(
join(SECRET_ROOT, desired.profileId, "turn-context.json")
),
bootTimeInstall: false,
publicListener: false,
nativeSlackListener: Boolean(proof.gatewayReadback?.running),
apiServerListener: Boolean(proof.apiServerReadiness),
apiServerReadiness: proof.apiServerReadiness ?? {
host: "127.0.0.1",
port: 8642,
cors: false,
keyGeneration: desired.apiServerKeyGeneration,
keyFingerprint: desired.apiServerKeyFingerprint,
admission: "HELD_UNTIL_GATEWAY_START",
},
sharedAppCredentialPresent: false,
});
return {
...candidateRuntime,
modelHealthReporter,
};
}
async function main() {
if (
!/^fly-runtime_[a-f0-9]{32}$/.test(RUNTIME_ID) ||
!/^runtime-[a-f0-9]{32}$/.test(CUSTOMER_KEY) ||
!AUDIENCE ||
!Number.isSafeInteger(RUNTIME_GENERATION) ||
RUNTIME_GENERATION < 1
) {
throw new Error("customer_configuration_rejected");
}
const scratch = pruneAbandonedRuntimeScratch({
hermesRoot: DATA_ROOT,
profilesRoot: PROFILES_ROOT,
});
if (scratch.removed.length > 0) {
process.stderr.write(
`${JSON.stringify({
ok: true,
code: scratch.code,
removed: scratch.removed,
})}\n`
);
}
let oidcToken = await obtainFlyOidcToken({ audience: AUDIENCE });
dropFlyRuntimePrivileges({
user: "hermes",
group: "hermes",
beforeSetuid: ({ gid }) => {
prepareSoulLockRoot({ dataRoot: DATA_ROOT, gid });
if (!allowTestProviders()) handoffFlyOidcSocket({ gid });
},
});
mkdirSync(SECRET_ROOT, { recursive: true, mode: 0o700 });
chmodSync(SECRET_ROOT, 0o700);
const identity = loadOrCreateFlyRuntimeIdentity({
identityRoot: join(DATA_ROOT, ".sellable-agent", "identity"),
persistent: true,
});
let exchange = await exchangeFlyCustomerIdentity({
origin: ORIGIN,
runtimeId: RUNTIME_ID,
audience: AUDIENCE,
identity,
bootSessionId: BOOT_SESSION_ID,
oidcToken,
});
oidcToken = null;
if (exchange.keyGeneration !== RUNTIME_GENERATION) {
throw new Error("customer_runtime_generation_rejected");
}
let client = createFlyCustomerControlClient({
origin: ORIGIN,
identity,
exchange,
renewExchange: async () =>
(
await renewFlyCustomerControlSession({
origin: ORIGIN,
runtimeId: RUNTIME_ID,
audience: AUDIENCE,
identity,
bootSessionId: BOOT_SESSION_ID,
runtimeGeneration: RUNTIME_GENERATION,
})
).exchange,
});
const controller = new AbortController();
let activeRuntime = null;
let shutdownMemorySnapshot = null;
const endpointRelay = createCustomerEndpointRelay({
getControlClient: () => client,
getActiveRuntime: () => activeRuntime,
runtimeGeneration: RUNTIME_GENERATION,
diagnostic: (value) => process.stderr.write(`${JSON.stringify(value)}\n`),
});
const endpointLoop = endpointRelay.run(controller.signal);
const beginShutdownSnapshot = () => {
if (!activeRuntime || shutdownMemorySnapshot) return;
const runtime = activeRuntime;
shutdownMemorySnapshot = (async () => {
try {
await endpointRelay.fenceActive("RUNTIME_SHUTDOWN");
await stopNativeGateway(runtime.gateway);
runtime.gateway = null;
return await flushHermesSnapshot({
client,
profileId: runtime.desired.profileId,
profileRoot: runtime.profileRoot,
});
} catch {
return {
ok: false,
status: "PENDING_RETRY",
code: "hermes_snapshot_shutdown_failed",
};
}
})();
};
const stop = () => {
controller.abort();
beginShutdownSnapshot();
};
process.once("SIGINT", stop);
process.once("SIGTERM", stop);
let operationHeartbeat = null;
let soulClaimTracker = null;
let failures = 0;
while (!controller.signal.aborted) {
try {
let claim;
try {
claim =
activeRuntime?.desired?.profileId && soulClaimTracker
? await claimWithSoulObservation({
client,
tracker: soulClaimTracker,
readSoul: (request) =>
executeExternalSoulBridgeOperation(request, {
dataRoot: DATA_ROOT,
callerProfileId: activeRuntime.desired.profileId,
}),
leaseSeconds: 60,
servingObservation: customerServingObservation(
activeRuntime,
exchange
),
})
: await client.claim(60);
} catch (error) {
if (!isFlyCustomerEnrollmentRenewalError(error)) throw error;
const renewed = await renewFlyCustomerControlSession({
origin: ORIGIN,
runtimeId: RUNTIME_ID,
audience: AUDIENCE,
identity,
bootSessionId: BOOT_SESSION_ID,
runtimeGeneration: RUNTIME_GENERATION,
});
exchange = renewed.exchange;
client = renewed.client;
failures = 0;
process.stderr.write(
`${JSON.stringify({
ok: true,
code: "customer_enrollment_renewed",
keyGeneration: exchange.keyGeneration,
})}\n`
);
continue;
}
if (controller.signal.aborted) break;
if (!claim) {
if (activeRuntime) {
await reconcileHermesSnapshot({
profileId: activeRuntime.desired.profileId,
profileRoot: activeRuntime.profileRoot,
hermesRoot: DATA_ROOT,
hermesVersion: HERMES_VERSION,
upload: ({ archiveDigest, archiveBytes, sourceProfileId, bytes }) =>
client.putMemorySnapshot(
{
archiveDigest,
archiveBytes,
hermesVersion: HERMES_VERSION,
sourceProfileId,
},
bytes
),
});
void sessionRetention.tick({
apiKeyFile: join(
SECRET_ROOT,
activeRuntime.desired.profileId,
"api-server-key"
),
statePath: join(
activeRuntime.profileRoot,
".sellable-agent",
"session-retention.json"
),
});
}
if (
activeRuntime?.action !== "INSTANTIATE" &&
activeRuntime?.gateway &&
!activeRuntime.gateway.readback().running
) {
activeRuntime.gateway = await startNativeGateway({
desired: activeRuntime.desired,
runtimeConfig: activeRuntime.runtimeConfig,
secrets: activeRuntime.secrets,
});
}
await sleep(1_000);
continue;
}
operationHeartbeat = startOperationHeartbeat({
claim,
exchange,
client,
});
if (isOperationBoundBackupClaim(claim)) {
try {
const snapshotIdentity = operationSnapshotIdentity(claim, exchange);
if (!snapshotIdentity) {
throw new Error("customer_operation_backup_claim_rejected");
}
const backup = await flushHermesSnapshot({
client,
profileId: claim.operation.profileId,
profileRoot: join(PROFILES_ROOT, claim.operation.profileId),
operationIdentity: snapshotIdentity,
forceDirty: true,
});
if (!backup.ok || backup.status !== "UPLOADED") {
throw new Error(
backup.code ?? "customer_operation_backup_proof_unavailable"
);
}
// BACKUP advances authority while deliberately keeping this gateway
// alive. Future serving renewals must therefore use the snapshot
// claim instead of the now-stale lifecycle operation generation.
if (activeRuntime) activeRuntime = { ...activeRuntime, claim };
} catch (error) {
const failureReceipt = customerRuntimeBackupFailureCompletionReceipt({
claim,
exchange,
error,
});
const failure = await client.complete(failureReceipt);
if (failure?.accepted !== true) {
throw new Error("customer_runtime_failure_report_rejected");
}
process.stderr.write(
`${JSON.stringify({
ok: true,
code: "customer_runtime_failure_reported",
errorClass: failureReceipt.errorClass,
})}\n`
);
operationHeartbeat.assertHealthy();
await operationHeartbeat.stop();
operationHeartbeat = null;
failures = 0;
continue;
}
operationHeartbeat.assertHealthy();
await operationHeartbeat.stop();
operationHeartbeat = null;
failures = 0;
continue;
}
await activeRuntime?.modelHealthReporter?.stop();
await endpointRelay.fenceActive("LIFECYCLE_REFRESH");
await stopNativeGateway(activeRuntime?.gateway);
// A failed new preparation must not revive the stopped predecessor or
// replace a new gateway whose ownership has not reached activeRuntime.
activeRuntime = null;
soulClaimTracker = null;
activeRuntime = await processOperation({
claim,
exchange,
client,
identity,
reattestAuthority: () => operationHeartbeat.reattest(),
});
soulClaimTracker = createSoulClaimObservationTracker({
runtimeId: RUNTIME_ID,
profileId: activeRuntime.desired.profileId,
});
operationHeartbeat.assertHealthy();
await operationHeartbeat.stop();
operationHeartbeat = null;
failures = 0;
} catch (error) {
try {
await operationHeartbeat?.stop();
} catch (heartbeatError) {
error = heartbeatError;
} finally {
operationHeartbeat = null;
}
failures = Math.min(failures + 1, 6);
process.stderr.write(
`${JSON.stringify({
ok: false,
code:
error instanceof Error ? error.message : "customer_runtime_failed",
})}\n`
);
await sleep(Math.min(30_000, 500 * 2 ** failures));
}
}
beginShutdownSnapshot();
const finalMemorySnapshot = await shutdownMemorySnapshot;
await endpointLoop;
if (finalMemorySnapshot?.ok === false) {
process.stderr.write(
`${JSON.stringify({
ok: false,
code: finalMemorySnapshot.code,
})}\n`
);
}
await stopNativeGateway(activeRuntime?.gateway);
await activeRuntime?.modelHealthReporter?.stop();
if (activeRuntime?.desired.profileId) {
await clearManagedAuth(activeRuntime.desired.profileId);
clearCustomerRuntimeSecrets({
secretsRoot: SECRET_ROOT,
profileId: activeRuntime.desired.profileId,
});
}
}
const executedPath = process.argv[1]
? pathToFileURL(process.argv[1]).href
: null;
if (executedPath === import.meta.url) {
main().catch((error) => {
process.stderr.write(
`${JSON.stringify({
ok: false,
code: error instanceof Error ? error.message : "customer_boot_failed",
})}\n`
);
process.exit(78);
});
}