@sellable/install/lib/sellable-agent/model-health-reporter.mjs

One-command installer for Sellable MCP in Claude Code, Codex, and Hermes

import { randomUUID } from "node:crypto";
import {
  chmodSync,
  existsSync,
  lstatSync,
  mkdirSync,
  readFileSync,
  renameSync,
  writeFileSync,
} from "node:fs";
import { dirname } from "node:path";

const SHA256 = /^[a-f0-9]{64}$/;
const SAFE_ID = /^[A-Za-z0-9_.:-]{1,256}$/;
const FINGERPRINT = /^[a-f0-9]{16,64}$/;
const SAFE_REASON = /^[a-z0-9_.:-]{1,128}$/;
const LEGACY_READBACK_SCHEMA = "sellable-agent-model-auth-readback/v2";
const CURRENT_READBACK_SCHEMA = "sellable-agent-model-auth-readback/v3";
const LEGACY_SLOTS = Object.freeze([
  Object.freeze({ slotId: "dittto-primary", priority: 1 }),
  Object.freeze({ slotId: "kaigo-secondary", priority: 2 }),
]);
const CURRENT_SLOTS = Object.freeze([
  Object.freeze({ slotId: "sellable-primary", priority: 1 }),
  Object.freeze({ slotId: "dittto-secondary", priority: 2 }),
  Object.freeze({ slotId: "kaigo-tertiary", priority: 3 }),
]);

function isRecord(value) {
  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
}

function rejectIdentity() {
  throw new Error("model_health_report_identity_rejected");
}

function safeOrigin(value, allowHttpLoopback) {
  try {
    const parsed = new URL(value);
    const testHttp =
      allowHttpLoopback === true &&
      parsed.protocol === "http:" &&
      ["127.0.0.1", "localhost", "host.docker.internal"].includes(
        parsed.hostname
      );
    if (
      (parsed.protocol !== "https:" && !testHttp) ||
      parsed.username ||
      parsed.password ||
      parsed.pathname !== "/" ||
      parsed.search ||
      parsed.hash
    ) {
      return null;
    }
    return parsed.origin;
  } catch {
    return null;
  }
}

function safeSnapshot(snapshot) {
  if (
    !isRecord(snapshot) ||
    ![LEGACY_READBACK_SCHEMA, CURRENT_READBACK_SCHEMA].includes(
      snapshot.schemaVersion
    ) ||
    !SAFE_ID.test(snapshot.activationId ?? "") ||
    !Number.isSafeInteger(snapshot.bundleGeneration) ||
    snapshot.bundleGeneration < 1 ||
    !SHA256.test(snapshot.policyHash ?? "") ||
    snapshot.provider !== "openai-codex" ||
    !SAFE_ID.test(snapshot.model ?? "") ||
    !Number.isSafeInteger(snapshot.fallbackDepth) ||
    snapshot.fallbackDepth < 0 ||
    snapshot.fallbackDepth > 8 ||
    !["converged", "unknown"].includes(snapshot.desiredStatus) ||
    !["current", "pending_start"].includes(snapshot.materializedStatus) ||
    !Array.isArray(snapshot.slots) ||
    snapshot.slots.length >
      (snapshot.schemaVersion === CURRENT_READBACK_SCHEMA ? 3 : 2) ||
    !isRecord(snapshot.openrouter) ||
    !FINGERPRINT.test(snapshot.openrouter.keyFingerprint ?? "") ||
    !["ok", "exhausted", "dead"].includes(snapshot.openrouter.state) ||
    (snapshot.activeProvider !== undefined &&
      snapshot.activeProvider !== null &&
      !["openai-codex", "openrouter"].includes(snapshot.activeProvider)) ||
    (snapshot.activeFallbackModelId !== undefined &&
      snapshot.activeFallbackModelId !== null &&
      (typeof snapshot.activeFallbackModelId !== "string" ||
        snapshot.activeFallbackModelId.length < 1 ||
        snapshot.activeFallbackModelId.length > 256))
  ) {
    throw new Error("model_health_snapshot_rejected");
  }
  const expectedSlots =
    snapshot.schemaVersion === CURRENT_READBACK_SCHEMA
      ? CURRENT_SLOTS
      : LEGACY_SLOTS;
  const slots = snapshot.slots.map((slot, index) => {
    const expected = expectedSlots[index];
    if (
      !isRecord(slot) ||
      slot.slotId !== expected?.slotId ||
      (snapshot.schemaVersion === CURRENT_READBACK_SCHEMA &&
        slot.priority !== expected.priority) ||
      !FINGERPRINT.test(slot.grantFingerprint ?? "") ||
      !Number.isFinite(Date.parse(slot.expiresAt ?? "")) ||
      !["ok", "exhausted", "dead"].includes(slot.state) ||
      (slot.lastErrorReason !== null &&
        !SAFE_REASON.test(slot.lastErrorReason ?? "")) ||
      (slot.lastErrorResetAt !== null &&
        !Number.isFinite(Date.parse(slot.lastErrorResetAt ?? "")))
    ) {
      throw new Error("model_health_snapshot_rejected");
    }
    return Object.freeze({
      slotId: slot.slotId,
      ...(snapshot.schemaVersion === CURRENT_READBACK_SCHEMA
        ? { priority: slot.priority }
        : {}),
      grantFingerprint: slot.grantFingerprint,
      expiresAt: new Date(slot.expiresAt).toISOString(),
      state: slot.state,
      lastErrorReason: slot.lastErrorReason,
      lastErrorResetAt:
        slot.lastErrorResetAt === null
          ? null
          : new Date(slot.lastErrorResetAt).toISOString(),
    });
  });
  return Object.freeze({
    schemaVersion: snapshot.schemaVersion,
    activationId: snapshot.activationId,
    bundleGeneration: snapshot.bundleGeneration,
    policyHash: snapshot.policyHash,
    provider: snapshot.provider,
    model: snapshot.model,
    fallbackDepth: snapshot.fallbackDepth,
    desiredStatus: snapshot.desiredStatus,
    materializedStatus: snapshot.materializedStatus,
    slots: Object.freeze(slots),
    openrouter: Object.freeze({
      keyFingerprint: snapshot.openrouter.keyFingerprint,
      state: snapshot.openrouter.state,
    }),
    activeProvider: snapshot.activeProvider ?? null,
    activeFallbackModelId: snapshot.activeFallbackModelId ?? null,
  });
}

function codexHealthState(slot) {
  if (slot.state === "ok") return "HEALTHY";
  if (slot.state === "exhausted") {
    return /(?:usage[_-]?limit[_-]?reached|quota|credits?|billing|subscription)/i.test(
      slot.lastErrorReason ?? ""
    )
      ? "EXHAUSTED"
      : "REFRESHING";
  }
  if (/revok/.test(slot.lastErrorReason ?? "")) return "REVOKED";
  if (/(?:auth|unauthor|invalid|expired)/.test(slot.lastErrorReason ?? "")) {
    return "REAUTH_REQUIRED";
  }
  return "DEAD";
}

function fallbackHealthState(snapshot) {
  const expectedSlotCount =
    snapshot.schemaVersion === CURRENT_READBACK_SCHEMA ? 3 : 2;
  const allCodexUnavailable =
    snapshot.slots.length === expectedSlotCount &&
    snapshot.slots.every((slot) => slot.state !== "ok");
  const allFailed =
    snapshot.slots.length === expectedSlotCount &&
    snapshot.slots.every((slot) => slot.state === "dead") &&
    snapshot.openrouter.state === "dead";
  if (allFailed) return Object.freeze({ state: "ALL_FAILED", modelId: null });
  if (
    snapshot.activeProvider === "openrouter" &&
    (snapshot.schemaVersion === LEGACY_READBACK_SCHEMA || allCodexUnavailable)
  ) {
    return Object.freeze({
      state: "OPENROUTER_ACTIVE",
      modelId:
        typeof snapshot.activeFallbackModelId === "string"
          ? snapshot.activeFallbackModelId
          : null,
    });
  }
  return Object.freeze({ state: "CODEX_ACTIVE", modelId: null });
}

function loadSequence(path, identity) {
  if (!path || !existsSync(path)) return 0;
  const stat = lstatSync(path);
  if (stat.isSymbolicLink() || !stat.isFile() || (stat.mode & 0o077) !== 0) {
    throw new Error("model_health_sequence_store_rejected");
  }
  try {
    const value = JSON.parse(readFileSync(path, "utf8"));
    if (
      value?.schemaVersion !== "sellable-agent-model-health-sequence/v1" ||
      value.runtimeGeneration !== identity.runtimeGeneration ||
      value.workloadKeyHash !== identity.workloadKeyHash
    ) {
      return 0;
    }
    if (!Number.isSafeInteger(value.sequence) || value.sequence < 0) {
      throw new Error("model_health_sequence_store_rejected");
    }
    return value.sequence;
  } catch (error) {
    if (
      error instanceof Error &&
      error.message === "model_health_sequence_store_rejected"
    ) {
      throw error;
    }
    throw new Error("model_health_sequence_store_rejected");
  }
}

function persistSequence(path, identity, sequence) {
  if (!path) return;
  mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
  const temporary = `${path}.${process.pid}.${randomUUID()}.tmp`;
  writeFileSync(
    temporary,
    `${JSON.stringify({
      schemaVersion: "sellable-agent-model-health-sequence/v1",
      runtimeGeneration: identity.runtimeGeneration,
      workloadKeyHash: identity.workloadKeyHash,
      sequence,
    })}\n`,
    { flag: "wx", mode: 0o600 }
  );
  chmodSync(temporary, 0o600);
  renameSync(temporary, path);
  chmodSync(path, 0o600);
}

export function modelHealthCadenceMs(workloadKeyHash) {
  if (!SHA256.test(workloadKeyHash ?? "")) rejectIdentity();
  const jitter = Number.parseInt(workloadKeyHash.slice(0, 8), 16) % 60_001;
  return 240_000 + jitter;
}

export function buildModelHealthReport(input) {
  if (
    !["CUSTOMER", "ADMIN"].includes(input?.kind) ||
    !Number.isSafeInteger(input.runtimeGeneration) ||
    input.runtimeGeneration < 1 ||
    !SHA256.test(input.workloadKeyHash ?? "") ||
    !SAFE_ID.test(input.machineId ?? "") ||
    !Number.isSafeInteger(input.sequence) ||
    input.sequence < 1 ||
    !Number.isFinite(Date.parse(input.observedAt ?? "")) ||
    (input.kind === "ADMIN"
      ? !SAFE_ID.test(input.adminRuntimeGenerationId ?? "") ||
        !/^[1-9][0-9]{0,19}$/.test(input.adminRuntimeGenerationFence ?? "")
      : input.adminRuntimeGenerationId !== null ||
        input.adminRuntimeGenerationFence !== null)
  ) {
    rejectIdentity();
  }
  const snapshot = safeSnapshot(input.snapshot);
  const slots = new Map(snapshot.slots.map((slot) => [slot.slotId, slot]));
  if (
    snapshot.schemaVersion === CURRENT_READBACK_SCHEMA &&
    snapshot.activeProvider === "openrouter" &&
    snapshot.slots.some((slot) => slot.state === "ok")
  ) {
    throw new Error("model_health_snapshot_rejected");
  }
  if (snapshot.schemaVersion === CURRENT_READBACK_SCHEMA) {
    if (
      CURRENT_SLOTS.some(
        (expected) =>
          !slots.has(expected.slotId) ||
          slots.get(expected.slotId)?.priority !== expected.priority
      )
    ) {
      throw new Error("model_health_snapshot_rejected");
    }
    return Object.freeze({
      schemaVersion: "sellable-agent-model-health/v2",
      sequence: input.sequence,
      observedAt: new Date(input.observedAt).toISOString(),
      machineId: input.machineId,
      runtimeGeneration: input.runtimeGeneration,
      workloadKeyHash: input.workloadKeyHash,
      kind: input.kind,
      adminRuntimeGenerationId: input.adminRuntimeGenerationId,
      adminRuntimeGenerationFence: input.adminRuntimeGenerationFence,
      activationId: snapshot.activationId,
      bundleGeneration: snapshot.bundleGeneration,
      policyHash: snapshot.policyHash,
      slots: Object.freeze(
        CURRENT_SLOTS.map((expected) => {
          const slot = slots.get(expected.slotId);
          return Object.freeze({
            slotId: expected.slotId,
            priority: expected.priority,
            state: codexHealthState(slot),
            grantFingerprint: slot.grantFingerprint,
            lastErrorReason: slot.lastErrorReason,
            lastErrorResetAt: slot.lastErrorResetAt,
          });
        })
      ),
      fallback: fallbackHealthState(snapshot),
    });
  }
  const primary = slots.get("dittto-primary");
  const secondary = slots.get("kaigo-secondary");
  if (!primary || !secondary) throw new Error("model_health_snapshot_rejected");
  return Object.freeze({
    schemaVersion: "sellable-agent-model-health/v1",
    sequence: input.sequence,
    observedAt: new Date(input.observedAt).toISOString(),
    machineId: input.machineId,
    runtimeGeneration: input.runtimeGeneration,
    workloadKeyHash: input.workloadKeyHash,
    kind: input.kind,
    adminRuntimeGenerationId: input.adminRuntimeGenerationId,
    adminRuntimeGenerationFence: input.adminRuntimeGenerationFence,
    activationId: snapshot.activationId,
    bundleGeneration: snapshot.bundleGeneration,
    policyHash: snapshot.policyHash,
    codex: Object.freeze({
      primary: codexHealthState(primary),
      secondary: codexHealthState(secondary),
    }),
    fallback: fallbackHealthState(snapshot),
  });
}

/**
 * @param {{
 *   origin: string,
 *   identity: Record<string, unknown>,
 *   observe: () => unknown,
 *   authorize: (body: unknown) => string,
 *   fetchImpl?: typeof fetch,
 *   now?: () => Date,
 *   onError?: (error: unknown) => void,
 *   setIntervalImpl?: typeof setInterval,
 *   clearIntervalImpl?: typeof clearInterval,
 *   allowHttpLoopback?: boolean,
 *   sequencePath?: string | null,
 * }} input
 */
export function createModelHealthReporter({
  origin,
  identity,
  observe,
  authorize,
  fetchImpl = fetch,
  now = () => new Date(),
  onError = () => {},
  setIntervalImpl = setInterval,
  clearIntervalImpl = clearInterval,
  allowHttpLoopback = false,
  sequencePath = null,
}) {
  const endpoint = safeOrigin(origin, allowHttpLoopback);
  if (
    !endpoint ||
    !isRecord(identity) ||
    typeof observe !== "function" ||
    typeof authorize !== "function" ||
    typeof fetchImpl !== "function" ||
    typeof now !== "function" ||
    (sequencePath !== null &&
      (typeof sequencePath !== "string" || !sequencePath))
  ) {
    rejectIdentity();
  }
  // Validate the workload binding before any timer or network side effect.
  const cadenceMs = modelHealthCadenceMs(identity.workloadKeyHash);
  buildModelHealthReport({
    ...identity,
    sequence: 1,
    observedAt: now().toISOString(),
    snapshot: observe(),
  });
  let sequence = loadSequence(sequencePath, identity);
  let inFlight = null;
  let timer = null;
  let stopped = false;

  const emit = (_transition = "periodic") => {
    if (stopped)
      return Promise.reject(new Error("model_health_reporter_stopped"));
    if (inFlight) return inFlight;
    const nextSequence = sequence + 1;
    const snapshot = observe();
    const body = buildModelHealthReport({
      ...identity,
      sequence: nextSequence,
      observedAt: now().toISOString(),
      snapshot,
    });
    persistSequence(sequencePath, identity, nextSequence);
    sequence = nextSequence;
    const authorization = authorize(body);
    if (
      typeof authorization !== "string" ||
      !authorization.startsWith("Worker ")
    ) {
      return Promise.reject(new Error("model_health_report_unsigned"));
    }
    inFlight = (async () => {
      const response = await fetchImpl(
        `${endpoint}/api/v3/sellable-agent/worker/model-health`,
        {
          method: "POST",
          headers: { authorization, "content-type": "application/json" },
          body: JSON.stringify(body),
          signal: AbortSignal.timeout(20_000),
        }
      );
      let result;
      try {
        result = await response.json();
      } catch {
        throw new Error("model_health_report_response_rejected");
      }
      if (!response.ok || result?.accepted !== true) {
        throw new Error("model_health_report_rejected");
      }
      return Object.freeze({ accepted: true, sequence });
    })().finally(() => {
      inFlight = null;
    });
    return inFlight;
  };

  return Object.freeze({
    cadenceMs,
    emit,
    async start(transition = "materialized") {
      try {
        return await emit(transition);
      } finally {
        if (!timer && !stopped) {
          timer = setIntervalImpl(() => {
            emit("periodic").catch(onError);
          }, cadenceMs);
          timer?.unref?.();
        }
      }
    },
    async stop() {
      stopped = true;
      if (timer) clearIntervalImpl(timer);
      timer = null;
      await inFlight;
    },
  });
}