@kontextmind/kxm/scripts/smoke-multi-pi.mjs

KXM local-first multi-agent orchestration and operator dashboard

#!/usr/bin/env node

import { spawn, spawnSync } from "node:child_process";
import { createHash, createHmac, randomBytes, randomUUID } from "node:crypto";
import { chmodSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { createServer } from "node:net";
import { tmpdir } from "node:os";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";

const root = resolve(dirname(fileURLToPath(import.meta.url)), "..");
const hubScript = join(root, "scripts", "kxm-hub.mjs");
const workerScript = join(root, "scripts", "kxm-worker.mjs");
const extensionPath = join(root, "plugins", "kxm", "src", "extension.ts");
const DEFAULT_TIMEOUT_MS = 120_000;
const MAX_CAPTURE_CHARS = 32_768;

export function parseSmokeModels(value) {
  const models = [...new Set(String(value ?? "").split(",").map((item) => item.trim()).filter(Boolean))];
  if (models.some((model) => !/^[A-Za-z0-9._:/-]{1,160}$/.test(model))) {
    throw new Error("KXM_SMOKE_MODELS contains an invalid model ID");
  }
  return models;
}

export function boundedSafeMessage(value, secrets = []) {
  let result = value instanceof Error ? value.message : String(value);
  for (const secret of secrets) {
    if (secret) result = result.replaceAll(secret, "[REDACTED]");
  }
  result = result.replace(/(?:sk|key|token|secret)[-_A-Za-z0-9]{12,}/gi, "[REDACTED]");
  return result.slice(0, 500);
}

export async function runSmokePhases(phases, cleanup) {
  const completed = [];
  try {
    const values = {};
    for (const [name, action] of phases) {
      values[name] = await action(values);
      completed.push(name);
    }
    return { values, completed };
  } finally {
    await cleanup();
  }
}

function writeOutcome(outcome) {
  process.stdout.write(`${JSON.stringify(outcome)}\n`);
}

function runCommand(command, args, options = {}) {
  return spawnSync(command, args, {
    encoding: "utf8",
    windowsHide: true,
    shell: process.platform === "win32" && /\.(?:cmd|bat)$/i.test(command),
    timeout: options.timeout ?? 15_000,
    env: options.env ?? process.env,
    stdio: ["ignore", "pipe", "pipe"],
  });
}

function resolvePiCommand() {
  const configured = process.env.KXM_SMOKE_PI_COMMAND?.trim();
  if (configured) return configured;
  const locator = runCommand(process.platform === "win32" ? "where.exe" : "which", ["pi"]);
  if (locator.status !== 0) return undefined;
  const candidates = String(locator.stdout).split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
  return process.platform === "win32"
    ? candidates.find((candidate) => /\.cmd$/i.test(candidate)) ?? candidates[0]
    : candidates[0];
}

function modelAuthReady(piCommand, model) {
  const checked = runCommand(piCommand, ["auth", "check", "--model", model, "--json", "--no-refresh"]);
  if (checked.status !== 0) return false;
  try {
    return JSON.parse(String(checked.stdout)).status === "ready";
  } catch {
    return false;
  }
}

async function freePort() {
  const server = createServer();
  await new Promise((accept, reject) => {
    server.once("error", reject);
    server.listen(0, "127.0.0.1", accept);
  });
  const address = server.address();
  if (!address || typeof address === "string") throw new Error("failed to allocate a local smoke port");
  await new Promise((accept, reject) => server.close((error) => error ? reject(error) : accept()));
  return address.port;
}

function captureProcess(child, label) {
  const state = { label, stdout: "", stderr: "", exit: undefined };
  child.stdout?.on("data", (chunk) => {
    state.stdout = `${state.stdout}${chunk}`.slice(-MAX_CAPTURE_CHARS);
  });
  child.stderr?.on("data", (chunk) => {
    state.stderr = `${state.stderr}${chunk}`.slice(-MAX_CAPTURE_CHARS);
  });
  child.once("exit", (code, signal) => {
    state.exit = { code, signal };
  });
  return state;
}

function spawnNode(script, env, label) {
  const child = spawn(process.execPath, [script], {
    cwd: root,
    env,
    stdio: ["ignore", "pipe", "pipe"],
    windowsHide: true,
  });
  return { child, capture: captureProcess(child, label) };
}

async function waitFor(check, timeoutMs, label, intervalMs = 250) {
  const deadline = Date.now() + timeoutMs;
  let lastError;
  while (Date.now() < deadline) {
    try {
      const value = await check();
      if (value) return value;
    } catch (error) {
      lastError = error;
    }
    await new Promise((accept) => setTimeout(accept, intervalMs));
  }
  throw new Error(`${label} timed out${lastError ? `: ${boundedSafeMessage(lastError)}` : ""}`);
}

async function api(baseUrl, path, options = {}) {
  const response = await fetch(`${baseUrl}${path}`, {
    method: options.method ?? "GET",
    headers: {
      "content-type": "application/json",
      authorization: `Bearer ${options.authToken}`,
      ...(options.identity ? {
        "x-kxm-agent-id": options.identity.agent.id,
        "x-kxm-agent-key": options.identity.agentKey,
      } : {}),
      ...(options.headers ?? {}),
    },
    ...(options.body === undefined ? {} : { body: typeof options.body === "string" ? options.body : JSON.stringify(options.body) }),
    signal: AbortSignal.timeout(options.timeoutMs ?? 10_000),
  });
  const text = await response.text();
  let body = {};
  if (text) {
    try {
      body = JSON.parse(text);
    } catch {
      throw new Error(`hub returned invalid JSON with HTTP ${response.status}`);
    }
  }
  if (!response.ok) throw new Error(`hub HTTP ${response.status}: ${body.code ?? "request_failed"}`);
  return body;
}

async function registerOperator(baseUrl, authToken, project) {
  return await api(baseUrl, "/v1/agents/register", {
    method: "POST",
    authToken,
    body: { name: "smoke-operator", purpose: "bounded smoke assertions", project },
  });
}

async function agents(baseUrl, authToken, operator) {
  const result = await api(baseUrl, "/v1/agents", { authToken, identity: operator });
  return result.agents;
}

export function buildSmokeRequest(target, content, idempotencyKey) {
  return { target, content, delivery: "followUp", idempotencyKey };
}

async function sendAndAwait(baseUrl, authToken, operator, target, content, timeoutMs, idempotencyKey) {
  const sent = await api(baseUrl, "/v1/messages", {
    method: "POST",
    authToken,
    identity: operator,
    body: buildSmokeRequest(target, content, idempotencyKey),
  });
  return await waitFor(async () => {
    const result = await api(baseUrl, `/v1/messages/${encodeURIComponent(sent.message.id)}`, {
      authToken,
      identity: operator,
    });
    if (["cancelled", "expired", "error"].includes(result.message.status)) {
      throw new Error(`peer request ended as ${result.message.status}`);
    }
    return result.message.status === "replied" && typeof result.message.reply?.content === "string"
      && result.message.reply.content.trim() ? result.message : undefined;
  }, timeoutMs, `reply from ${target}`, 500);
}

function createPiShim(tempRoot, piCommand) {
  if (process.platform === "win32") {
    const commandShim = join(tempRoot, "pi-smoke-shim.cmd");
    const quoteBatch = (value) => `"${String(value).replaceAll('"', '""')}"`;
    writeFileSync(commandShim, [
      "@echo off",
      `call ${quoteBatch(piCommand)} --extension ${quoteBatch(extensionPath)} --approve --no-context-files --no-skills --no-prompt-templates --no-themes %*`,
      "exit /b %errorlevel%",
      "",
    ].join("\r\n"), "utf8");
    return commandShim;
  }
  const shim = join(tempRoot, "pi-smoke-shim.mjs");
  writeFileSync(shim, [
    "#!/usr/bin/env node",
    'import { spawn } from "node:child_process";',
    `const command = ${JSON.stringify(piCommand)};`,
    `const prefix = ${JSON.stringify(["--extension", extensionPath, "--approve", "--no-context-files", "--no-skills", "--no-prompt-templates", "--no-themes"])};`,
    "const windowsScript = process.platform === 'win32' && /\\.(?:cmd|bat)$/i.test(command);",
    "const quote = (value) => `\"${String(value).replaceAll('\\\"', '\\\"\\\"')}\"`;",
    "const executable = windowsScript ? process.env.ComSpec || 'cmd.exe' : command;",
    "const args = [...prefix, ...process.argv.slice(2)];",
    "const launchArgs = windowsScript ? ['/d', '/s', '/v:off', '/c', `\"${[command, ...args].map(quote).join(' ')}\"`] : args;",
    "const child = spawn(executable, launchArgs, { stdio: 'inherit', windowsHide: true });",
    "child.once('error', (error) => { process.stderr.write(`${error.message}\\n`); process.exit(1); });",
    "child.once('exit', (code, signal) => { if (signal) process.kill(process.pid, signal); else process.exit(code ?? 1); });",
    "for (const signal of ['SIGINT', 'SIGTERM']) process.once(signal, () => child.kill(signal));",
  ].join("\n"), { encoding: "utf8", mode: 0o700 });
  chmodSync(shim, 0o700);
  return shim;
}

function readPidRecord(path) {
  try {
    const value = JSON.parse(readFileSync(path, "utf8"));
    return Number.isInteger(value.pid) && typeof value.startedAt === "string" && typeof value.controlFile === "string"
      ? value : undefined;
  } catch {
    return undefined;
  }
}

function requestGracefulStop(stateDir, role, name, project) {
  const safeName = name?.replace(/[^A-Za-z0-9_.-]/g, "_").slice(0, 32) || "agent";
  const safeProject = project?.replace(/[^A-Za-z0-9_.-]/g, "_").slice(0, 24) || "project";
  const digest = name && project
    ? createHash("sha256").update(JSON.stringify({ project, agentName: name })).digest("hex").slice(0, 24)
    : undefined;
  const pidPath = join(stateDir, role === "hub" ? "hub.pid" : `worker-${safeProject}-${safeName}-${digest}.pid`);
  const record = readPidRecord(pidPath);
  if (!record) return false;
  writeFileSync(join(stateDir, record.controlFile), `${JSON.stringify({ version: 1, startedAt: record.startedAt, ...(record.generation ? { generation: record.generation } : {}), requestedAt: new Date().toISOString() })}\n`, {
    encoding: "utf8",
    mode: 0o600,
  });
  return true;
}

async function waitForExit(processRecord, timeoutMs) {
  if (processRecord.capture.exit) return;
  await Promise.race([
    new Promise((accept) => processRecord.child.once("exit", accept)),
    new Promise((_, reject) => setTimeout(() => reject(new Error(`${processRecord.capture.label} did not stop gracefully`)), timeoutMs)),
  ]);
}

function forceTreeStop(child) {
  if (!child || child.exitCode !== null) return;
  if (process.platform === "win32" && child.pid) {
    spawnSync("taskkill.exe", ["/pid", String(child.pid), "/t", "/f"], { windowsHide: true, stdio: "ignore" });
  } else {
    child.kill("SIGKILL");
  }
}

async function stopRecord(processRecord, stateDir, role, name, project) {
  if (!processRecord || processRecord.capture.exit) return;
  requestGracefulStop(stateDir, role, name, project);
  try {
    await waitForExit(processRecord, 20_000);
  } catch {
    forceTreeStop(processRecord.child);
    await waitForExit(processRecord, 5_000).catch(() => undefined);
  }
}

export async function runRealSmoke(options = {}) {
  const models = options.models ?? parseSmokeModels(process.env.KXM_SMOKE_MODELS);
  if (models.length < 2) throw new Error("KXM_SMOKE_MODELS must name at least two distinct models");
  const piCommand = options.piCommand ?? resolvePiCommand();
  if (!piCommand) return { ok: true, skipped: true, reason: "real Pi binary unavailable" };
  const probe = runCommand(piCommand, ["--version"]);
  if (probe.status !== 0) return { ok: true, skipped: true, reason: "real Pi binary unavailable" };
  const unavailableModels = models.filter((model) => !modelAuthReady(piCommand, model));
  if (unavailableModels.length > 0) {
    return { ok: true, skipped: true, reason: "model credentials unavailable", unavailableModels };
  }

  const timeoutMs = Number(process.env.KXM_SMOKE_TIMEOUT_MS ?? DEFAULT_TIMEOUT_MS);
  if (!Number.isInteger(timeoutMs) || timeoutMs < 30_000 || timeoutMs > 600_000) {
    throw new Error("KXM_SMOKE_TIMEOUT_MS must be an integer from 30000 to 600000");
  }
  const tempRoot = mkdtempSync(join(tmpdir(), "pi-mesh-real-smoke-"));
  const workspaceDir = join(tempRoot, ".kxm");
  const stateDir = join(workspaceDir, "state");
  const logsDir = join(workspaceDir, "logs");
  const assetsDir = join(workspaceDir, "assets");
  for (const directory of [workspaceDir, stateDir, logsDir, assetsDir]) mkdirSync(directory, { recursive: true });
  const port = await freePort();
  const baseUrl = `http://127.0.0.1:${port}`;
  const project = `smoke-${randomUUID()}`;
  const authToken = randomBytes(32).toString("hex");
  const webhookSecret = randomBytes(32).toString("hex");
  const shim = createPiShim(tempRoot, piCommand);
  const workerNames = ["smoke-alpha", "smoke-beta"];
  const workflow = [{
    id: "real-pi-smoke",
    source: "generic",
    project,
    target: "smoke-operator",
    secretEnv: "KXM_SMOKE_WEBHOOK_SECRET",
    event: "smoke.requested",
    delivery: "followUp",
    promptTemplate: "Run isolated smoke {{runId}}",
    stages: [{ id: "verify", label: "Verify", instructions: "Record the deterministic smoke result", requiredEvidence: ["real Pi evidence"], maxAttempts: 1 }],
  }];
  const commonEnv = {
    ...process.env,
    KXM_HOST: "127.0.0.1",
    KXM_PORT: String(port),
    KXM_SERVER_URL: baseUrl,
    KXM_AUTH_TOKEN: authToken,
    KXM_PROJECT: project,
    KXM_WORKDIR: tempRoot,
    KXM_WORKSPACE_DIR: workspaceDir,
    KXM_CONFIG_DIR: join(workspaceDir, "config"),
    KXM_LOGS_DIR: logsDir,
    KXM_ASSETS_DIR: assetsDir,
    KXM_STATE_DIR: stateDir,
    KXM_DATA_PATH: join(stateDir, "kxm.db"),
    KXM_WEBHOOK_WORKFLOWS: JSON.stringify(workflow),
    KXM_SMOKE_WEBHOOK_SECRET: webhookSecret,
  };
  const processes = { hub: undefined, workers: new Map() };
  let stage = "start";
  try {
    const startHub = () => {
      const record = spawnNode(hubScript, commonEnv, "hub");
      processes.hub = record;
      return record;
    };
    const startWorker = (index) => {
      const name = workerNames[index];
      const record = spawnNode(workerScript, {
        ...commonEnv,
        KXM_AGENT_NAME: name,
        KXM_AGENT_PURPOSE: "real Pi smoke peer",
        KXM_WORKER_MODEL: models[index],
        KXM_PI_COMMAND: shim,
        KXM_WORKER_DRAIN_MS: "10000",
        KXM_WORKER_MAX_RESTARTS: "2",
        KXM_WORKER_SESSION_ISOLATION: "workflow",
      }, name);
      processes.workers.set(name, record);
      return record;
    };
    await runSmokePhases([
      ["hub-ready", async () => {
        stage = "hub-ready";
        startHub();
        await waitFor(async () => {
          const response = await fetch(`${baseUrl}/health`, { signal: AbortSignal.timeout(1_000) });
          return response.ok;
        }, 20_000, "hub readiness");
        return registerOperator(baseUrl, authToken, project);
      }],
      ["discovery", async (values) => {
        stage = "discovery";
        const operator = values["hub-ready"];
        startWorker(0);
        startWorker(1);
        const discovered = await waitFor(async () => {
          const current = await agents(baseUrl, authToken, operator);
          return workerNames.every((name) => current.some((agent) => agent.name === name)) ? current : undefined;
        }, timeoutMs, "two real Pi workers");
        return { operator, originalIds: Object.fromEntries(workerNames.map((name) => [name, discovered.find((agent) => agent.name === name).id])) };
      }],
      ["request-reply", async (values) => {
        stage = "request-reply";
        const { operator } = values.discovery;
        return sendAndAwait(baseUrl, authToken, operator, workerNames[0], "This is a transport smoke check. Reply with a short acknowledgement and do not call tools.", timeoutMs, `single-${randomUUID()}`);
      }],
      ["fanout", async (values) => {
        stage = "fanout";
        const { operator } = values.discovery;
        const replies = await Promise.all(workerNames.map((name) => sendAndAwait(baseUrl, authToken, operator, name, "Independently confirm the transport smoke check in one short sentence. Do not call tools.", timeoutMs, `fanout-${name}-${randomUUID()}`)));
        if (replies.length !== 2) throw new Error("fanout did not return two replies");
        return replies;
      }],
      ["durable-restart-resume", async (values) => {
        stage = "durable-restart-resume";
        const { originalIds } = values.discovery;
        await stopRecord(processes.workers.get(workerNames[0]), stateDir, "worker", workerNames[0], project);
        await stopRecord(processes.hub, stateDir, "hub");
        startHub();
        await waitFor(async () => {
          const response = await fetch(`${baseUrl}/health`, { signal: AbortSignal.timeout(1_000) });
          return response.ok;
        }, 20_000, "restarted hub readiness");
        const resumedOperator = await registerOperator(baseUrl, authToken, project);
        startWorker(0);
        const resumed = await waitFor(async () => {
          const current = await agents(baseUrl, authToken, resumedOperator);
          return workerNames.every((name) => current.some((agent) => agent.name === name)) ? current : undefined;
        }, timeoutMs, "worker resume");
        for (const name of workerNames) {
          if (resumed.find((agent) => agent.name === name).id !== originalIds[name]) throw new Error(`durable identity changed for ${name}`);
        }
        await sendAndAwait(baseUrl, authToken, resumedOperator, workerNames[0], "Confirm transport still works after the durable restart. Reply briefly and do not call tools.", timeoutMs, `post-restart-${randomUUID()}`);
        return resumedOperator;
      }],
      ["journal-checkpoint", async (values) => {
        stage = "workflow-journal-checkpoint";
        const resumedOperator = values["durable-restart-resume"];
        const payload = JSON.stringify({ event: "smoke.requested" });
        // KXM webhook contract (workflowWebhookSignedMaterial in plugins/kxm/src/workflow.ts).
        const deliveryId = `smoke-${randomUUID()}`;
        const timestamp = String(Math.floor(Date.now() / 1_000));
        const signedMaterial = ["kxm-webhook-v1", "start", timestamp, deliveryId, "real-pi-smoke", "", ""].join("\n") + "\n" + payload;
        const started = await api(baseUrl, "/v1/webhooks/real-pi-smoke", {
          method: "POST",
          authToken,
          headers: { "x-kxm-delivery-id": deliveryId, "x-kxm-timestamp": timestamp, "x-kxm-signature": `sha256=${createHmac("sha256", webhookSecret).update(signedMaterial).digest("hex")}` },
          body: payload,
        });
        await api(baseUrl, `/v1/workflows/${encodeURIComponent(started.run.id)}/journal`, {
          method: "POST",
          authToken,
          identity: resumedOperator,
          body: { category: "lesson", area: "harness", severity: "info", summary: "Real Pi smoke transport completed", evidence: ["discovery", "request-reply", "fanout", "post-restart-request-reply"] },
        });
        const checkpoint = await api(baseUrl, `/v1/workflows/${encodeURIComponent(started.run.id)}/checkpoints`, {
          method: "POST",
          authToken,
          identity: resumedOperator,
          body: { stageId: "verify", status: "passed", summary: "All real Pi smoke assertions passed", evidence: { "real Pi evidence": "discovery, request/reply, fanout, and post-restart request/reply passed" } },
        });
        if (!checkpoint.completed) throw new Error("workflow checkpoint did not complete");
        const recorded = await api(baseUrl, `/v1/workflows/${encodeURIComponent(started.run.id)}`, { authToken, identity: resumedOperator });
        if (recorded.run.status !== "completed"
          || recorded.run.stages[0]?.status !== "passed"
          || !recorded.journal.some((entry) => entry.category === "lesson" && entry.area === "harness")) {
          throw new Error("workflow journal/checkpoint was not durable");
        }
      }],
    ], async () => {
      for (const [name, record] of processes.workers) await stopRecord(record, stateDir, "worker", name, project);
      await stopRecord(processes.hub, stateDir, "hub");
      for (const record of processes.workers.values()) forceTreeStop(record.child);
      forceTreeStop(processes.hub?.child);
      rmSync(tempRoot, { recursive: true, force: true });
    });
    return { ok: true, skipped: false, realPiWorkers: 2, assertions: ["discovery", "request-reply", "fanout", "durable-restart-resume", "post-restart-request-reply", "journal", "checkpoint"] };
  } catch (error) {
    throw new Error(`${stage}: ${boundedSafeMessage(error, [authToken, webhookSecret])}`);
  }
}

async function main() {
  const enabled = process.env.KXM_SMOKE === "1" || process.argv.includes("--real-pi");
  if (!enabled) {
    writeOutcome({ ok: true, skipped: true, reason: "KXM_SMOKE is not 1" });
    return;
  }
  try {
    writeOutcome(await runRealSmoke());
  } catch (error) {
    writeOutcome({ ok: false, skipped: false, error: boundedSafeMessage(error) });
    process.exitCode = 1;
  }
}

if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) await main();