akm-cli
Version:
akm (Agent Knowledge Manager) — a portable, local-first capability library for AI agents. Discover, load, share, and improve reusable skills, scripts, workflows, and knowledge across any shell-capable coding agent, including Claude Code, OpenCode, and Cur
1,187 lines • 57.8 kB
JavaScript
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
import { spawnSync } from "node:child_process";
import { loadConfig } from "../../core/config/config.js";
import { listEnvsRecursive } from "../../core/env-secret-ref.js";
import { ConfigError } from "../../core/errors.js";
import { EXTRACT_INFRASTRUCTURE_SKIP_REASONS } from "../../core/improve-types.js";
import { listPendingStateMigrations } from "../../core/state-db.js";
import { withEngineFallback } from "../../integrations/agent/engine-fallback.js";
import { isLlmCredentialAvailable, resolveEngine, } from "../../integrations/agent/engine-resolution.js";
import { executionEngineDefinitionsFromConfig } from "../../integrations/agent/execution.js";
import { loadModelMap, mergeModelMapLayers, parseModelMapLayer, readInstalledModelMapText, resolveModelMapAlias, userModelMapPath, } from "../../integrations/agent/model-map.js";
import { probeEndpointOnce } from "../../llm/client.js";
import { STATE_DB_FREELIST_WARN_RATIO, } from "../../storage/state-db-integrity.js";
import { listKeys } from "../env/env.js";
import { MODEL_CALLING_PROCESSES, resolveImprovePlan } from "../improve/improve-strategies.js";
import { ENGINE_LAST_USED_LOOKBACK_DAYS } from "./engine-usage.js";
import { ACTIVE_RUN_WARN_MS, TASK_FAIL_RATE_WARN, } from "./types.js";
/** Probe one connection's reachability, once per endpoint; `undefined` when no probe seam is supplied. */
function probeConnectionReachable(connection, deps, cache) {
if (!deps.probeReachable)
return Promise.resolve(undefined);
return probeEndpointOnce(connection, cache, deps.probeReachable);
}
function reachabilityEvidence(reach) {
return reach
? { probed: true, reachable: reach.reachable, ...(reach.error ? { error: reach.error } : {}) }
: { probed: false, reachable: null };
}
/**
* #914: an unreachable LLM engine is a hard `fail` when it is the resolved
* `default-llm-engine` (extraction and improve depend on it) but only a
* `warn` elsewhere. Only a reachability warn escalates; a credential warn
* stays a warn.
*/
function escalateDefaultLlmEngineFailure(result) {
if (result.status === "warn" && result.evidence?.reachable === false) {
return { ...result, status: "fail" };
}
return result;
}
/** Rolling window the `session-extraction` check reads from the extract ledger (#914). */
export const SESSION_EXTRACTION_LEDGER_WINDOW_DAYS = 7;
/**
* Delegates to {@link isLlmCredentialAvailable} (#953) so `akm health` and the
* improve-strategy probe (#957 builds on this same helper) share one
* non-throwing "is a credential present" check with the real dispatch
* boundary — env value, file-backed (#905), or secret-store-backed (#953).
*/
function credentialAvailable(credential, env, apiKeyFile, apiKeySecretRef) {
return isLlmCredentialAvailable({ credential, apiKeyFile, apiKeySecretRef }, env);
}
/**
* #950: when a required `$VAR`-style credential is missing from the shell,
* find the env asset (if any) whose key names include one of
* `credential.names` — so the warn can say "run under env/lab" instead of a
* bare "unavailable". Returns the asset's ref, or `null` when no credential,
* no candidate names, or no matching env asset. Never returns or logs the
* variable name itself (only ref/path/keys ever leave `listEnvsRecursive`,
* and only the ref is read here) — `tests/health-engine-probe.test.ts` pins
* that a credential's env-var name never appears in health's JSON output.
* Best-effort: any failure walking env assets (unreadable stash, bad config)
* degrades to `null`, never a crash. `deps.listEnvAssets` is expected to
* already be normalised (see {@link withMemoizedEnvAssets}) by the time this
* runs, so the walk happens at most once per probe run — every caller of
* this function passes deps that have already gone through
* {@link withMemoizedEnvAssets}, so there is no separate default to fall
* back to here.
*/
function findSuppliedByEnvAsset(credential, deps) {
if (!credential || credential.names.length === 0)
return null;
try {
const envs = deps.listEnvAssets?.();
if (!envs)
return null;
for (const envAsset of envs) {
if (credential.names.some((name) => envAsset.keys.includes(name)))
return envAsset.ref;
}
}
catch {
// Best-effort — env-asset discovery must never break a health probe.
}
return null;
}
/**
* #950: normalise `deps.listEnvAssets` once per probe run so
* {@link findSuppliedByEnvAsset}'s env-asset walk happens at most once even
* when a run probes several engines with missing credentials — the walk is
* real IO (or config-injected IO in tests) with no reason to repeat per
* engine. Wraps whatever seam is already present (test-injected or the
* config-honouring default) in a memoising closure; called once by each of
* the four probe entry points that share {@link runConfiguredEngineProbe}
* (`runDefaultEngineProbe`, `runDefaultLlmEngineProbe`,
* `runConfiguredEnginesProbe`, `runHealthEngineProbes`).
*/
function withMemoizedEnvAssets(deps) {
const listEnvAssets = deps.listEnvAssets ?? (() => listEnvsRecursive(listKeys, (deps.loadConfig ?? loadConfig)()));
let cached;
return { ...deps, listEnvAssets: () => (cached ??= listEnvAssets()) };
}
async function runConfiguredEngineProbe(checkName, engineName, config, deps, reachabilityCache) {
if (!engineName) {
return {
name: checkName,
kind: "deterministic",
status: "unknown",
confidence: "high",
message: checkName === "default-llm-engine"
? "No default LLM engine is configured."
: "No default engine is configured.",
};
}
const env = deps.env ?? process.env;
const configuredEngine = config.engines?.[engineName];
if (configuredEngine?.kind === "agent" && configuredEngine.platform === "opencode-sdk") {
let packageAvailable = false;
try {
const resolvePackage = deps.resolvePackage ?? ((name) => import.meta.resolve(name));
resolvePackage("@opencode-ai/sdk");
packageAvailable = true;
}
catch {
packageAvailable = false;
}
const binary = configuredEngine.bin ?? "opencode";
let binaryAvailable;
try {
const version = (deps.spawnSync ?? spawnSync)(binary, ["--version"], { encoding: "utf8", timeout: 5_000 });
binaryAvailable = (version.status ?? 1) === 0;
}
catch {
return {
name: checkName,
kind: "deterministic",
status: "warn",
confidence: "high",
message: `SDK engine "${engineName}" executable availability could not be checked.`,
evidence: { engine: engineName, runtimeKind: "sdk", binaryAvailable: false },
};
}
const fallbackEngine = configuredEngine.llmEngine;
let fallback;
let fallbackCredential;
let fallbackApiKeyFile;
let fallbackApiKeySecretRef;
let sdkRunner;
const resolve = deps.resolveEngine ?? resolveEngine;
try {
const resolved = resolve(engineName, config);
if (resolved.kind === "sdk")
sdkRunner = resolved;
}
catch {
sdkRunner = undefined;
}
if (sdkRunner?.fallbackConnection && fallbackEngine) {
fallback = { kind: "llm", engine: fallbackEngine, connection: sdkRunner.fallbackConnection };
fallbackCredential = sdkRunner.fallbackCredential;
fallbackApiKeyFile = sdkRunner.fallbackApiKeyFile;
fallbackApiKeySecretRef = sdkRunner.fallbackApiKeySecretRef;
}
else if (fallbackEngine) {
try {
const resolved = resolve(fallbackEngine, config);
if (resolved.kind === "llm") {
fallback = resolved;
fallbackCredential = resolved.credential;
fallbackApiKeyFile = resolved.apiKeyFile;
fallbackApiKeySecretRef = resolved.apiKeySecretRef;
}
}
catch {
fallback = undefined;
}
}
const configuredModel = configuredEngine.model;
const effectiveModel = sdkRunner?.profile.model ?? configuredModel ?? fallback?.connection.model;
const fallbackCredentialAvailable = credentialAvailable(fallbackCredential, env, fallbackApiKeyFile, fallbackApiKeySecretRef);
const fallbackSuppliedByEnvAsset = !fallbackCredentialAvailable
? findSuppliedByEnvAsset(fallbackCredential, deps)
: null;
const missing = [
!packageAvailable ? "@opencode-ai/sdk package" : undefined,
!binaryAvailable ? `${binary} binary` : undefined,
fallbackEngine && !fallback ? "configured fallback LLM connection" : undefined,
!fallbackCredentialAvailable
? fallbackSuppliedByEnvAsset
? `required fallback credential (available via env asset ${fallbackSuppliedByEnvAsset}; run under it: akm env run ${fallbackSuppliedByEnvAsset} -- ...)`
: "required fallback credential"
: undefined,
].filter((value) => value !== undefined);
const sdkEvidence = {
engine: engineName,
platform: configuredEngine.platform,
runtimeKind: "sdk",
binary,
binaryAvailable,
package: "@opencode-ai/sdk",
packageAvailable,
model: effectiveModel ?? null,
configuredModel: configuredModel ?? null,
modelSource: configuredModel ? "sdk" : effectiveModel ? "fallback" : null,
fallbackEngine: fallbackEngine ?? null,
fallbackEndpoint: fallback?.connection.endpoint ?? null,
fallbackModel: fallback?.connection.model ?? null,
requiredCredentialAvailable: fallbackCredentialAvailable,
suppliedByEnvAsset: fallbackSuppliedByEnvAsset,
};
if (missing.length > 0) {
return {
name: checkName,
kind: "deterministic",
status: "warn",
confidence: "high",
message: `SDK engine "${engineName}" is incomplete: missing ${missing.join(", ")}.`,
evidence: sdkEvidence,
};
}
// The SDK's LLM fallback is the only endpoint this engine talks to;
// probe it once every local precondition above is satisfied (#914).
const reach = fallback ? await probeConnectionReachable(fallback.connection, deps, reachabilityCache) : undefined;
return {
name: checkName,
kind: "deterministic",
status: reach && !reach.reachable ? "warn" : "pass",
confidence: "high",
message: !fallback
? `SDK engine "${engineName}" is available.`
: !reach
? `SDK engine "${engineName}" is available. Reachability was not probed.`
: reach.reachable
? `SDK engine "${engineName}" is available and its LLM fallback is reachable.`
: `SDK engine "${engineName}" is available, but its LLM fallback is not reachable: ${reach.error ?? "unknown error"}`,
evidence: fallback ? { ...sdkEvidence, ...reachabilityEvidence(reach) } : sdkEvidence,
};
}
try {
const runner = (deps.resolveEngine ?? resolveEngine)(engineName, config);
if (runner.kind === "llm") {
const requiredCredentialAvailable = credentialAvailable(runner.credential, env, runner.apiKeyFile, runner.apiKeySecretRef);
const llmEvidence = {
engine: engineName,
platform: null,
runtimeKind: "llm",
model: runner.connection.model,
endpoint: runner.connection.endpoint,
requiredCredentialAvailable,
};
if (!requiredCredentialAvailable) {
// #950: a missing-in-shell credential is common when the operator's
// real workflow is `akm env run env/lab -- akm improve` — name the env
// asset that supplies it (never the variable name) so the warn is
// actionable instead of looking like a broken engine.
const suppliedByEnvAsset = findSuppliedByEnvAsset(runner.credential, deps);
return {
name: checkName,
kind: "deterministic",
status: "warn",
confidence: "high",
message: suppliedByEnvAsset
? `LLM engine "${engineName}" is configured, but its required credential is not available in this shell; env asset ${suppliedByEnvAsset} supplies it — run under it (akm env run ${suppliedByEnvAsset} -- ...).`
: `LLM engine "${engineName}" is configured, but its required credential is unavailable.`,
evidence: { ...llmEvidence, suppliedByEnvAsset },
};
}
const reach = await probeConnectionReachable(runner.connection, deps, reachabilityCache);
return {
name: checkName,
kind: "deterministic",
status: reach && !reach.reachable ? "warn" : "pass",
confidence: "high",
message: !reach
? `LLM engine "${engineName}" is configured. Reachability was not probed.`
: reach.reachable
? `LLM engine "${engineName}" is configured and reachable.`
: `LLM engine "${engineName}" is not reachable: ${reach.error ?? "unknown error"}`,
evidence: { ...llmEvidence, ...reachabilityEvidence(reach) },
};
}
const profile = runner.profile;
if (runner.kind === "sdk")
throw new Error(`SDK engine "${engineName}" has no matching SDK config.`);
const version = (deps.spawnSync ?? spawnSync)(profile.bin, ["--version"], { encoding: "utf8", timeout: 5_000 });
if ((version.status ?? 1) !== 0) {
return {
name: checkName,
kind: "deterministic",
status: "warn",
confidence: "medium",
message: `Agent engine "${engineName}" was found but \`--version\` failed.`,
evidence: {
engine: engineName,
platform: profile.platform ?? null,
runtimeKind: "agent",
model: profile.model ?? null,
},
};
}
return {
name: checkName,
kind: "deterministic",
status: "pass",
confidence: "high",
message: `Agent engine "${engineName}" is available.`,
evidence: {
engine: engineName,
platform: profile.platform ?? null,
runtimeKind: "agent",
model: profile.model ?? null,
},
};
}
catch (error) {
return {
name: checkName,
kind: "deterministic",
status: "warn",
confidence: "high",
message: error instanceof Error ? error.message : String(error),
};
}
}
function projectEngineProbe(result, name) {
return result.name === name ? result : { ...result, name };
}
function projectSelectedEngineProbe(availability, engineName, checkName) {
const result = availability.get(engineName);
if (result) {
const projected = projectEngineProbe(result, checkName);
return checkName === "default-llm-engine" ? escalateDefaultLlmEngineFailure(projected) : projected;
}
return {
name: checkName,
kind: "deterministic",
status: "warn",
confidence: "high",
message: `Engine "${engineName}" availability could not be checked.`,
evidence: { engine: engineName },
};
}
function unconfiguredEngineProbe(name) {
return {
name,
kind: "deterministic",
status: "unknown",
confidence: "high",
message: name === "default-llm-engine" ? "No default LLM engine is configured." : "No default engine is configured.",
};
}
/**
* #949: `enableThinking: false` is a request an engine's endpoint (or a
* gateway in front of it) can silently ignore — `chatCompletionAttemptOnce`
* (src/llm/client.ts) already warns to stderr whenever a response reports
* reasoning tokens despite it. This makes that same signal visible in
* `akm health` as a structured finding, purely by re-reading the window's
* `llm_usage` aggregate already threaded into the context: no extra
* completion call, no live probe — a cold local model must not be woken just
* to run `akm health`.
*/
function projectThinkingControlCheck(thinkingOffEngines, llmUsage, since) {
if (thinkingOffEngines.length === 0) {
return {
name: "thinking-control",
kind: "deterministic",
status: "unknown",
confidence: "high",
message: "No configured engine sets enableThinking: false — nothing to verify.",
evidence: { engines: [] },
};
}
const engines = [...thinkingOffEngines].sort().map((engine) => {
const usage = llmUsage.byEngine[engine];
const calls = usage?.calls ?? 0;
const reasoningTokens = usage?.reasoningTokens ?? 0;
const status = calls === 0 ? "unknown" : reasoningTokens > 0 ? "warn" : "pass";
return { engine, status, calls, reasoningTokens };
});
const warning = engines.filter((e) => e.status === "warn");
const status = warning.length > 0 ? "warn" : engines.every((e) => e.status === "unknown") ? "unknown" : "pass";
let message;
if (warning.length > 0) {
message = warning
.map((e) => `LLM engine "${e.engine}" returned ${e.reasoningTokens} reasoning tokens since ${since} despite enableThinking: false — the endpoint (or a gateway in front of it) is not honouring the thinking-off control.`)
.join(" ");
}
else if (status === "unknown") {
message = `No calls were recorded for the engine(s) configured with enableThinking: false (${engines.map((e) => e.engine).join(", ")}) in the report window.`;
}
else {
message = `${engines.length} engine(s) configured with enableThinking: false (${engines.map((e) => e.engine).join(", ")}) returned no reasoning tokens in the report window.`;
}
return {
name: "thinking-control",
kind: "deterministic",
status,
confidence: "high",
message,
evidence: { engines },
};
}
function configuredEnginesProjection(engineNames, availability) {
if (engineNames.length === 0) {
return {
name: "configured-engines",
kind: "deterministic",
status: "unknown",
confidence: "high",
message: "No engines are explicitly configured.",
evidence: { engines: [] },
};
}
const engines = engineNames.map((engine) => ({
engine,
status: availability.get(engine)?.status === "pass" ? "pass" : "warn",
}));
const unavailable = engines.filter((engine) => engine.status === "warn").length;
return {
name: "configured-engines",
kind: "deterministic",
status: unavailable === 0 ? "pass" : "warn",
confidence: "high",
message: unavailable === 0
? `All ${engines.length} explicitly configured engines are available.`
: `${unavailable} of ${engines.length} explicitly configured engines ${unavailable === 1 ? "is" : "are"} unavailable.`,
evidence: { engines },
};
}
export async function runDefaultEngineProbe(deps = {}) {
// Probe the effective view: an install with no `defaults.engine` but a usable
// opencode binary DOES have a working default, and reporting otherwise would
// contradict what `workflow run` / `task run` actually do.
const { config } = withEngineFallback(deps.loadConfig?.() ?? loadConfig(), deps.which);
return runConfiguredEngineProbe("default-engine", config.defaults?.engine, config, withMemoizedEnvAssets(deps), new Map());
}
export async function runDefaultLlmEngineProbe(deps = {}) {
const config = deps.loadConfig?.() ?? loadConfig();
const engineName = config.defaults?.llmEngine;
if (!engineName)
return unconfiguredEngineProbe("default-llm-engine");
const result = await runConfiguredEngineProbe("configured-engine", engineName, config, withMemoizedEnvAssets(deps), new Map());
return projectSelectedEngineProbe(new Map([[engineName, result]]), engineName, "default-llm-engine");
}
/** Probe every explicitly configured engine without exposing connection or model material. */
export async function runConfiguredEnginesProbe(deps = {}) {
const config = deps.loadConfig?.() ?? loadConfig();
const engineNames = Object.keys(config.engines ?? {}).sort();
const cache = new Map();
const runDeps = withMemoizedEnvAssets(deps);
const availability = new Map(await Promise.all(engineNames.map(async (engine) => [engine, await runConfiguredEngineProbe("configured-engine", engine, config, runDeps, cache)])));
return configuredEnginesProjection(engineNames, availability);
}
/**
* Build the per-health-run availability snapshot. Each distinct selected or
* explicitly configured engine is probed once (reachability probes further
* de-duped by endpoint, see {@link ReachabilityCache}); the three public
* checks are projections of that immutable result set.
*/
export async function runHealthEngineProbes(deps = {}) {
const configured = deps.loadConfig?.() ?? loadConfig();
const effective = withEngineFallback(configured, deps.which).config;
const defaultEngineName = effective.defaults?.engine;
const defaultLlmEngineName = configured.defaults?.llmEngine;
const explicitEngineNames = Object.keys(configured.engines ?? {}).sort();
const probeNames = [...new Set([...explicitEngineNames, defaultEngineName, defaultLlmEngineName])]
.filter((name) => name !== undefined)
.sort();
const cache = new Map();
const runDeps = withMemoizedEnvAssets(deps);
const availability = new Map(await Promise.all(probeNames.map(async (engine) => [engine, await runConfiguredEngineProbe("configured-engine", engine, effective, runDeps, cache)])));
return Object.freeze({
defaultEngine: defaultEngineName
? projectSelectedEngineProbe(availability, defaultEngineName, "default-engine")
: unconfiguredEngineProbe("default-engine"),
defaultLlmEngine: defaultLlmEngineName
? projectSelectedEngineProbe(availability, defaultLlmEngineName, "default-llm-engine")
: unconfiguredEngineProbe("default-llm-engine"),
configuredEngines: configuredEnginesProjection(explicitEngineNames, availability),
});
}
/**
* #950: probes the active improve strategy and returns both its
* `HealthCheckResult` and the typed process→engine map, computed once. See
* {@link runActiveImproveStrategyProbe} for the plain-`HealthCheckResult`
* wrapper existing callers/tests use.
*/
export function probeActiveImproveStrategy(deps = {}) {
const config = deps.loadConfig?.() ?? loadConfig();
const strategyName = config.defaults?.improveStrategy ?? "default";
const env = deps.env ?? process.env;
try {
// #957: credential availability (including the triage-judgment engine) is
// now derived once, inside `buildImprovePlan` itself — the same
// `plan.engineUnavailable` the real `improve` run reads to skip processes
// and populate `AkmImproveResult.skippedProcesses`. This check is a pure
// projection of that list rather than its own re-derivation.
//
// `allowAllDisabled: true` because a strategy left with every process
// disabled purely by credential unavailability must still come back as
// an inspectable plan here, not a thrown ConfigError — the
// `allRequiredUnavailable`/`fail` logic below needs `plan.engineUnavailable`
// to run this check's fail path instead of falling into the generic
// catch block's warn/unknown fallback (round 2 finding on #957). A
// strategy with no engine configured at all still throws (see
// `allowAllDisabled`'s doc comment) and keeps hitting that catch block,
// unchanged from before this fix.
const plan = resolveImprovePlan(strategyName, config, { env, allowAllDisabled: true });
const unavailableProcesses = plan.engineUnavailable.map((item) => item.process);
// #913: name the engine each process actually resolved to, so a
// strategy-level `engine` pin that shadows `defaults.llmEngine` is
// visible on `akm health` instead of requiring config archaeology.
const engines = {};
for (const [name, process] of Object.entries(plan.processes)) {
if (process.enabled && process.runner)
engines[name] = process.runner.engine;
}
if (plan.triageJudgment)
engines["triage.judgment"] = plan.triageJudgment.engine;
const engineList = Object.entries(engines)
.sort(([a], [b]) => a.localeCompare(b))
.map(([process, engine]) => `${process}: "${engine}"`)
.join(", ");
// #957: fail only when the strategy's model work would be a total no-op —
// every process the strategy actually enabled among the ones that call a
// model themselves (MODEL_CALLING_PROCESSES, any engine kind) ended up
// unavailable. A partial failure (some processes still have a working
// engine) stays a `warn`, matching #914's "a credential warn stays a warn"
// policy for the general per-engine probes; this is the strategy-scoped
// "is the whole run a no-op" question instead.
const modelProcessNames = [...MODEL_CALLING_PROCESSES];
const requiredModelProcessNames = modelProcessNames.filter((name) => plan.processes[name].enabled || plan.engineUnavailable.some((item) => item.process === name));
const availableModelProcessNames = modelProcessNames.filter((name) => plan.processes[name].enabled);
const allRequiredUnavailable = requiredModelProcessNames.length > 0 && availableModelProcessNames.length === 0;
const status = unavailableProcesses.length === 0 ? "pass" : allRequiredUnavailable ? "fail" : "warn";
return {
check: {
name: "active-improve-strategy",
kind: "deterministic",
status,
confidence: "high",
message: unavailableProcesses.length === 0
? `Active improve strategy "${plan.strategy.name}" has available process engines${engineList ? ` (${engineList})` : ""}.`
: allRequiredUnavailable
? `Active improve strategy "${plan.strategy.name}" cannot run any LLM-backed process; the nightly run would be a no-op: ${unavailableProcesses.join(", ")}${engineList ? ` (engines: ${engineList})` : ""}.`
: `Active improve strategy "${plan.strategy.name}" has unavailable required credentials for: ${unavailableProcesses.join(", ")}${engineList ? ` (engines: ${engineList})` : ""}.`,
evidence: {
strategy: plan.strategy.name,
unavailableProcesses,
engines,
},
},
processEngines: Object.freeze({ ...engines }),
};
}
catch (error) {
const explicitlyConfigured = config.defaults?.improveStrategy !== undefined || Object.keys(config.improve?.strategies ?? {}).length > 0;
return {
check: {
name: "active-improve-strategy",
kind: "deterministic",
status: explicitlyConfigured ? "warn" : "unknown",
confidence: "high",
message: `Active improve strategy "${strategyName}" is unavailable: ${error instanceof Error ? error.message : String(error)}`,
evidence: { strategy: strategyName, unavailableProcesses: [] },
},
processEngines: {},
};
}
}
/** Plain-`HealthCheckResult` wrapper around {@link probeActiveImproveStrategy} for existing callers/tests. */
export function runActiveImproveStrategyProbe(deps = {}) {
return probeActiveImproveStrategy(deps).check;
}
/**
* #950: the `engine-last-used` advisory. Pure projection of context computed
* once in `health.ts` (`activeImproveStrategyEngines`, `engineLastUsed`,
* `improveRunsInLookbackWindow`) — no IO here, mirrors `thinking-control`.
*
* `unknown` (not a noisy `warn`) both when no engine is bound to an enabled
* process, AND when no improve run has been recorded (started) in the
* lookback window at all — a fresh install has never been given the chance
* to use its engines.
*/
function projectEngineLastUsedCheck(processEngineMap, lastUsed, improveRunsInLookbackWindow, lookbackDays) {
const engineProcesses = new Map();
for (const [process, engine] of Object.entries(processEngineMap)) {
const processes = engineProcesses.get(engine) ?? [];
processes.push(process);
engineProcesses.set(engine, processes);
}
const engines = [...engineProcesses.keys()].sort();
if (engines.length === 0) {
return {
name: "engine-last-used",
kind: "deterministic",
status: "unknown",
confidence: "high",
message: "No engine is bound to an enabled improve process.",
evidence: { engines: [] },
};
}
if (improveRunsInLookbackWindow === 0) {
return {
name: "engine-last-used",
kind: "deterministic",
status: "unknown",
confidence: "high",
message: `No improve runs in the last ${lookbackDays} days.`,
evidence: {
engines: engines.map((engine) => ({ engine, processes: (engineProcesses.get(engine) ?? []).sort() })),
},
};
}
const entries = engines.map((engine) => ({
engine,
processes: (engineProcesses.get(engine) ?? []).sort(),
lastUsedAt: lastUsed.get(engine)?.lastUsedAt ?? null,
}));
const idle = entries.filter((entry) => entry.lastUsedAt === null);
const status = idle.length > 0 ? "warn" : "pass";
const message = idle.length > 0
? idle
.map((entry) => {
const quoted = entry.processes.map((process) => `"${process}"`).join(", ");
const noun = entry.processes.length === 1 ? "process" : "processes";
return `Engine "${entry.engine}" is bound to ${noun} ${quoted} but has not been used in the last ${lookbackDays} days.`;
})
.join(" ")
: entries
.map((entry) => {
const process = lastUsed.get(entry.engine)?.process ?? entry.processes.join(", ");
return `Engine "${entry.engine}" last used by "${process}" at ${entry.lastUsedAt}.`;
})
.join(" ");
return {
name: "engine-last-used",
kind: "deterministic",
status,
confidence: "high",
message,
evidence: { engines: entries },
};
}
/**
* Validate the immutable installed model map and optional user overlay.
* Installed corruption is a package defect (fail); a bad optional user file
* (including an `engine`-backed profile referencing a missing/broken engine,
* once resolved against real `config.engines` — #946) is operator-fixable
* configuration (warn); absence is the normal state.
*/
export function runModelMapProbe(options = {}) {
let installedText;
try {
installedText = readInstalledModelMapText(options);
mergeModelMapLayers(parseModelMapLayer(installedText, "installed models.json"));
}
catch (error) {
return {
name: "model-map-files",
kind: "deterministic",
status: "fail",
confidence: "high",
message: `Installed models.json is invalid; this AKM installation cannot resolve model aliases: ${error instanceof Error ? error.message : String(error)}`,
evidence: { installedSource: "package asset" },
};
}
// Load config OUTSIDE the models.json try/catch below: a broken
// config.json must surface as a config problem reported by the config
// checks, never get misreported as a models.json warning carrying the
// config error's own text. When config fails to load, validate the model
// map shape-only (no `engine` resolution), matching pre-#946 behavior.
let engines;
try {
engines = (options.loadConfig?.() ?? loadConfig()).engines;
}
catch {
engines = undefined;
}
try {
const loaded = loadModelMap({ ...options, installedText, engines });
return {
name: "model-map-files",
kind: "deterministic",
status: "pass",
confidence: "high",
message: loaded.userStatus === "loaded"
? `Installed defaults and user models.json at ${loaded.userPath} are valid.`
: "Installed model defaults are valid; no optional user models.json is present.",
evidence: {
installedSource: "package asset",
userPath: loaded.userPath,
userStatus: loaded.userStatus,
aliases: Object.keys(loaded.map.aliases).sort(),
},
};
}
catch (error) {
const hint = error instanceof ConfigError ? error.hint() : undefined;
return {
name: "model-map-files",
kind: "deterministic",
status: "warn",
confidence: "high",
message: `${error instanceof Error ? error.message : String(error)}${hint ? ` ${hint}` : ""}`,
evidence: {
installedSource: "package asset",
userPath: userModelMapPath(options.env),
userStatus: "invalid",
},
};
}
}
/**
* Check model selections owned by configured engines against the common model
* map. Unknown identifiers are deliberate exact-model pass-throughs; only an
* alias known somewhere in the map but absent for the selected engine warns.
*/
export function runSelectedModelAliasesProbe(deps = {}) {
const config = deps.loadConfig?.() ?? loadConfig();
const definitions = executionEngineDefinitionsFromConfig(config);
const selected = Object.entries(definitions)
.sort(([left], [right]) => left.localeCompare(right))
.flatMap(([engine, definition]) => {
const model = definition.defaults?.model;
const modelMapKey = definition.modelMapKey ?? definition.selection.platform ?? definition.selection.name;
return typeof model === "string" ? [{ engine, alias: model, modelMapKey }] : [];
});
if (selected.length === 0) {
return {
name: "selected-model-aliases",
kind: "deterministic",
status: "unknown",
confidence: "high",
message: "No configured engines select a model.",
evidence: { checked: [], missing: [] },
};
}
let modelMap;
try {
modelMap = (deps.loadModelMap ?? loadModelMap)({
env: deps.env,
installedText: deps.installedText,
engines: config.engines,
});
}
catch {
return {
name: "selected-model-aliases",
kind: "deterministic",
status: "unknown",
confidence: "high",
message: "Configured model selections could not be checked because the model map is invalid.",
evidence: { checked: [], missing: [] },
};
}
const outcomes = selected.map(({ engine, alias, modelMapKey }) => {
try {
const resolution = resolveModelMapAlias(alias, modelMapKey, modelMap.map);
if (resolution.interpretation === "alias") {
return { kind: "alias", evidence: { engine, alias, modelMapKey } };
}
return resolution.unmappedForEngine
? { kind: "missing", evidence: { engine, alias, modelMapKey } }
: { kind: "exact" };
}
catch {
return { kind: "missing", evidence: { engine, alias, modelMapKey } };
}
});
const checked = outcomes.flatMap((outcome) => (outcome.kind === "exact" ? [] : [outcome.evidence]));
const missing = outcomes.flatMap((outcome) => (outcome.kind === "missing" ? [outcome.evidence] : []));
return {
name: "selected-model-aliases",
kind: "deterministic",
status: missing.length === 0 ? "pass" : "warn",
confidence: "high",
message: missing.length === 0
? `All ${selected.length} configured model selections resolve for their selected engines.`
: `${missing.length} of ${selected.length} configured model selections ${missing.length === 1 ? "has" : "have"} no mapping for ${missing.length === 1 ? "its" : "their"} selected engine${missing.length === 1 ? "" : "s"}.`,
evidence: { checked, missing },
};
}
/**
* `state-db-migrations` when the open itself failed: the migrations still
* pending after a failed apply, from a read-only listing that never applies
* anything. A `fail` row, not a crash, is what a bundler watches for instead
* of grepping akm's error text.
*/
export function runPendingStateMigrationsCheck(stateDbPath, deps = {}) {
const pending = (deps.listPendingStateMigrations ?? listPendingStateMigrations)(stateDbPath);
if (pending.length === 0) {
return {
name: "state-db-migrations",
kind: "deterministic",
status: "pass",
confidence: "high",
message: "state.db has no pending migrations.",
evidence: { path: stateDbPath, pending: [] },
};
}
const range = pending.length > 1 ? `${pending[0]} … ${pending[pending.length - 1]}` : pending[0];
return {
name: "state-db-migrations",
kind: "deterministic",
status: "fail",
confidence: "high",
message: `${pending.length} pending state.db migration(s) (${range}); run \`akm migrate apply\`.`,
evidence: { path: stateDbPath, pending },
};
}
/**
* Hard check: state.db's ledger is current. Every open applies the pending
* migrations (copying the file aside first when one drops schema), so after
* health's own open nothing is pending; this names what that open applied, so
* an upgrade that migrated state.db is visible in the report.
*/
export function appliedStateMigrationsCheck(stateDbPath, report) {
const { applied, backupPath } = report;
const range = applied.length > 1 ? `${applied[0]} … ${applied[applied.length - 1]}` : applied[0];
return {
name: "state-db-migrations",
kind: "deterministic",
status: "pass",
confidence: "high",
message: applied.length === 0
? "state.db has no pending migrations."
: `Applied ${applied.length} pending state.db migration(s) on open (${range})` +
(backupPath ? `; the pre-migration copy is ${backupPath}.` : "."),
evidence: { path: stateDbPath, pending: [], applied, ...(backupPath ? { backupPath } : {}) },
};
}
/**
* #943: name the dominant `detail.reason` behind `task-fail-rate`'s warning
* when one reason accounts for at least half of the window's command-task
* failures — e.g. "(timeout-dominant: 9/12 command-task failures)" — so an
* operator sees the shape of the failure from data instead of grepping task
* logs. Ties break on reason name for determinism. `undefined` when there are
* no counted failures or no single reason reaches the 50% floor.
*/
export function dominantAgentFailureReason(counts) {
const entries = Object.entries(counts);
const total = entries.reduce((sum, [, count]) => sum + count, 0);
if (total === 0)
return undefined;
const [reason, count] = entries.reduce((best, entry) => entry[1] > best[1] || (entry[1] === best[1] && entry[0] < best[0]) ? entry : best);
return count / total >= 0.5 ? { reason, count, total } : undefined;
}
/**
* The ordered health-check registry. ORDER IS LOAD-BEARING: `akmHealth`
* iterates this array and appends to hardChecks/advisories in sequence, so the
* declaration order below is exactly the emission order (hard checks first in
* their array, advisories in theirs). Each `run` is a verbatim copy of the
* corresponding former inline block.
*/
export const HEALTH_CHECKS = [
{
name: "state-db-schema",
channel: "hard",
run: (ctx) => ({
name: "state-db-schema",
kind: "deterministic",
status: ctx.missingTables.length === 0 ? "pass" : "fail",
confidence: "high",
message: ctx.missingTables.length === 0
? "state.db opened and required tables are present."
: `state.db is missing required tables: ${ctx.missingTables.join(", ")}`,
evidence: { path: ctx.stateDbPath, tables: ctx.tableNames },
}),
},
{
name: "state-db-round-trip",
channel: "hard",
run: (ctx) => ({
name: "state-db-round-trip",
kind: "deterministic",
status: ctx.probe.ok ? "pass" : "fail",
confidence: "high",
message: ctx.probe.ok
? "state.db append/read round-trip succeeded."
: `state.db round-trip failed: ${ctx.probe.error}`,
evidence: { path: ctx.stateDbPath, durationMs: ctx.probe.durationMs },
}),
},
{
// R0: nothing looked at state.db's own SQLite-level integrity before
// this — the round-trip probe above only proves one row can be appended
// and read back, which stays true on a database that fails
// `PRAGMA integrity_check` elsewhere (corrupt indexes, out-of-order rowids).
// Also reports the freelist ratio (fraction of pages VACUUM could
// reclaim) so a bloated-but-uncorrupted file is visible as a warning
// rather than silence.
name: "state-db-integrity",
channel: "hard",
run: (ctx) => {
const { ok, lines, error } = ctx.stateDbIntegrity;
const { ratio: freelistRatio, error: freelistError } = ctx.stateDbFreelist;
if (!ok) {
const detail = error ?? lines.join("; ");
return {
name: "state-db-integrity",
kind: "deterministic",
status: "fail",
confidence: "high",
message: `state.db failed PRAGMA integrity_check: ${detail}. Repair: back up state.db, then run ` +
`sqlite3 -readonly state.db ".recover" | sqlite3 state.new.db, verify state.new.db passes integrity_check, stop ` +
"every akm process, then delete state.db-wal and state.db-shm before swapping state.new.db in as " +
"state.db — a leftover WAL from the OLD database is replayed onto the new one and corrupts it.",
evidence: { path: ctx.stateDbPath, lines, freelistRatio },
};
}
if (freelistError) {
return {
name: "state-db-integrity",
kind: "deterministic",
status: "fail",
confidence: "high",
message: `state.db passed PRAGMA integrity_check, but reading its freelist/page-count failed: ${freelistError}.`,
evidence: { path: ctx.stateDbPath, lines, freelistError },
};
}
const freelistWarn = freelistRatio > STATE_DB_FREELIST_WARN_RATIO;
return {
name: "state-db-integrity",
kind: "deterministic",
status: freelistWarn ? "warn" : "pass",
confidence: "high",
message: freelistWarn
? `state.db passed PRAGMA integrity_check, but ${(freelistRatio * 100).toFixed(1)}% of its pages are free (reclaimable by VACUUM).`
: "state.db passed PRAGMA integrity_check.",
evidence: {
path: ctx.stateDbPath,
freelistCount: ctx.stateDbFreelist.freelistCount,
pageCount: ctx.stateDbFreelist.pageCount,
freelistRatio,
},
};
},
},
{
name: "state-db-migrations",
channel: "hard",
run: (ctx) => appliedStateMigrationsCheck(ctx.stateDbPath, ctx.stateDbMigrations),
},
{
name: "task-history-read",
channel: "hard",
run: (ctx) => ({
name: "task-history-read",
kind: "deterministic",
status: "pass",
confidence: "high",
message: `Read ${ctx.taskRowCount} task-history row(s) since ${ctx.since}.`,
evidence: { rows: ctx.taskRowCount, since: ctx.since },
}),
},
{
name: "active-runs",
channel: "hard",
run: (ctx) => {
// Name the stuck task_ids (deduped, oldest first) so an operator knows
// WHICH tasks to investigate, not just how many rows are stale. No
// pid/liveness detection — that's out of scope here.
const named = [...ctx.stuckActiveTasks].sort((a, b) => b.ageMs - a.ageMs);
const detail = named.map((t) => `${t.taskId} (${Math.round(t.ageMs / 60000)}m)`).join(", ");
return {
name: "active-runs",
kind: "deterministic",
status: ctx.stuckActiveRuns === 0 ? "pass" : "warn",
confidence: "high",
message: ctx.stuckActiveRuns === 0
? "No active task runs exceeded the stale threshold."
: `${ctx.stuckActiveRuns} active task run(s) are older than ${Math.round(ACTIVE_RUN_WARN_MS / 60000)} minutes: ${detail}.`,
evidence: { stuckActiveRuns: ctx.stuckActiveRuns, stuckActiveTasks: ctx.stuckActiveTasks },
};
},
},
{
name: "default-engine",
channel: "hard",
run: (ctx) => ctx.engineProbes.defaultEngine,
},
{
name: "model-map-files",
channel: "hard",
run: () => runModelMapProbe(),
},
{
name: "selected-model-aliases",
channel: "hard",
run: () => runSelectedModelAliasesProbe(),
},
{
name: "default-llm-engine",
channel: "hard",
run: (ctx) => ctx.engineProbes.defaultLlmEngine,
},
{
name: "configured-engines",
channel: "hard",
run: (ctx) => ctx.engineProbes.configuredEngines,
},
{
name: "active-improve-strategy",
channel: "hard",
// #950: projects the context field instead of recomputing, so
// `engine-last-used` below can reuse the same process→engine map without
// a second `runActiveImproveStrategyProbe()` call.
run: (ctx) => ctx.activeImproveStrategy,
},
{
// C2 (13-bus-factor): the cron task-failure rate was computed and rendered
// in the HTML report but never surfaced as an advisory, so a sustained
// 15–16% fail rate stayed invisible on `akm health`. Warn at/above the SAME
// 5% threshold the html-report badge uses (see TASK_FAIL_RATE_WARN).
//
// Item 6: the aggregate can hide a single consistently-failing task inside
// a large, mostly-healthy population (e.g. 1 flaky task at 100 rows total
// reads as <5% aggregate). Also warn when the single worst task_id (with
// enough rows to be a real signal — see MIN_ROWS_FOR_WORST_TASK_FAIL_RATE)
// crosses the same threshold, naming it explicitly.
name: "task-fail-rate",
channel: "advisory",
run: (ctx) => {
const pctStr = `${(ctx.taskFailRate * 100).toFixed(1)}%`;
const thresholdPct = `${(TASK_FAIL_RATE_WARN * 100).toFixed(0)}%`;
const aggregateWarn = ctx.taskFailRate >= TASK_FAIL_RATE_WARN;
const worst = ctx.worstTaskFailRate;
const worstWarn = worst !== null && worst.rate >= TASK_FAIL_RATE_WARN;
const warn = aggregateWarn || worstWarn;
// #943: name the dominant command-task failure reason (timeout,
// non_zero_exit, spawn_failed, …) when the check is already warning, so
// "timeout-dominant" is visible without grepping task logs.
const agentFailureReasonCounts = ctx.agentFailureReasonCounts ?? {};
const dominant = warn ? dominantAgentFailureReason(agentFailureReasonCounts) : undefined;
const dominantSuffix = dominant
? ` (${dominant.reason}-dominant: ${dominant.count}/${dominant.total} command-task failures)`
: "";
let message;
if (ctx.taskRowCount === 0) {
message = `No cron tasks ran since ${ctx.since} — no task-fail-rate signal.`;
}
else if (warn) {
const parts = [];
if (aggregateWarn) {
parts.push(`aggregate ${pctStr} across ${ctx.taskRowCount} task(s) since ${ctx.since} ≥ ${thresholdPct}`);
}
if (worstWarn && worst) {
const worstPctStr = `${(worst.rate * 100).toFixed(1)}%`;
parts.push(`task "${worst.taskId}" fails ${worstPctStr} of its ${worst.rows} run(s) ≥ ${thresholdPct}`);
}
message = `Cron task fail rate warning: ${parts.join("; ")}${dominantSuffix} — inspect failed runs (ok=false) for early-exit/harness errors.`;
}
else {
message = `Cron task fail rate ${pctStr} across ${ctx.taskRowCount} task(s) since ${ctx.since} (below ${thresholdPct} threshold).`;
}
return {
name: "task-fail-rate",
kind: "deterministic",
status: warn ? "warn" : "pass",
confidence: "high",
message,
evidence: {
taskFailRate: ctx.taskFailRate,
taskRowCount: ctx.taskRowCount,
threshold: TASK_FAIL_RATE_WARN,
worstTaskFailRate: worst,
// #943: always present (even pass/empty) so scripts reading evidence
// never need to branch on check status to find the breakdown.
agentFailureReasonCounts,
},
};
},
},
{
// #914: derived from the `extract_sessions_seen` LEDGER, not
// `improve_runs.result_json` — the hook-driven `akm proposal extract
// --session-id ...` (the standard Claude Code plugin setup) never writes
// an `improve_runs` row, so the old improve_runs-only source reported
// "not active" as `pass` unconditionally on a plugin-driven machine, no
// matter how healthy or broken extraction actually was.
name: "session-extraction",
channel: "advisory",
run: (ctx) => {
const { since: ledgerSince, rows } = ctx.sessionExtractionLedger;
const days = SESSION_EXTRACTION_LEDGER_WINDOW_DAYS;
const base = {
name: "session-extraction",
kind: "heuristic",
confidence: "medium",
evidence: { ledgerSince, ledgerRows: rows },
};
const totalRows = rows.reduce((sum, row) => sum + row.count, 0);
if (totalRows === 0) {
return { ...base, status: "unknown", message: `No extraction recorded in the last ${days} days.` };
}
// `read_failed` / `exception` land in the ledger as `failed`, not `skipped`.
const nothingHarvested = rows.every((row) => row.outcome === "skipped" || row.outcome === "failed");
const knownSkipReasons = [...new Set(rows.flatMap((row) => (row.skipReason ? [row.skipReason] : [])))].sort();
const allKnownReasonsAreInfra = knownSkipReasons.every((reason) => EXTRACT_INFRASTRUCTURE_SKIP_REASONS.includes(reason));
if (nothingHarvested && allKnownReasonsAreInfra) {
const reasonClauses = knownSkipReasons.map((reason) => {
const engines = [
...new Set(rows.flatMap((row) => (row.skipReason === reason && row.engine ? [row.engine] : []))),
].sort();
return engines.length > 0 ? `${reason} (engine ${engines.map((e) => `"${e}"`).join(", ")})` : reason;
});
if (rows.some((row) => row.skipReason === null))
reasonClauses.push("unknown reason");
const verb = rows.some((row) => row.outcome === "failed") ? "skipped or failed" : "skipped";
return {
...base,
status: "warn",
message: `${totalRows} of ${totalRows} sessions in the last ${days} days ${verb}: ${reasonClauses.join(", ")}.`,
};
}
const outcomeCounts = new Map();
for (const row of rows)
outcomeCounts.set(row.outcome, (outcomeCounts.get(row.outcome) ?? 0) + row.count);
const outcomeSummary = [...outcomeCounts.entries()]
.sort(([a], [b]) => a.localeCompare(b))
.map(([outcome, count]) => `${outcome}: ${count}`)
.join(", ");
return {
...base,
status: "pass",
message: `Session extraction active in the last ${days} days (${outcomeSummary}).`,
};
},
},
{
name: "auto-accept-validation",
channel: "advisory",
run: (ctx) => {
const aa = ctx.autoAccept;
return {
name: "auto-accept-validation",
kind: "heuristic",
status: aa.validationFailed > 0 ? "warn" : "pass",
confidence: aa.promoted + aa.validationFailed > 0 ? "high" : "low",
message: aa.validationFailed > 0
? `${aa.validationFailed} auto-accept validation attempt(s) failed after passing the confidence threshold (truncated description, invalid frontmatter, etc.) — the affected proposals remain pending for manual review.`
: aa.promoted > 0
? `Auto-accept healthy: ${aa.promoted} proposal(s) promoted, 0 validation failures.`
: "Auto-accept gate did not run (disabled or no proposals above threshold).",
evidence: { promoted: aa.promoted, validationFailed: aa.validationFailed },
};
},
},
{
// #949: advisory channel, but `kind: "deterministic"` (same as the
// pre-existing task-fail-rate advisory) — a warn here DOES flip
// AkmHealthResult.status to "warn" and akm health's exit code to
// EXIT_HEALTH_WARN, because the overall-status computation ORs
// deterministic warns across both hardChecks and advisories.
name: "thinking-control",
channel: "advisory",
run: (ctx) => projectThinkingControlCheck(ctx.thinkingOffEngines, ctx.llmUsage, ctx.since),
},
{
// #950: best-effort "installed vs latest release" advisory, gated behind
// the same --probe/--no-probe flag as engine reachability. Computed once
// in health.ts (network IO), projected here like engineProbes.
name: "cli-version",
channel: "advisory",
run: (ctx) => ctx.versionDrift,
},
{
// #950: advisory channel, `kind: "deterministic"` — same
// exit-code-gating rationale as thinking-control above.
name: "engine-last-used",
channel: "advisory",
run: (ctx) => projectEngineLastUsedCheck(ctx.activeImproveStrategyEngines, ctx.engineLastUsed, ctx.improveRunsInLookbackWindow, ENGINE_LAST_USED_LOOKBACK_DAYS),
},
{
// #953: registered last — order is load-bearing (see the HEALTH_CHECKS
// doc comment above). Best-effort scheduler-binary-drift advisory, gated
// behind the same --probe/--no-probe flag as engine reachability and
// cli-version. Computed once in health.ts (process-spawn IO), projected
// here like versionDrift/engineProbes.
name: "scheduler-binary",
channel: "advisory",
run: (ctx) => ctx.schedulerBinaryDrift,
},
];