akm-cli
Version:
akm (Agent Knowledge Manager) — a portable, local-first capability library for AI agents. Discover, load, share, and improve reusable skills, scripts, workflows, and knowledge across any shell-capable coding agent, including Claude Code, OpenCode, and Cur
470 lines (469 loc) • 26.9 kB
JavaScript
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
import { getImproveProcessConfig } from "../../core/config/config.js";
import { ConfigError } from "../../core/errors.js";
import { parseEmbeddedJsonResponse } from "../../core/parse.js";
import { defaultFeedback } from "../../core/structured.js";
import { warn } from "../../core/warn.js";
import { runnerLlmConnection } from "../../integrations/agent/runner.js";
import { callStructured, dispatchFailureReason, dispatchFailureResult, } from "../../llm/structured-call.js";
import { currentLlmStage, withLlmStage } from "../../llm/usage-telemetry.js";
import { isProceduralRejection } from "../proposal/proposal-types.js";
import { createProposal, listProposalsReadOnly, proposalContentHash, recordGateDecision, } from "../proposal/repository.js";
import { resolveImproveExecution } from "./execution.js";
/** Normalize an unknown thrown value to a message. */
export function errMessage(e) {
return e instanceof Error ? e.message : String(e);
}
/** The lowering notices a stage's dispatches emitted, each once. */
export function noticeSet(forward) {
const byKey = new Map();
const add = (notices) => {
for (const notice of notices)
byKey.set(JSON.stringify(notice), notice);
forward?.(notices);
};
const list = () => Object.freeze([...byKey.values()]);
return { add, list, fields: () => (byKey.size > 0 ? { notices: list() } : {}) };
}
/**
* A stage's runner: the one the improve plan froze for it (an own
* `llmRunner` key, `null` meaning "none"), else the process engine cascade.
*/
export function stageRunner(frozen, config, profile, processName, onNotices) {
if (Object.hasOwn(frozen, "llmRunner"))
return frozen.llmRunner ?? undefined;
const resolved = resolveImproveExecution({
config,
profile,
process: getImproveProcessConfig(processName, profile),
processName,
});
if (resolved)
onNotices?.(resolved.notices);
return resolved?.runner;
}
/**
* One model call. Provider trouble (transport error, timeout, abort, a
* disabled feature) comes back as `{ ok: false }`; only a configuration
* failure throws. A reply to a call with `request.responseSchema` that the
* stage's own `parse` rejects gets one corrective retry; the last reply comes
* back either way, and the caller parses it again.
*/
export async function callStage(call) {
const reply = await callStageOnce(call);
if (!reply.ok || !call.request?.responseSchema)
return reply;
if ((call.parse ?? parseEmbeddedJsonResponse)(reply.raw) !== undefined)
return reply;
const feedback = defaultFeedback({ reason: "parse_error", errors: [] });
const retry = await callStageOnce({ ...call, prompt: `${call.prompt}\n\n${feedback}` });
// A retry that fails in transport keeps the first reply, which the caller may still accept.
return retry.ok ? retry : reply;
}
/** Timeout and abort come from the dispatch's own reason, whatever the runner's kind. */
function failureReason(err) {
const reason = dispatchFailureReason(err);
return reason === "timeout" || reason === "aborted" ? reason : "error";
}
/** A failed call: its reason and message, and the dispatch's own result when it reached a transport. */
function failedCall(err) {
const result = dispatchFailureResult(err);
return { ok: false, reason: failureReason(err), error: errMessage(err), ...(result ? { result } : {}) };
}
/**
* One dispatch with no validation, for a caller that parses and repairs the
* reply itself (reflect's repair turn, extract's own structured loop).
*/
export async function callStageOnce(call) {
const messages = [
...(call.system ? [{ role: "system", content: call.system }] : []),
...(call.history ?? []),
{ role: "user", content: call.prompt },
];
let failure;
try {
const dispatch = () => callStructured({
feature: call.feature,
...(call.gate
? { akmConfig: call.gate.config, ...(call.gate.enabled !== undefined ? { enabled: call.gate.enabled } : {}) }
: {}),
runner: call.runner,
messages,
...(call.request ? { request: call.request } : {}),
...(call.current ? { current: call.current } : {}),
...(call.onNotices ? { onNotices: call.onNotices } : {}),
parse: (r) => r ?? "",
onError: (_cls, err) => {
failure = failedCall(err);
return undefined;
},
fallback: undefined,
onFallback: (event) => {
failure ??= { ok: false, reason: event.reason, ...(event.error ? { error: event.error.message } : {}) };
},
});
// Usage is credited to the engine that serves the call, which for a gate's own judge
// (#1011) is not the stage's planned engine.
const stage = currentLlmStage();
const raw = await (stage === undefined
? dispatch()
: withLlmStage(stage, dispatch, { engine: call.runner.engine }));
return raw === undefined ? (failure ?? { ok: false, reason: "error" }) : { ok: true, raw };
}
catch (err) {
if (err instanceof ConfigError)
throw err;
return failedCall(err);
}
}
/** Attribute a stage's LLM calls to its process and planned engine (the usage report). */
const STAGE_LABELS = {
reflect: "reflect",
distill: "distill",
consolidate: "consolidate",
extract: "session-extraction",
memoryInference: "memory-inference",
validation: "validation",
};
export function attributeStage(plan, process, fn) {
return withLlmStage(STAGE_LABELS[process], fn, { engine: plan?.processes[process].runner?.engine, process });
}
/** How many prior rejected proposals are shown to the model as "don't repeat this". */
export const MAX_REJECTED_PROPOSALS = 3;
/**
* Reflexion context: the newest reviewer rejections for `ref`. Procedural
* refusals (expiry, stale target, missing asset) are not judgements on the
* content and are left out. Reads never create state.db, and an improve run's
* live connection (`eventsCtx.db`) is read through, not copied.
*/
export function rejectedProposalContext(stash, ref, ctx, eventsCtx) {
if (!ref)
return [];
const proposalsCtx = eventsCtx?.db ? { ...ctx, db: eventsCtx.db } : ctx;
return listProposalsReadOnly(stash, { ref, status: "rejected", includeArchive: true }, proposalsCtx)
.filter((p) => !isProceduralRejection(p))
.sort((a, b) => new Date(b.updatedAt ?? 0).getTime() - new Date(a.updatedAt ?? 0).getTime())
.slice(0, MAX_REJECTED_PROPOSALS)
.map((p) => ({
ref: p.ref,
reason: p.review?.reason ?? "no reason given",
// `payload.content` is populated on every row, including legacy ones.
contentPreview: p.payload.content.slice(0, 500),
}));
}
// ── Mint ─────────────────────────────────────────────────────────────────────
/**
* Create a stage's proposal. `judged` (the passing verdict) stamps a `staged`
* gate decision with the judged content's hash and the judge's scores and reason
* (the triage drain accepts it while the content still matches); `review`
* leaves it `deferred` for a human (`review_needed` in the improve ledger).
*/
export function mintProposal(stash, proposalsCtx, input, verdict = {}) {
const proposal = createProposal(stash, input, proposalsCtx);
if (verdict.review) {
return recordGateDecision(stash, proposal.id, { outcome: "deferred", ...verdict.review }, proposalsCtx) ?? proposal;
}
return verdict.judged ? stageJudgedProposal(stash, proposal, verdict.judged, proposalsCtx) : proposal;
}
/**
* Stamp a proposal the quality judge passed. Best-effort: a failed stamp only
* means the triage drain judges it again.
*/
export function stageJudgedProposal(stash, proposal, judged, proposalsCtx) {
try {
return (recordGateDecision(stash, proposal.id, {
outcome: "staged",
reason: "quality-judge",
gate: "quality-gate",
contentHash: proposalContentHash(proposal),
...(judged?.criteria ? { scores: judged.criteria } : {}),
...(judged ? { judgeReason: judged.reason } : {}),
}, proposalsCtx) ?? proposal);
}
catch (error) {
warn(`[akm] failed to record the quality-judge pass for ${proposal.id}: ${errMessage(error)}`);
return proposal;
}
}
/**
* The judge a quality gate names for itself (#1011): the gate's `engine`,
* `model`, `timeoutMs` and `llm` over the process's own settings, as
* `processes.triage.judgment` resolves over triage. `undefined` when the gate
* is off or sets none of them, so the caller keeps its own judge. The engine
* may be of any kind; config validation has required one that confines the
* model-work tool policy. Throws when they resolve to no engine at all,
* before anything is generated: the gate never falls back to another judge.
*/
export function resolveQualityGateJudge(config, profile, processName, onNotices) {
const process = profile?.processes?.[processName];
const gate = process?.qualityGate;
if (!gate || gate.enabled === false)
return undefined;
if (!["engine", "model", "timeoutMs", "llm"].some((key) => Object.hasOwn(gate, key)))
return undefined;
const resolved = resolveImproveExecution({
config,
processName: `${processName}-quality-judge`,
...(profile ? { profile } : {}),
...(process ? { process } : {}),
current: gate,
});
if (!resolved) {
throw new ConfigError(`The ${processName} quality gate's judge has no engine. Set processes.${processName}.qualityGate.engine.`, "INVALID_CONFIG_FILE");
}
onNotices?.(resolved.notices);
return resolved.runner;
}
/** Lesson judge prompt; similar existing lessons let it mark near-duplicates down. */
export function buildJudgePrompt(lessonContent, sourceContent, similarLessons) {
const lines = [
"You are evaluating a proposed lesson asset for an akm knowledge base.",
"",
"Score this lesson on each criterion from 1 (poor) to 5 (excellent):",
"1. NOVELTY: Does the lesson add information not already present in the source asset?",
"2. NON-REDUNDANCY: Is this lesson meaningfully different from what the source already says?",
"3. GROUNDING: Is the lesson about what the source asset is about? Score 1-2 only if it is about a different subject than the source; 3 if it is on the source's subject but goes beyond or corrects what the source says (it may draw on feedback you are not shown); 4-5 if the source supports it. A lesson may generalize the source's point.",
"",
"Source asset content:",
"```",
// The window distill generates from (buildDistillPrompt): grounding can reject, so the judge reads all of it.
sourceContent.slice(0, 3000),
"```",
];
if (similarLessons && similarLessons.length > 0) {
lines.push("", "Existing similar lessons (top-3 by similarity). Rate NOVELTY and NON-REDUNDANCY lower if the proposed lesson is substantially similar to any of these:");
for (const sl of similarLessons)
lines.push(`\nExisting lesson ref: ${sl.ref}`, "```", sl.content.slice(0, 500), "```");
}
lines.push("", "Proposed lesson content:", "```", lessonContent.slice(0, 1000), "```", "", 'Return ONLY valid JSON, no prose: {"scores": {"novelty": <1-5 integer>, "nonRedundancy": <1-5 integer>, "grounding": <1-5 integer>}, "reason": "<one sentence>"}');
return lines.join("\n");
}
function boundedDocument(content, maxChars = 6000) {
if (content.length <= maxChars)
return content;
const half = Math.floor((maxChars - 80) / 2);
return `${content.slice(0, half)}\n\n[... middle omitted for bounded judge context ...]\n\n${content.slice(-half)}`;
}
function buildChangedRegion(sourceContent, candidateContent) {
const source = sourceContent.split("\n");
const candidate = candidateContent.split("\n");
let prefix = 0;
while (prefix < source.length && prefix < candidate.length && source[prefix] === candidate[prefix])
prefix++;
let suffix = 0;
while (suffix < source.length - prefix &&
suffix < candidate.length - prefix &&
source[source.length - 1 - suffix] === candidate[candidate.length - 1 - suffix]) {
suffix++;
}
const removed = source.slice(prefix, source.length - suffix).join("\n");
const added = candidate.slice(prefix, candidate.length - suffix).join("\n");
return boundedDocument(`Removed or replaced:\n${removed || "(none)"}\n\nAdded or replacement:\n${added || "(none)"}`);
}
/**
* What the judge may do with tools when it runs on an agent engine: verify a
* fact the revision adds or alters, and nothing else. The plain judge's prompt
* is unchanged (its rubric is tuned and measured without this paragraph).
*/
function reflectJudgeToolRules(ref) {
const asset = ref ? `The asset is \`${ref}\`: read it with akm_show, ` : "Read an asset with akm_show ";
return `Tools: ${asset}or an asset the changed region names, only to verify a fact the revision adds or alters; the text above already shows every change. Do not search, do not read anything else, and do not use a tool to judge structure or wording. One or two reads at most. Before scoring, check two lists: (1) every statement the revision adds: find each in the asset, or as a fact the feedback states about the subject, and score QUALITY 1-2 if any is in neither; a statement is found only when the asset or the feedback says it, in any words: a new step, cause, consequence or detail that merely seems to follow is not found; feedback says what to fix and is not content, so an added statement about how the asset was used, found or verified is unsupported; (2) every fact, caveat and field of the source: find each in the revision, and score PRESERVATION 1-3 if any is missing. A read that finds nothing wrong raises no score above what these lists support. Then reply with the JSON.`;
}
/**
* The reflect judge's rubric, for the frontmatter-only revisions reflect makes, and the paragraph that says what
* drove the revision: negative feedback (any `[negative]` line) or maintenance. Tuned on the production model
* against 113 reviewed proposals (the stash's eval/judge-gate/tuning/reflect/judge-fm, rubric f07).
*/
const REFLECT_JUDGE_INTRO = "You are evaluating a proposed revision of an existing akm asset's frontmatter. The revision may change only the `description`, the `when_to_use` and the title (a level-1 heading added when the body has none); the body is unchanged.";
const REFLECT_JUDGE_NEGATIVE = "This revision answers negative feedback. It cannot change the body, so it need not resolve the feedback: judge only the fields it changes, and never fault it for a field it leaves as it was. Most negative feedback says the asset did not help with a task it was retrieved for: a retrieval miss, not a defect. It justifies changing a field only when the field claims more than the body covers (narrow it to what the body covers), or when the feedback calls the asset stale, outdated, superseded or historical (a new `when_to_use` must then name the version or date the body records). Feedback about a task the asset never claims to cover justifies no change. Repairing a missing or broken field is always needed, whatever the feedback says.";
const REFLECT_JUDGE_MAINTENANCE = "This revision is maintenance: there is no negative feedback. Only a missing or broken field needs a change; rewording a sound `description` or `when_to_use` is churn, however accurate.";
/** Judge prompt for an in-place revision. `tools` is set when the judge runs on an agent engine. */
export function buildReflectJudgePrompt(candidateContent, sourceContent, feedback, tools) {
return [
REFLECT_JUDGE_INTRO,
"",
feedback.some((line) => line.startsWith("[negative]")) ? REFLECT_JUDGE_NEGATIVE : REFLECT_JUDGE_MAINTENANCE,
"",
"Score this revision on each criterion from 1 (poor) to 5 (excellent):",
"1. NEED: Does every changed field fix a real problem? Real problems: a missing `description`, `when_to_use` or title; a broken description (a sentence split by a stray period at a line wrap, an escaped or unbalanced quote, a truncated ending, a heading fragment); and, for a revision answering negative feedback, a field that claims more than the body covers, or a stale note's `when_to_use` that does not name the version or date the body records. Replacing a stray period that splits a sentence with a comma, a word or nothing repairs a broken description, however small the change looks. Score 4-5 when every changed field fixes one. Score 1-2 when any changed field rewrites a sound field. Score NEED on the changed fields alone: leaving negative feedback about the body unresolved never lowers it.",
"2. PRESERVATION: Does the new description keep every fact the old one carried: names, identifiers, numbers, versions, paths, qualifiers and status words such as 'Proposal' or 'draft'? Are all other frontmatter fields unchanged? Score 1-3 when anything is dropped or changed.",
"3. QUALITY: Is every new value supported by the body, without inventing, over-claiming or misdescribing? Check each new value as a claim against the body; restating the body in other words is supported. Score only values the revision adds or changes: a field it leaves as it was, however stale, is never this revision's fault. Score 1-2 when a new value says something the body does not support or the opposite of what it says, keeps a truncated or garbled fragment, or offers a dated or historical note for current work: when the feedback calls the note stale, outdated, superseded or historical, or the body records the version or date it was true for, a new `when_to_use` that does not name that version or date, or a new value that calls a dated snapshot 'current', scores 1-2.",
"",
"Feedback:",
"```",
(feedback.length > 0 ? feedback.join("\n") : "No explicit feedback supplied.").slice(0, 1000),
"```",
"",
"Source asset content:",
"```",
boundedDocument(sourceContent),
"```",
"",
"Proposed revision:",
"```",
boundedDocument(candidateContent),
"```",
"",
"Changed region:",
"```",
buildChangedRegion(sourceContent, candidateContent),
"```",
"",
...(tools ? [reflectJudgeToolRules(tools.ref), ""] : []),
'Return ONLY valid JSON, no prose: {"scores": {"need": <1-5 integer>, "preservation": <1-5 integer>, "quality": <1-5 integer>}, "reason": "<one sentence>"}',
].join("\n");
}
/**
* `grounding` is scored with the other lesson criteria but left out of their
* mean: a lesson about a different subject than its source reads as novel and
* non-redundant, so they would pass it (or, in the review band, mint it as
* a pending proposal). The rubric reserves 1-2 for a different subject. A score
* of {@link UNGROUNDED_MAX_SCORE} or less is a rejection whatever the mean says
* (#999). A higher score up to {@link BORDERLINE_GROUNDING_MAX_SCORE} is only
* borderline: a lesson on its source's subject that advises beyond it has scored
* 2, and a score can move a point between runs (see `runQualityJudge`), so it
* goes to a person unless the mean alone already rejects it. A lesson that goes
* beyond or corrects its source is on its subject: distill folds feedback into
* the lesson, and the judge is never shown it. A contradiction of the source is
* the optional fidelity check's to send to a human (`judgeAndQueue` in
* distill.ts), so the rubric must not pre-empt it.
*/
const GROUNDING_CRITERION = "grounding";
const UNGROUNDED_MAX_SCORE = 1;
const BORDERLINE_GROUNDING_MAX_SCORE = 2;
const LESSON_JUDGE_CRITERIA = ["novelty", "nonRedundancy", GROUNDING_CRITERION];
const REFLECT_JUDGE_CRITERIA = ["need", "preservation", "quality"];
/**
* Read a judge response: the per-criterion shape (averaged here, `grounding`
* aside; `lowest` is the lowest score in that mean) or the older `{"score"}`
* shape. Only the expected criteria are read; any missing or out-of-range
* (1..5) value is a parse failure, extra keys are ignored.
*/
function parseJudgeResponse(raw, keys) {
const parsed = parseEmbeddedJsonResponse(raw);
if (!parsed || typeof parsed.reason !== "string")
return undefined;
const reason = parsed.reason;
const inRange = (value) => typeof value === "number" && Number.isFinite(value) && value >= 1 && value <= 5;
if (parsed.scores !== undefined) {
if (typeof parsed.scores !== "object" || parsed.scores === null || Array.isArray(parsed.scores))
return undefined;
const scores = parsed.scores;
const criteria = {};
for (const key of keys) {
const value = scores[key];
if (!inRange(value))
return undefined;
criteria[key] = value;
}
const averaged = Object.entries(criteria)
.filter(([key]) => key !== GROUNDING_CRITERION)
.map(([, value]) => value);
return {
score: averaged.reduce((a, b) => a + b, 0) / averaged.length,
lowest: Math.min(...averaged),
reason,
criteria,
};
}
return inRange(parsed.score) ? { score: parsed.score, lowest: parsed.score, reason } : undefined;
}
function judgeResponseSchema(keys) {
return {
type: "object",
required: ["scores", "reason"],
additionalProperties: false,
properties: {
scores: {
type: "object",
required: [...keys],
additionalProperties: false,
properties: Object.fromEntries(keys.map((key) => [key, { type: "integer", minimum: 1, maximum: 5 }])),
},
reason: { type: "string" },
},
};
}
/**
* The quality judge. Fails closed: no runner, an unparseable verdict or a
* provider failure never passes content. Bands: every criterion in the mean
* >= 4 passes, otherwise a mean >= 2.5 is review and a lower one reject; a
* `grounding` score of {@link UNGROUNDED_MAX_SCORE} or less rejects whatever
* the mean is, and one of {@link BORDERLINE_GROUNDING_MAX_SCORE} routes a lesson
* that would pass to review (a mean that rejects stays a rejection).
* Temperature is set to 0, which reduces run-to-run variation but does not
* remove it: on some servers (llama.cpp batching, for one) the same request can
* score a point apart, so the routing rules are chosen with that margin in mind.
*/
async function runQualityJudge(feature, config, prompt, keys, chat, options) {
const resolved = !options.runnerSelectionFrozen && !options.llmRunner
? resolveImproveExecution({ config, processName: `${feature}-judge` })
: null;
if (resolved)
options.onNotices?.(resolved.notices);
const runner = options.llmRunner ?? resolved?.runner;
if (!runner)
return { pass: false, score: -1, reason: "no engine configured — cannot judge, failing closed" };
const outcome = await callStage({
feature,
runner,
system: "Return only valid JSON. No prose.",
prompt,
request: {
// Off unless the judge's own engine enables thinking (a slower, separate judge engine, #1011).
enableThinking: runnerLlmConnection(runner)?.enableThinking === true,
temperature: 0,
responseSchema: judgeResponseSchema(keys),
...(Object.hasOwn(options, "timeoutMs") ? { timeoutMs: options.timeoutMs } : {}),
...(options.signal ? { signal: options.signal } : {}),
...(chat ? { chat } : {}),
},
parse: (raw) => parseJudgeResponse(raw, keys),
...(options.onNotices ? { onNotices: options.onNotices } : {}),
});
if (!outcome.ok) {
return { pass: false, score: -1, reason: "judge timeout/error — routed to review", reviewNeeded: true };
}
const parsed = parseJudgeResponse(outcome.raw, keys);
if (!parsed)
return { pass: false, score: -1, reason: "judge parse failed — routed to review", reviewNeeded: true };
const { score, lowest, reason, criteria } = parsed;
const grounding = criteria?.[GROUNDING_CRITERION];
if (criteria && grounding !== undefined && grounding <= UNGROUNDED_MAX_SCORE) {
return {
pass: false,
score,
reason: `Off-subject for its source (grounding ${grounding}/5): ${reason}`,
criteria,
};
}
const verdict = lowest >= 4 ? { pass: true } : score >= 2.5 ? { pass: false, reviewNeeded: true } : { pass: false };
// Borderline grounding is a person's call even when the lesson would pass; a mean that rejects stays rejected.
if (criteria &&
grounding !== undefined &&
grounding <= BORDERLINE_GROUNDING_MAX_SCORE &&
(verdict.pass || verdict.reviewNeeded)) {
return {
pass: false,
reviewNeeded: true,
score,
reason: `Borderline on grounding (${grounding}/5), routed to review: ${reason}`,
criteria,
};
}
return { ...verdict, score, reason, ...(criteria ? { criteria } : {}) };
}
/** Judge a proposed lesson (or knowledge promotion) against its source. */
export function runLessonQualityJudge(config, lessonContent, sourceContent, chat, options = {}) {
const prompt = buildJudgePrompt(lessonContent, sourceContent, options.similarLessons);
return runQualityJudge("lesson_quality_gate", config, prompt, LESSON_JUDGE_CRITERIA, chat, options);
}
/** Judge an in-place reflect revision without new-lesson novelty criteria. */
export function runReflectQualityJudge(config, candidateContent, sourceContent, feedback, chat, options = {}) {
// A judge on an agent engine gets the tool rules; the runner is the frozen one or none.
const tools = options.llmRunner && options.llmRunner.kind !== "llm" ? { ref: options.ref } : undefined;
const prompt = buildReflectJudgePrompt(candidateContent, sourceContent, feedback, tools);
return runQualityJudge("proposal_quality_gate", config, prompt, REFLECT_JUDGE_CRITERIA, chat, options);
}