akm-cli
Version:
akm (Agent Knowledge Manager) — a portable, local-first capability library for AI agents. Discover, load, share, and improve reusable skills, scripts, workflows, and knowledge across any shell-capable coding agent, including Claude Code, OpenCode, and Cur
986 lines (985 loc) • 45.3 kB
JavaScript
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
/**
* `akm consolidate` — show the model the memory pool in chunks of similar
* memories and queue a knowledge proposal for each memory it says should be
* promoted. Promotion emits a reviewable proposal and never touches the
* memory directly; accepting it later retires the source memory (O1, in
* `proposal/repository.ts`). Memories the improve ledger judged recently and
* that have not changed since are not judged again, and a memory whose
* promotion was accepted or rejected waits until its body changes. A memory
* that a neighbouring knowledge doc already covers is not promoted
* (`consolidate/coverage.ts`).
*
* Accounting invariant (the promote pass only): `processed == promoted +
* judgedNoAction + Σ(skipReasons) + failedChunkMemories`.
*
* A second pass, the pair pass (`consolidate/pair-pass.ts`, alpha.9), runs
* alongside this one and keeps its own separate counters (`pairPass` on the
* result) — it judges near-duplicate and superseding pairs across the wider
* memory tier and mints `retire` proposals; see that module's own doc
* comment.
*/
import fs from "node:fs";
import path from "node:path";
import consolidateSystemPrompt from "../../assets/prompts/consolidate-system.md" with { type: "text" };
import { detectAdapterId } from "../../core/adapter/detect-adapter.js";
import { assembleAssetFromString, serializeFrontmatter } from "../../core/asset/asset-serialize.js";
import { parseFrontmatter } from "../../core/asset/frontmatter.js";
import { conceptIdFromTypeName, displayRef, parseRefInput } from "../../core/asset/resolve-ref.js";
import { getImproveProcessConfig, loadConfig } from "../../core/config/config.js";
import { parseEmbeddedJsonResponse } from "../../core/parse.js";
import { openStateDatabase } from "../../core/state-db.js";
import { warn, warnVerbose } from "../../core/warn.js";
import { resolveWriteTarget } from "../../core/write-source.js";
import { deriveInstallations } from "../../indexer/installations.js";
import { resolveSourceEntries } from "../../indexer/search/search-source.js";
import { USAGE_EVENT_RETENTION_DAYS } from "../../indexer/usage/usage-events.js";
import { runnerLlmConnection } from "../../integrations/agent/runner.js";
import { assertRunnerCredentials } from "../../integrations/agent/runner-dispatch.js";
import { cosineSimilarity, embedBatch, resolveEmbeddingModelId } from "../../llm/embedder.js";
import { getBodyEmbeddings, upsertBodyEmbeddings } from "../../storage/repositories/embeddings-repository.js";
import { closeDatabase, openExistingDatabase, openReadonlyExistingDatabase, } from "../../storage/repositories/index-connection.js";
import { getAllEntries } from "../../storage/repositories/index-entries-repository.js";
import { listProposals, listProposalsReadOnly, proposalContent } from "../proposal/repository.js";
import { hasHotCaptureMode, hasSupersededStatus, validateProposalFrontmatter, } from "../proposal/validators/proposal-quality-validators.js";
import { buildChunkPrompt, computeSafeChunkSize, DEFAULT_CONTEXT_LENGTH_TOKENS } from "./consolidate/chunking.js";
import { openKnowledgeCoverage } from "./consolidate/coverage.js";
import { runConsolidatePairPass } from "./consolidate/pair-pass.js";
import { sanitizeMergedContent } from "./consolidate/sanitize.js";
import { contentHash } from "./content-hash.js";
import { resolveImproveStrategy, resolveProcessEnabled } from "./improve-strategies.js";
import { isContentDrivenRow, isLedgerBlocked, ledgerKey, loadLedgerSnapshot, recordLedgerAttempt } from "./ledger.js";
import { isInRetrievalScope, loadRetrievalScope } from "./retrieval-scope.js";
import { callStage, mintProposal, noticeSet, stageRunner } from "./stage.js";
function parsePlan(raw) {
const plan = parseEmbeddedJsonResponse(raw);
return plan && Array.isArray(plan.operations) ? { ...plan, operations: plan.operations } : undefined;
}
/** A plan op worth acting on. Retired advisory ops (merge/delete/contradict) are dropped, never thrown on. */
export function isValidOp(op) {
if (typeof op !== "object" || op === null)
return false;
const o = op;
return o.op === "promote" && typeof o.ref === "string" && typeof o.knowledgeRef === "string";
}
/** Reconcile the per-chunk plans: one promotion per source memory, the last chunk's wins. */
export function mergePlans(chunks) {
const byRef = new Map();
for (const chunk of chunks)
for (const op of chunk)
byRef.set(op.ref, op);
return [...byRef.values()];
}
export function isConsolidationEligibleMemoryName(name) {
return !name.endsWith(".derived");
}
/**
* A `captureMode: hot` memory (written deliberately with `akm remember`). A
* missing file is not hot; an unreadable one is treated as hot — the check is
* a protection and must not fail open.
*/
export function isHotCapturedMemory(filePath) {
if (!fs.existsSync(filePath))
return false;
try {
return hasHotCaptureMode(parseFrontmatter(fs.readFileSync(filePath, "utf8")).data);
}
catch {
return true;
}
}
/**
* Structured-output schema for a plan. Promote-only: merge/delete/contradict
* were removed in 0.9.17-alpha.1 (`e82eec811`) after running in production —
* they cost thousands of completion tokens.
*/
export const CONSOLIDATE_PLAN_JSON_SCHEMA = {
type: "object",
required: ["operations"],
additionalProperties: false,
properties: {
operations: {
type: "array",
description: "Ordered list of promote operations the planner proposes.",
items: {
type: "object",
required: ["op", "ref", "knowledgeRef", "reason"],
additionalProperties: false,
properties: {
op: { type: "string", enum: ["promote"] },
ref: { type: "string", minLength: 1 },
knowledgeRef: { type: "string", minLength: 1 },
reason: { type: "string", minLength: 1, maxLength: 200 },
description: { type: "string" },
confidence: { type: "number", minimum: 0, maximum: 1 },
},
},
},
},
};
/**
* Order memories so similar ones sit together and land in the same chunk:
* a greedy nearest-neighbour chain over description+tag embeddings (cached in
* `body_embeddings` under the text's hash). Keeps the original order without
* an embedding config, for fewer than three memories, or when embedding fails.
*/
async function clusterMemoriesBySimilarity(memories, config, stateDb, signal) {
const telemetry = { embedMs: 0, cacheHits: 0, cacheMisses: 0 };
if (memories.length < 3 || !config.embedding)
return { ordered: memories, embedTelemetry: telemetry };
const modelId = resolveEmbeddingModelId(config.embedding);
const texts = memories.map((m) => [m.description, m.tags.join(" ")].filter(Boolean).join(". ") || m.name);
const hashes = texts.map((t) => contentHash(t));
let cached = new Map();
if (stateDb) {
try {
cached = getBodyEmbeddings(stateDb, hashes, modelId);
}
catch {
cached = new Map();
}
}
const missIndices = hashes.flatMap((hash, i) => (cached.has(hash) ? [] : [i]));
telemetry.cacheHits = memories.length - missIndices.length;
telemetry.cacheMisses = missIndices.length;
const vectors = new Map(cached);
if (missIndices.length > 0) {
const embedStart = Date.now();
let missVecs;
try {
missVecs = await embedBatch(missIndices.map((i) => texts[i]), config.embedding, signal);
}
catch {
return { ordered: memories, embedTelemetry: telemetry };
}
finally {
telemetry.embedMs += Date.now() - embedStart;
}
const fresh = missIndices.flatMap((idx, pos) => {
const embedding = missVecs[pos];
return embedding ? [{ contentHash: hashes[idx], embedding, modelId }] : [];
});
for (const entry of fresh)
vectors.set(entry.contentHash, entry.embedding);
// A document the embedder skipped has no vector to cache.
if (stateDb && missVecs.length === missIndices.length) {
try {
upsertBodyEmbeddings(stateDb, fresh);
}
catch {
// Cache writes are best-effort.
}
}
}
const embeddings = hashes.map((hash) => vectors.get(hash));
if (embeddings.some((vec) => !vec))
return { ordered: memories, embedTelemetry: telemetry };
const used = new Array(memories.length).fill(false);
const ordered = [memories[0]];
used[0] = true;
let current = 0;
for (let step = 1; step < memories.length; step++) {
let bestIdx = -1;
let bestSim = -Infinity;
for (let j = 0; j < memories.length; j++) {
if (used[j])
continue;
const sim = cosineSimilarity(embeddings[current], embeddings[j]);
if (sim > bestSim) {
bestSim = sim;
bestIdx = j;
}
}
if (bestIdx === -1)
break;
ordered.push(memories[bestIdx]);
used[bestIdx] = true;
current = bestIdx;
}
return { ordered, embedTelemetry: telemetry };
}
/**
* Anti-collapse (default on, `antiCollapse.enabled: false` opts out): a small
* deterministic sample of the pool is spread through the similarity order so
* consolidation is not purely similarity-driven.
*/
function injectRandomClusterMembers(memories, profile, warnings) {
const config = getImproveProcessConfig("consolidate", profile)?.antiCollapse ?? {};
if (config.enabled === false || memories.length <= 2)
return memories;
const fraction = config.randomClusterFraction ?? 0.05;
const randomCount = Math.max(1, Math.floor(memories.length * fraction));
const sample = [...memories]
.sort((a, b) => contentHash(a.name).localeCompare(contentHash(b.name)))
.slice(0, randomCount);
const sampled = new Set(sample.map((m) => m.name));
const interval = Math.max(2, Math.floor(memories.length / randomCount));
const out = [];
let next = 0;
for (let i = 0; i < memories.length; i++) {
const m = memories[i];
if (m && !sampled.has(m.name))
out.push(m);
if (i > 0 && i % interval === 0 && next < sample.length)
out.push(sample[next++]);
}
while (next < sample.length)
out.push(sample[next++]);
warnings.push(`Anti-collapse: injected ${randomCount} random (non-similarity-driven) cluster member(s) into consolidation pool (fraction=${fraction}).`);
return out;
}
/** Body hashes of pending consolidate proposals, so the prompt can mark memories already queued. */
function loadPendingConsolidateProposalHashes(stashDir, proposalsCtx) {
const hashes = new Set();
try {
for (const p of listProposalsReadOnly(stashDir, { status: "pending" }, proposalsCtx)) {
if (p.source !== "consolidate")
continue;
try {
hashes.add(contentHash(proposalContent(p), "body"));
}
catch {
// A malformed payload cannot dedup anyway.
}
}
}
catch {
// Annotate nothing; the model still proposes.
}
return hashes;
}
/**
* Body hashes of the live knowledge assets, read from disk (the index may lag
* a just-written asset), so an accepted promotion is not proposed again.
*/
export function loadExistingKnowledgeBodyHashes(targetRoot) {
const hashes = new Set();
const visit = (dir) => {
let entries;
try {
entries = fs.readdirSync(dir, { withFileTypes: true });
}
catch {
return;
}
for (const entry of entries) {
const entryPath = path.join(dir, entry.name);
if (entry.isDirectory())
visit(entryPath);
else if (entry.isFile() && entry.name.endsWith(".md")) {
try {
hashes.add(contentHash(fs.readFileSync(entryPath, "utf8"), "body"));
}
catch {
// An unreadable asset is no duplicate evidence.
}
}
}
};
visit(path.join(targetRoot, "knowledge"));
return hashes;
}
/** A provenance ref in its canonical display spelling. */
function canonicalXref(ref) {
try {
const p = parseRefInput(ref);
return displayRef({ type: p.type, name: p.name, bundleId: p.origin });
}
catch {
return ref;
}
}
export function makeConsolidateResult(overrides) {
return {
schemaVersion: 1,
ok: true,
shape: "consolidate-result",
dryRun: false,
previewOnly: false,
processed: 0,
merged: 0,
deleted: 0,
promoted: [],
contradicted: 0,
warnings: [],
...overrides,
};
}
function resolveConsolidationWriteTarget(opts, config) {
if (opts.writeTarget) {
const root = path.resolve(opts.writeTarget.source.path);
return {
...opts.writeTarget,
source: {
...opts.writeTarget.source,
path: root,
adapterId: opts.writeTarget.source.adapterId ?? detectAdapterId(root),
},
};
}
if (!opts.target && opts.stashDir) {
const root = path.resolve(opts.stashDir);
return {
source: { kind: "filesystem", name: "stash", path: root, adapterId: detectAdapterId(root) },
config: { type: "filesystem", name: "stash", path: root, writable: true },
};
}
const target = resolveWriteTarget(config, opts.target);
return { ...target, source: { ...target.source, path: path.resolve(target.source.path) } };
}
export async function akmConsolidate(opts = {}) {
const startMs = Date.now();
const config = opts.config ?? loadConfig();
const writeTarget = resolveConsolidationWriteTarget(opts, config);
const profile = opts.improveProfile ?? resolveImproveStrategy(undefined, config).config;
const stashDir = writeTarget.source.path;
const notices = noticeSet(opts.onNotices);
const enabled = resolveProcessEnabled("consolidate", profile);
const runner = enabled ? stageRunner(opts, config, profile, "consolidate", notices.add) : undefined;
opts = {
...opts,
target: writeTarget.source.name,
writeTarget,
improveProfile: profile,
onNotices: notices.add,
sourceRun: opts.sourceRun ?? `consolidate-${startMs}`,
// Every later reader sees this one runner snapshot.
llmRunner: runner ?? null,
};
if (!enabled) {
const target = opts.target ?? stashDir;
return {
...makeConsolidateResult({ dryRun: opts.dryRun ?? false, target, durationMs: Date.now() - startMs }),
...notices.fields(),
};
}
// One state.db handle for the embedding cache; unavailable means no cache.
let stateDb;
if (config.embedding) {
try {
stateDb = openStateDatabase();
}
catch {
stateDb = undefined;
}
}
try {
return { ...(await consolidate(opts, config, stashDir, startMs, stateDb)), ...notices.fields() };
}
finally {
stateDb?.close();
}
}
function pushSkipReason(acc, op, ref, reason) {
if (acc.judgedNoActionRefs.delete(ref))
acc.judgedNoAction--;
const existing = acc.skipReasonByRef.get(ref);
if (existing) {
// One entry per ref keeps the invariant; the extra reason is kept for observability.
existing.skips.push({ op, reason });
return;
}
const entry = { ref, skips: [{ op, reason }] };
acc.skipReasonByRef.set(ref, entry);
acc.skipReasons.push(entry);
}
function resolveConsolidationSourceOwner(opts, stashDir) {
const targetRoot = path.resolve(opts.writeTarget?.source.path ?? stashDir);
try {
const sources = resolveSourceEntries(stashDir, opts.config);
const targetIndex = sources.findIndex((source) => path.resolve(source.path) === targetRoot);
const target = deriveInstallations(sources)[targetIndex];
if (!target)
return undefined;
return {
bundleId: target.id,
sourceRoot: targetRoot,
excludedSourceRoots: new Set(sources
.filter((_, index) => index !== targetIndex)
.map((source) => path.resolve(source.path))
.filter((sourceRoot) => sourceRoot.startsWith(`${targetRoot}${path.sep}`))),
};
}
catch {
return undefined;
}
}
const mtimeMsOf = (memory) => {
try {
return fs.statSync(memory.filePath).mtimeMs;
}
catch {
return 0;
}
};
/**
* The exact pool the live pass consumes, with no embedding, model call or
* write: on-disk eligible memories, minus those the ledger holds, narrowed
* incrementally, minus bodies already in `knowledge/`, capped to `limit`
* (oldest-modified first). Shared by preview and execution.
*/
export function inspectConsolidationPool(opts, stashDir, warnings, existingKnowledgeBodyHashes = new Set(), access) {
const readOnly = access?.readOnly === true;
let memories = loadMemoriesForSource(resolveConsolidationSourceOwner(opts, stashDir), warnings, readOnly);
const staleCount = memories.filter((memory) => !fs.existsSync(memory.filePath)).length;
if (staleCount > 0) {
warnings.push(`Pre-flight: filtered ${staleCount} stale DB entr${staleCount === 1 ? "y" : "ies"} (file absent on disk) from memory pool before chunking.`);
}
memories = memories.filter((memory) => fs.existsSync(memory.filePath));
const poolSize = memories.length;
// A memory judged within its revisit window comes back once it is edited. A
// promotion that was accepted or rejected holds its memory until the body
// changes, however long that takes (#998).
const ledger = loadLedgerSnapshot({ proposalsCtx: opts.proposalsCtx, readOnly }, stashDir, ["consolidate"]);
if (ledger.size > 0) {
const nowIso = new Date().toISOString();
memories = memories.filter((memory) => {
const row = ledger.get(ledgerKey("consolidate", conceptIdFromTypeName("memory", memory.name)));
if (!row)
return true;
if (isContentDrivenRow(row)) {
let bodyHash;
try {
bodyHash = contentHash(fs.readFileSync(memory.filePath, "utf8"), "body");
}
catch {
bodyHash = undefined;
}
return row.contentHash !== bodyHash;
}
let changedAt;
try {
changedAt = fs.statSync(memory.filePath).mtime.toISOString();
}
catch {
changedAt = undefined;
}
return !isLedgerBlocked(row, nowIso, changedAt);
});
}
const judgedUnchanged = poolSize - memories.length;
// Only what retrieval returned or new material improve never processed (#986).
const retrievalScope = loadRetrievalScope({ proposalsCtx: opts.proposalsCtx, readOnly }, stashDir);
const beforeScope = memories.length;
memories = memories.filter((memory) => isInRetrievalScope(retrievalScope, conceptIdFromTypeName("memory", memory.name), memory.filePath));
const outsideRetrievalScope = beforeScope - memories.length;
const dedupPoolSize = memories.length;
if (opts.limit === undefined && memories.length > 150) {
warnings.push(`Consolidation: pool has ${memories.length} memories and no limit is set. Consider adding a limit to your consolidate config to prevent timeouts on slow LLM endpoints.`);
}
// Before the limit, so the cap picks from memories the run can act on.
let prefilteredAlreadyPromoted = 0;
if (existingKnowledgeBodyHashes.size > 0) {
memories = memories.filter((memory) => {
let raw;
try {
raw = fs.readFileSync(memory.filePath, "utf8");
}
catch {
return true;
}
const duplicate = existingKnowledgeBodyHashes.has(contentHash(raw, "body"));
if (duplicate)
prefilteredAlreadyPromoted++;
return !duplicate;
});
}
if (opts.limit !== undefined && memories.length > opts.limit) {
const mtimes = new Map(memories.map((memory) => [memory.filePath, mtimeMsOf(memory)]));
memories = [...memories].sort((a, b) => (mtimes.get(a.filePath) ?? 0) - (mtimes.get(b.filePath) ?? 0));
warnings.push(`Consolidation: pool capped at ${opts.limit} of ${memories.length} memories (limit option, oldest-modified first).`);
memories = memories.slice(0, opts.limit);
}
return {
poolSize,
candidatePoolSize: memories.length,
dedupPoolSize,
memories,
prefilteredAlreadyPromoted,
judgedUnchanged,
outsideRetrievalScope,
};
}
const ABORT_MIN_CHUNKS = 4;
const ABORT_FAILURE_RATE = 0.5;
/**
* The chunk loop: stop cleanly on the budget signal, abort once ≥50% of at
* least 4 chunks failed (the model is likely down), skip an all-hot chunk
* without a call (the only thing the model could do with it is refused), and
* count every memory into exactly one accounting bucket.
*/
async function judgeConsolidationChunks(args) {
const { chunks, opts, config, warnings, acc } = args;
const llmRunner = opts.llmRunner ?? undefined;
const memRef = (m) => conceptIdFromTypeName("memory", m.name);
const failChunk = (message, chunk) => {
warn(message);
warnings.push(message);
acc.totalChunksFailed++;
acc.failedChunkMemories += chunk.length;
};
const skipRemaining = (from) => {
for (let i = from; i < chunks.length; i++)
acc.failedChunkMemories += chunks[i].length;
};
const planned = [];
let processed = 0;
for (let chunkIdx = 0; chunkIdx < chunks.length; chunkIdx++) {
const label = `chunk ${chunkIdx + 1}`;
if (opts.signal?.aborted) {
const msg = `[consolidate] budget signal aborted before chunk ${chunkIdx + 1}/${chunks.length}; ${chunks.length - chunkIdx} chunk(s) not processed (partial_timeout — work done so far committed).`;
warn(msg);
warnings.push(msg);
skipRemaining(chunkIdx);
break;
}
if (processed >= ABORT_MIN_CHUNKS) {
const failureRate = acc.totalChunksFailed / processed;
if (failureRate >= ABORT_FAILURE_RATE) {
const msg = `Consolidation aborted — failure rate ${(failureRate * 100).toFixed(0)}% over ${processed} chunks (>= ${ABORT_FAILURE_RATE * 100}% threshold). LLM may be unavailable. ${chunks.length - chunkIdx} chunk(s) skipped.`;
warn(msg);
warnings.push(msg);
skipRemaining(chunkIdx);
break;
}
}
const chunk = chunks[chunkIdx];
if (chunk.length > 0 && chunk.every((m) => isHotCapturedMemory(m.filePath))) {
for (const m of chunk) {
acc.judgedNoActionRefs.add(memRef(m));
acc.judgedRefs.add(memRef(m));
}
acc.judgedNoAction += chunk.length;
warn(`[consolidate] chunk ${chunkIdx + 1}/${chunks.length}: all ${chunk.length} memories are captureMode: hot — skipping LLM (judged no-action).`);
continue;
}
warn(`[consolidate] chunk ${chunkIdx + 1}/${chunks.length} (${chunk.length} memories) …`);
processed++;
if (!llmRunner) {
failChunk("No LLM configured for consolidation", chunk);
continue;
}
// The transport already retries once; a failed chunk is not retried here.
const outcome = await callStage({
feature: "memory_consolidation",
runner: llmRunner,
system: consolidateSystemPrompt,
prompt: buildChunkPrompt(args.sourceName, chunk, chunkIdx, chunks.length, args.bodyTruncation, args.pendingProposalBodyHashes),
gate: { config, enabled: true },
request: {
responseSchema: CONSOLIDATE_PLAN_JSON_SCHEMA,
enableThinking: false,
...(Object.hasOwn(llmRunner, "timeoutMs") ? { timeoutMs: llmRunner.timeoutMs } : {}),
signal: opts.signal,
},
parse: parsePlan,
...(opts.onNotices ? { onNotices: opts.onNotices } : {}),
});
if (!outcome.ok) {
failChunk(outcome.reason === "error" && outcome.error ? outcome.error : `${label} failed`, chunk);
continue;
}
warnVerbose(`[akm:consolidate] ${label} raw response (first 500 chars): ${outcome.raw.slice(0, 500)}`);
const parsed = parsePlan(outcome.raw);
if (!parsed) {
const hint = outcome.raw.trim() === "" ? " (empty response — if using a thinking model, disable thinking mode)" : "";
const msg = `Chunk ${chunkIdx + 1}: invalid plan from AI — skipping.${hint}`;
warn(msg);
warnings.push(msg);
acc.totalChunksFailed++;
acc.failedChunkMemories += chunk.length;
continue;
}
const ops = [];
for (const op of parsed.operations) {
if (isValidOp(op))
ops.push(op);
else
warnings.push(`Chunk ${chunkIdx + 1}: skipping invalid operation: ${JSON.stringify(op)}`);
}
for (const w of Array.isArray(parsed.warnings) ? parsed.warnings : [])
if (typeof w === "string")
warnings.push(w);
// Memories the model saw but proposed nothing for.
const targeted = new Set(ops.map((op) => op.ref));
for (const m of chunk) {
acc.judgedRefs.add(memRef(m));
if (targeted.has(memRef(m)))
continue;
acc.judgedNoAction++;
acc.judgedNoActionRefs.add(memRef(m));
}
planned.push(ops);
}
return planned;
}
/**
* The model's plan for the narrowed pool: chunk size from the context window,
* an up-front cap when the remaining budget cannot cover every chunk (oldest
* first, the rest deferred), similarity clustering, anti-collapse, then the
* chunk loop.
*/
async function planConsolidation(opts, config, stashDir, memories, warnings, stateDb, acc) {
const llmRunner = opts.llmRunner ?? undefined;
// 500 body chars per memory keep the judgement useful; chunk size varies instead.
const bodyTruncation = 500;
const chunkSize = computeSafeChunkSize((llmRunner && runnerLlmConnection(llmRunner)?.contextLength) ?? DEFAULT_CONTEXT_LENGTH_TOKENS, bodyTruncation, opts.maxChunkSize);
const sourceName = opts.target ?? stashDir;
let budgeted = memories;
const budgetMs = opts.signal?.remainingBudgetMs;
if (opts.signal && budgetMs !== undefined) {
const safeChunks = Math.max(0, Math.floor((Math.max(0, budgetMs) / 1000 / (opts.p90ChunkSecondsDefault ?? 30)) * 0.6));
if (safeChunks * chunkSize < memories.length) {
budgeted = [...memories]
.sort((a, b) => mtimeMsOf(a) - mtimeMsOf(b) || a.name.localeCompare(b.name))
.slice(0, safeChunks * chunkSize);
const msg = `[consolidate] cold-start budget: reducing pool from ${memories.length} to ${budgeted.length} memories (${safeChunks} safe chunks; remainder deferred).`;
warn(msg);
warnings.push(msg);
}
}
const slice = (list) => Array.from({ length: Math.ceil(list.length / chunkSize) }, (_, i) => list.slice(i * chunkSize, (i + 1) * chunkSize));
const willDispatch = slice(budgeted).some((chunk) => chunk.length > 0 && !chunk.every((memory) => isHotCapturedMemory(memory.filePath)));
if (llmRunner && willDispatch)
assertRunnerCredentials(llmRunner);
const { ordered, embedTelemetry } = await clusterMemoriesBySimilarity(budgeted, config, stateDb, opts.signal);
const chunks = slice(injectRandomClusterMembers(ordered, opts.improveProfile, warnings));
const pendingProposalBodyHashes = loadPendingConsolidateProposalHashes(stashDir, opts.proposalsCtx);
warn(`[consolidate] ${budgeted.length} memories / ${chunks.length} chunk(s) / chunk_size=${chunkSize}` +
` / pending-proposal hashes: ${pendingProposalBodyHashes.size}`);
const planned = await judgeConsolidationChunks({
chunks,
opts,
config,
sourceName,
bodyTruncation,
pendingProposalBodyHashes,
warnings,
acc,
});
return {
allOps: mergePlans(planned),
totalChunks: chunks.length,
llmPoolSize: budgeted.length,
deferredMemories: memories.length - budgeted.length,
embedTelemetry,
sourceName,
};
}
async function consolidate(opts, config, stashDir, startMs, stateDb) {
const warnings = [];
const existingKnowledgeBodyHashes = opts.existingKnowledgeBodyHashes ?? loadExistingKnowledgeBodyHashes(stashDir);
const pool = inspectConsolidationPool(opts, stashDir, warnings, existingKnowledgeBodyHashes);
const { memories, prefilteredAlreadyPromoted } = pool;
const plural = (n) => `memor${n === 1 ? "y" : "ies"}`;
if (pool.judgedUnchanged > 0) {
warnings.push(`Consolidation: skipped ${pool.judgedUnchanged} ${plural(pool.judgedUnchanged)} judged within the revisit window, or already promoted or rejected, and unchanged since.`);
}
if (pool.outsideRetrievalScope > 0) {
warnings.push(`Consolidation: skipped ${pool.outsideRetrievalScope} ${plural(pool.outsideRetrievalScope)} already judged that retrieval has not returned in the last ${USAGE_EVENT_RETENTION_DAYS} days.`);
}
if (prefilteredAlreadyPromoted > 0) {
warnings.push(`Consolidation: pre-filtered ${prefilteredAlreadyPromoted} ${plural(prefilteredAlreadyPromoted)} whose body already exists verbatim in knowledge/ before chunking.`);
}
const target = opts.target ?? stashDir;
// The pair pass (alpha.9) has its own initiator/candidate selection (it
// sees .derived memories, flat knowledge and lessons, not just the
// promote pool above), so it runs regardless of whether the promote pool
// is empty — every return path below carries its result.
const bundleId = resolveConsolidationSourceOwner(opts, stashDir)?.bundleId;
const pairPass = await runConsolidatePairPass(opts, config, stashDir, bundleId, warnings);
if (memories.length === 0) {
return makeConsolidateResult({
dryRun: opts.dryRun ?? false,
target,
warnings,
durationMs: Date.now() - startMs,
prefilteredAlreadyPromoted,
pairPass,
});
}
const acc = {
judgedNoAction: 0,
failedChunkMemories: 0,
totalChunksFailed: 0,
skipReasons: [],
skipReasonByRef: new Map(),
judgedNoActionRefs: new Set(),
judgedRefs: new Set(),
};
const plan = await planConsolidation(opts, config, stashDir, memories, warnings, stateDb, acc);
// Evaluated at return time: a promotion skip can move a ref out of judgedNoAction.
const summary = () => ({
target: plan.sourceName,
processed: plan.llmPoolSize,
failedChunks: acc.totalChunksFailed,
totalChunks: plan.totalChunks,
judgedNoAction: acc.judgedNoAction,
skipReasons: acc.skipReasons,
mergedSecondaries: 0,
failedChunkMemories: acc.failedChunkMemories,
deferredMemories: plan.deferredMemories,
planned: plan.allOps,
warnings,
prefilteredAlreadyPromoted,
durationMs: Date.now() - startMs,
});
if (opts.dryRun)
return makeConsolidateResult({ ...summary(), dryRun: true, previewOnly: true, pairPass });
warn(`[consolidate] plan: ${plan.allOps.length} operation(s)`);
const ctx = {
config,
stashDir,
sourceRun: opts.sourceRun ?? `consolidate-${startMs}`,
proposalsCtx: opts.proposalsCtx,
target: opts.writeTarget,
memoryByRef: new Map(memories.map((memory) => [conceptIdFromTypeName("memory", memory.name), memory])),
promoted: [],
promotedSourceRefs: new Set(),
existingKnowledgeBodyHashes,
promotionFailures: { count: 0 },
warnings,
pushSkipReason: (op, ref, reason) => pushSkipReason(acc, op, ref, reason),
};
const coverage = plan.allOps.length > 0 ? openKnowledgeCoverage(bundleId) : undefined;
if (coverage)
ctx.coveringKnowledge = coverage.find;
try {
for (const op of plan.allOps)
await emitPromotionProposal(op, ctx);
}
finally {
coverage?.close();
}
// Every other judged memory waits out its revisit window (or its next edit);
// a promotion that failed to persist is retried next run.
recordLedgerAttempt({ proposalsCtx: opts.proposalsCtx }, [...acc.judgedRefs]
.filter((ref) => !ctx.promotedSourceRefs.has(ref) &&
!acc.skipReasonByRef.get(ref)?.skips.some((skip) => skip.reason === "promote_create_failed"))
.map((ref) => ({ stashDir, ref, source: "consolidate", outcome: "judged_no_action" })));
return makeConsolidateResult({
...summary(),
promoted: ctx.promoted,
failedPromotions: ctx.promotionFailures.count,
pairPass,
perfTelemetry: {
dedupPoolSize: pool.dedupPoolSize,
llmPoolSize: plan.llmPoolSize,
embedMs: plan.embedTelemetry.embedMs,
embedCacheHits: plan.embedTelemetry.cacheHits,
embedCacheMisses: plan.embedTelemetry.cacheMisses,
},
});
}
/** The conceptId a ref maps to, or undefined for an invalid ref. */
function conceptIdForRef(ref) {
try {
const p = parseRefInput(ref);
return conceptIdFromTypeName(p.type, p.name);
}
catch {
return undefined;
}
}
/** A slug with dates, counters and word order folded away, for spotting variants. */
function normalizeSlugForDedup(ref) {
const monthRe = /(?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)/i;
return parseRefInput(ref)
.name.toLowerCase()
.split("-")
.filter((tok) => tok.length > 0 && !/^\d+$/.test(tok) && !monthRe.test(tok))
.sort()
.join("-");
}
const PROMOTE_BODY_MIN_CHARS = 100;
/**
* Queue one promotion as a proposal. Refused (with a skip reason) when the
* memory is unknown, already promoted this run, already pending or present
* as knowledge (by concept, body hash or slug variant), already covered by a
* neighbouring knowledge doc (`coverage.ts`), unreadable, fails
* sanitization, is superseded, has a body too small to be knowledge, or has
* no valid description.
* @internal Exported for promotion-path integration tests.
*/
export async function emitPromotionProposal(op, ctx) {
const { stashDir, target, warnings, pushSkipReason } = ctx;
const entry = ctx.memoryByRef.get(op.ref);
if (!entry) {
// A phantom ref was never counted as processed, so it gets no skip reason.
warnings.push(`Promote: ${op.ref} not found in loaded memories — skipping.`);
return;
}
const skip = (reason, message) => {
warnings.push(message);
pushSkipReason("promote", op.ref, reason);
};
if (ctx.promotedSourceRefs.has(op.ref)) {
return skip("promote_already_promoted_this_run", `Skipping promote: ${op.ref} already promoted in this run`);
}
const slug = (op.knowledgeRef.split("/").filter(Boolean).at(-1) ??
entry.name.split("/").filter(Boolean).at(-1) ??
"promoted-memory")
.replace(/[^a-z0-9-]/gi, "-")
.replace(/-+/g, "-")
.replace(/^-|-$/g, "")
.toLowerCase();
const knowledgeRef = conceptIdFromTypeName("knowledge", slug);
const parsedKnowledgeRef = parseRefInput(knowledgeRef);
if (knowledgeRef !== op.knowledgeRef)
warnings.push(`Normalized generated ref "${op.knowledgeRef}" → "${knowledgeRef}"`);
const pending = listProposals(stashDir, { status: "pending" });
const wantConcept = conceptIdForRef(knowledgeRef);
if (wantConcept !== undefined && pending.some((p) => conceptIdForRef(p.ref) === wantConcept)) {
return skip("promote_pending_proposal_exists", `Skipping promote: pending proposal already exists for ${knowledgeRef}`);
}
if (fs.existsSync(path.join(target.source.path, "knowledge", `${parsedKnowledgeRef.name}.md`))) {
return skip("promote_already_exists", `Skipping promote: ${knowledgeRef} already exists in source`);
}
let memoryContent;
try {
memoryContent = fs.readFileSync(entry.filePath, "utf8");
}
catch (e) {
return skip("promote_read_failed", `Promote: could not read ${op.ref}: ${String(e)}`);
}
// O1 hash nit: the RAW body, before sanitization — accept re-reads the
// source with a plain fs.readFileSync and never re-sanitizes, so hashing
// anything else here would compare two different representations of the
// same unedited memory and report a false "changed since mint" (measured:
// 3 of 767 real memories sanitize to different bytes than their raw body).
const sourceRawBodyHash = contentHash(memoryContent, "body");
const sanitized = sanitizeMergedContent(memoryContent);
if (!sanitized.ok) {
return skip("promote_sanitization_failed", `Promote: rejected ${op.ref} — source memory failed sanitization (${sanitized.reason}).`);
}
memoryContent = sanitized.result.content;
if (hasSupersededStatus(sanitized.result.frontmatter)) {
return skip("promote_superseded", `Promote: refused for ${op.ref} → ${knowledgeRef} — source memory has status:superseded; superseded memories are not promotable knowledge.`);
}
const parsedMemory = parseFrontmatter(memoryContent);
const sourceBody = parsedMemory.content.trim();
if (sourceBody.length < PROMOTE_BODY_MIN_CHARS) {
return skip("promote_source_too_small", `Promote: rejected ${op.ref} → ${knowledgeRef} — source memory body is too small (${sourceBody.length} chars; need ≥${PROMOTE_BODY_MIN_CHARS}) to make useful knowledge.`);
}
// The body is the load-bearing content: twins that differ only in
// bookkeeping frontmatter, or an earlier run's differently-slugged proposal,
// are the same promotion.
const bodyHash = contentHash(memoryContent, "body");
if (ctx.existingKnowledgeBodyHashes.has(bodyHash)) {
return skip("dedup_existing_knowledge", `Skipping promote: identical body already exists in knowledge; skipping duplicate for ${op.ref} → ${knowledgeRef}`);
}
const pendingConsolidate = listProposals(stashDir, { status: "pending" }).filter((p) => p.source === "consolidate");
const sameBody = pendingConsolidate.find((p) => contentHash(proposalContent(p), "body") === bodyHash);
if (sameBody) {
return skip("dedup_pending_proposal", `Skipping promote: identical body already pending as proposal ${sameBody.id} (ref: ${sameBody.ref}); skipping duplicate for ${op.ref} → ${knowledgeRef}`);
}
// #998: the copies the two exact checks above cannot see — an earlier
// promotion that was edited or re-slugged, a doc that quotes the memory.
const covering = ctx.coveringKnowledge?.(entry.filePath, sourceBody);
if (covering) {
return skip("dedup_covered_by_knowledge", `Skipping promote: ${op.ref} → ${knowledgeRef} is already covered by ${covering.ref} (${Math.round(covering.containment * 100)}% of its text appears there).`);
}
try {
const description = (typeof op.description === "string" && op.description.trim()
? op.description.trim()
: parsedMemory.data?.description?.trim()) ?? "";
const fmCheck = validateProposalFrontmatter({ description });
if (!fmCheck.ok) {
return skip("promote_invalid_frontmatter", `Promote: rejected ${op.ref} → ${knowledgeRef} — ${fmCheck.reason}.`);
}
// The description goes into the body frontmatter, which accept-time validation reads.
const xrefs = Array.isArray(parsedMemory.data?.xrefs) ? parsedMemory.data.xrefs.map(String) : [];
const mergedFrontmatter = {
...(parsedMemory.data ?? {}),
description,
xrefs: [...new Set([...xrefs, op.ref].map(canonicalXref))],
};
const normalized = normalizeSlugForDedup(knowledgeRef);
const variant = pendingConsolidate.find((p) => normalizeSlugForDedup(p.ref) === normalized);
if (variant) {
return skip("promote_dedup_window", `Promote: skipped ${op.ref} → ${knowledgeRef} — slug-variant of pending proposal ${variant.id} (${variant.ref}).`);
}
const proposal = mintProposal(stashDir, ctx.proposalsCtx, {
ref: knowledgeRef,
target: { source: target.source.name, root: target.source.path },
source: "consolidate",
sourceRun: ctx.sourceRun,
payload: {
content: assembleAssetFromString(serializeFrontmatter(mergedFrontmatter), parsedMemory.content),
frontmatter: { description, xrefs: [canonicalXref(op.ref)] },
},
...(typeof op.confidence === "number" ? { confidence: op.confidence } : {}),
// The ledger keys the attempt by the source memory.
attemptedRefs: [op.ref],
// O1 (alpha.9): on accept, promoteProposal retires this source memory
// (and its .derived twin) so promotion no longer leaves a duplicate.
promotionSource: op.ref,
// B3: recorded so accept can refuse to archive a source that was
// edited after this promotion was queued — the RAW body hash (see
// sourceRawBodyHash's own comment above), not `bodyHash`, which is the
// sanitized-for-knowledge representation accept never re-derives.
promotionSourceHash: sourceRawBodyHash,
});
ctx.promoted.push(proposal.id);
ctx.promotedSourceRefs.add(op.ref);
}
catch (e) {
ctx.promotionFailures.count++;
skip("promote_create_failed", `Promote: createProposal failed for ${op.ref}: ${String(e)}`);
}
}
/** The target bundle's eligible memories from the index, else walked from disk. */
function loadMemoriesForSource(source, warnings, readOnly) {
let memories = [];
let db;
try {
db = readOnly ? openReadonlyExistingDatabase(undefined, { isolatedSnapshot: true }) : openExistingDatabase();
if (!db)
throw new Error("index unavailable");
memories = getAllEntries(db, "memory")
.filter((e) => source !== undefined && e.bundleId === source.bundleId)
.filter((e) => isConsolidationEligibleMemoryName(e.entry.name) && fs.existsSync(e.filePath))
.map((e) => ({
name: e.entry.name,
filePath: e.filePath,
description: e.entry.description ?? "",
tags: e.entry.tags ?? [],
stashDir: source?.sourceRoot ?? "",
}));
}
catch {
memories = [];
}
finally {
if (db)
closeDatabase(db);
}
if (memories.length > 0 || !source)
return memories;
const memoriesDir = path.join(source.sourceRoot, "memories");
if (fs.existsSync(memoriesDir)) {
const pending = [memoriesDir];
while (pending.length > 0) {
const current = pending.pop();
for (const entry of fs.readdirSync(current, { withFileTypes: true })) {
const filePath = path.join(current, entry.name);
if (entry.isDirectory()) {
if (!source.excludedSourceRoots.has(path.resolve(filePath)))
pending.push(filePath);
continue;
}
if (!entry.isFile() || !entry.name.endsWith(".md"))
continue;
const name = path.relative(memoriesDir, filePath).replace(/\.md$/, "").split(path.sep).join("/");
if (isConsolidationEligibleMemoryName(name)) {
memories.push({ name, filePath, description: "", tags: [], stashDir: source.sourceRoot });
}
}
}
}
if (memories.length > 0)
warnings.push("DB not found or empty — loaded memories directly from filesystem.");
return memories;
}