akm-cli
Version:
akm (Agent Knowledge Manager) — a portable, local-first capability library for AI agents. Discover, load, share, and improve reusable skills, scripts, workflows, and knowledge across any shell-capable coding agent, including Claude Code, OpenCode, and Cur
298 lines (297 loc) • 12.9 kB
JavaScript
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
/**
* `index.db` utility-score + retrieval-count repository (MemRL signals).
*
* Owns the raw SQL for `utility_scores` and the retrieval-frequency counting
* over `usage_events`. The bounded-step EMA policy itself lives in
* `indexer/feedback/utility-policy`; this repo only reads/writes.
*/
import { conceptIdFromTypeName, parseRefInput } from "../../core/asset/resolve-ref.js";
import { computeNextUtility } from "../../indexer/feedback/utility-policy.js";
import { SQLITE_CHUNK_SIZE } from "./index-sql.js";
/**
* Get the utility score for an entry, or undefined if none exists.
*/
export function getUtilityScore(db, entryId) {
const row = db
.prepare("SELECT entry_id, utility, show_count, search_count, select_rate, last_used_at, updated_at FROM utility_scores WHERE entry_id = ?")
.get(entryId);
if (!row)
return undefined;
return {
entryId: row.entry_id,
utility: row.utility,
showCount: row.show_count,
searchCount: row.search_count,
selectRate: row.select_rate,
lastUsedAt: row.last_used_at ?? undefined,
updatedAt: row.updated_at,
};
}
/**
* Batch-load utility scores for multiple entry IDs, keyed by entry_id.
*/
export function getUtilityScoresByIds(db, ids) {
const scores = new Map();
// Process in chunks to stay within SQLITE_MAX_VARIABLE_NUMBER
for (let i = 0; i < ids.length; i += SQLITE_CHUNK_SIZE) {
const chunk = ids.slice(i, i + SQLITE_CHUNK_SIZE);
const placeholders = chunk.map(() => "?").join(",");
const rows = db
.prepare(`SELECT entry_id, utility, show_count, search_count, select_rate, last_used_at, updated_at FROM utility_scores WHERE entry_id IN (${placeholders})`)
.all(...chunk);
for (const row of rows) {
scores.set(row.entry_id, {
entryId: row.entry_id,
utility: row.utility,
showCount: row.show_count,
searchCount: row.search_count,
selectRate: row.select_rate,
lastUsedAt: row.last_used_at ?? undefined,
updatedAt: row.updated_at,
});
}
}
return scores;
}
/**
* Insert or update a utility score for an entry.
*/
export function upsertUtilityScore(db, entryId, data) {
// Pre-flight FK guard (mirrors `upsertEmbedding`): when an entry is
// deleted between when its id is aggregated from usage_events and when
// this INSERT runs, the FK constraint fails and rolls back the entire
// finalize transaction. A cheap SELECT here turns the race into a
// clean skip. Returns false when the entry no longer exists.
const exists = db.prepare("SELECT 1 FROM entries WHERE id = ?").get(entryId);
if (!exists)
return false;
db.prepare(`
INSERT INTO utility_scores (entry_id, utility, show_count, search_count, select_rate, last_used_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, datetime('now'))
ON CONFLICT(entry_id) DO UPDATE SET
utility = excluded.utility,
show_count = excluded.show_count,
search_count = excluded.search_count,
select_rate = excluded.select_rate,
last_used_at = excluded.last_used_at,
updated_at = datetime('now')
`).run(entryId, data.utility, data.showCount, data.searchCount, data.selectRate, data.lastUsedAt ?? null);
return true;
}
/**
* Reduce a stored ref to its bare form, dropping any `origin//` / `bundle//`
* prefix, matching the counting SQL's `substr` after the first `//`.
*/
function bareRef(ref) {
const boundary = ref.indexOf("//");
return boundary >= 0 ? ref.slice(boundary + 2) : ref;
}
/**
* The bare-form (conceptId) candidate a caller ref matches a stored
* `usage_events` row under. Post-Chunk-8 every stored row is conceptId-spelled
* (the one-time §11.4 re-key ran at the migration cutover), so only the conceptId
* candidate remains — the legacy `type:name` sibling is retired.
*/
function bareRefCandidates(ref) {
const bare = bareRef(ref.trim());
const parsed = parseRefInput(bare);
return [conceptIdFromTypeName(parsed.type, parsed.name)];
}
/**
* Count retrieval events for the given entry refs.
*
* Counts `search`, `show`, and `curate` usage events. Returns a
* Map<inputRef, count> keyed by the *input* ref strings (only those with at
* least one matching event appear). Used by the improve loop's
* proactive-maintenance selector to rank zero-feedback assets by retrieval
* frequency.
*
* Source-scoped callers require either an event linked to an entry in the
* selected stash root or a detached ref qualified with the selected source
* name.
*
* `curate` events are included: their per-item rows are written with
* entry_ref populated (see logCurateEvent), so curation is a real retrieval
* signal here. Summary-only rows with a NULL entry_ref contribute nothing.
*
* Machine/eval/unattributed/extension events are excluded: this count feeds
* salience and ranking, so only explicitly attributed `user` traffic counts.
*/
export function getRetrievalCounts(indexDb, stateDb, refs, options = {}) {
if (refs.length === 0)
return new Map();
if (options.sourceName) {
return getSourceScopedRetrievalCounts(indexDb, stateDb, refs, options);
}
// Map each conceptId back to the input refs that produced it so DB results
// can be re-keyed onto the caller's strings.
const bareToInputs = new Map();
for (const ref of refs) {
for (const bare of bareRefCandidates(ref)) {
const existing = bareToInputs.get(bare);
if (existing)
existing.push(ref);
else
bareToInputs.set(bare, [ref]);
}
}
const bareForms = [...bareToInputs.keys()];
// Accumulate counts per bare form across chunks before re-keying.
const countsByBare = new Map();
// Chunk to stay within SQLITE_MAX_VARIABLE_NUMBER (same pattern as getUtilityScoresByIds).
for (let i = 0; i < bareForms.length; i += SQLITE_CHUNK_SIZE) {
const chunk = bareForms.slice(i, i + SQLITE_CHUNK_SIZE);
const placeholders = chunk.map(() => "?").join(", ");
// Normalize the stored entry_ref to its bare form inside SQL by stripping
// everything up to and including the last `//` separator. SQLite has no
// rfind, but stored origins never themselves contain `//`, so a stash ref
// has exactly one `//` and `substr(... instr ...)` is exact; bare refs have
// no `//` and are excluded above.
// usage_events lives in state.db (Chunk-8 WI-8.3) — this global count needs
// no entries join, so it reads state.db directly.
const rows = stateDb
.prepare(`SELECT
CASE
WHEN instr(entry_ref, '//') > 0
THEN substr(entry_ref, instr(entry_ref, '//') + 2)
ELSE entry_ref
END AS bare_ref,
COUNT(*) AS cnt
FROM usage_events
WHERE event_type IN ('search','show','curate')
AND entry_ref IS NOT NULL
AND instr(entry_ref, '//') > 0
AND source = 'user'
AND CASE
WHEN instr(entry_ref, '//') > 0
THEN substr(entry_ref, instr(entry_ref, '//') + 2)
ELSE entry_ref
END IN (${placeholders})
GROUP BY bare_ref`)
.all(...chunk);
for (const r of rows) {
countsByBare.set(r.bare_ref, (countsByBare.get(r.bare_ref) ?? 0) + r.cnt);
}
}
// Re-key bare-form counts onto every input ref that maps to that bare form.
const result = new Map();
for (const [bare, count] of countsByBare) {
for (const input of bareToInputs.get(bare) ?? []) {
result.set(input, count);
}
}
return result;
}
function getSourceScopedRetrievalCounts(indexDb, stateDb, refs, options) {
const bareToInputs = new Map();
for (const ref of refs) {
for (const bare of bareRefCandidates(ref)) {
const inputs = bareToInputs.get(bare);
if (inputs)
inputs.push(ref);
else
bareToInputs.set(bare, [ref]);
}
}
// Cross-DB: usage rows come from state.db; resolve the linked entry's
// canonical bundle identity from index.db by entry_id.
const entryBundle = indexDb.prepare("SELECT bundle_id FROM entries WHERE id = ?");
const bundleFor = (entryId) => {
if (entryId === null)
return null;
const row = entryBundle.get(entryId);
return row?.bundle_id ?? null;
};
const countsByBare = new Map();
const bareForms = [...bareToInputs.keys()];
const selectedBundle = options.sourceName;
for (let i = 0; i < bareForms.length; i += SQLITE_CHUNK_SIZE) {
const chunk = bareForms.slice(i, i + SQLITE_CHUNK_SIZE);
const placeholders = chunk.map(() => "?").join(", ");
const rawRows = stateDb
.prepare(`SELECT ue.entry_ref, ue.entry_id
FROM usage_events ue
WHERE ue.event_type IN ('search','show','curate')
AND ue.entry_ref IS NOT NULL
AND instr(ue.entry_ref, '//') > 0
AND ue.source = 'user'
AND CASE
WHEN instr(ue.entry_ref, '//') > 0
THEN substr(ue.entry_ref, instr(ue.entry_ref, '//') + 2)
ELSE ue.entry_ref
END IN (${placeholders})`)
.all(...chunk);
const rows = rawRows.map((r) => ({ ...r, bundle_id: bundleFor(r.entry_id) }));
for (const row of rows) {
const bare = bareRef(row.entry_ref);
const linkedToSelectedBundle = row.entry_id !== null && row.bundle_id === selectedBundle;
const detached = row.entry_id === null;
const qualifiedForSource = detached && options.sourceName !== undefined && row.entry_ref === `${options.sourceName}//${bare}`;
if (!linkedToSelectedBundle && !qualifiedForSource)
continue;
countsByBare.set(bare, (countsByBare.get(bare) ?? 0) + 1);
}
}
const result = new Map();
for (const [bare, count] of countsByBare) {
for (const input of bareToInputs.get(bare) ?? [])
result.set(input, count);
}
return result;
}
/**
* Apply accumulated feedback counts to the utility score of an entry, persisting
* the result. The bounded-step EMA policy itself (MemRL, F-5 / #386,
* arXiv:2601.03192) lives in {@link computeNextUtility} (feedback/utility-policy);
* this function only reads the current utility, applies the policy, and writes
* the new value.
*
* A new entry starts at 0.5 (neutral midpoint) before the EMA step is applied.
* When there is no feedback (both counts zero) the score is left untouched — no
* DB write. Returns a {@link FeedbackUtilityResult} so the caller can detect a
* previously high-utility asset crossing below the review threshold and escalate.
*/
export function applyFeedbackToUtilityScore(db, entryId, positiveCount, negativeCount) {
const existing = getUtilityScore(db, entryId);
const previousUtility = existing?.utility ?? 0.5;
const result = computeNextUtility(previousUtility, positiveCount, negativeCount);
if (positiveCount === 0 && negativeCount === 0) {
return result;
}
const now = new Date().toISOString();
db.prepare(`
INSERT INTO utility_scores (entry_id, utility, show_count, search_count, select_rate, last_used_at, updated_at)
VALUES (?, ?, 0, 0, 0, ?, ?)
ON CONFLICT(entry_id) DO UPDATE SET
utility = ?,
updated_at = ?
`).run(entryId, result.nextUtility, now, now, result.nextUtility, now);
return result;
}
/**
* Every indexed entry with its `utility_scores` row (when it has one), for
* `akm metrics`. `utility_scores` is keyed by the unstable `entry_id`, so the
* join to `entries` is what yields a durable ref.
*/
export function listUtilityWithRefs(db) {
const rows = db
.prepare(`SELECT e.item_ref AS ref, u.utility, u.show_count, u.search_count, u.select_rate, u.last_used_at
FROM entries e LEFT JOIN utility_scores u ON u.entry_id = e.id
ORDER BY e.item_ref`)
.all();
return rows.map((row) => row.utility === null
? { ref: row.ref }
: {
ref: row.ref,
score: {
utility: row.utility,
showCount: row.show_count ?? 0,
searchCount: row.search_count ?? 0,
selectRate: row.select_rate ?? 0,
lastUsedAt: row.last_used_at ?? undefined,
},
});
}