akm-cli
Version:
akm (Agent Knowledge Manager) — a portable, local-first capability library for AI agents. Discover, load, share, and improve reusable skills, scripts, workflows, and knowledge across any shell-capable coding agent, including Claude Code, OpenCode, and Cur
827 lines (826 loc) • 39.3 kB
JavaScript
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
/**
* `index.db` entries repository — CRUD, lookup, re-key, delete-cascade, and the
* usage-event relink + workflow-document + tag-set reads that key on `entries`.
*
* Owns ALL raw SQL against the `entries` table. The shared row/option shapes
* come from the leaf types + mapper modules.
*/
import fs from "node:fs";
import path from "node:path";
import { parseBundleRef } from "../../core/asset/asset-ref.js";
import { conceptIdFromTypeName } from "../../core/asset/resolve-ref.js";
import { bestEffort } from "../../core/best-effort.js";
import { isPathAbsent } from "../../core/path-access.js";
import { getStateDbPath, withStateDb } from "../../core/state-db.js";
import { warn } from "../../core/warn.js";
import { getMarkdownFragmentContent, hasMarkdownFragmentContent, } from "../../indexer/passes/metadata.js";
import { buildSearchText } from "../../indexer/search/search-fields.js";
import { sha256Hex } from "../../runtime.js";
import { ENTRY_COLUMNS, rowToIndexedEntry } from "./index-entry-mapper.js";
import { deleteFtsEntries, replaceFtsEntry } from "./index-fts-repository.js";
import { deleteEntryLinks, replaceEntryLinks } from "./index-links-repository.js";
import { SQLITE_CHUNK_SIZE } from "./index-sql.js";
import { deleteEntryVectors } from "./index-vec-repository.js";
// ── Entry operations ────────────────────────────────────────────────────────
/**
* Hash of the text an entry's vector is embedded from (`buildSearchText`),
* stored as `entries.embed_hash`: a change deletes the entry's vector, and the
* next embedding pass re-embeds it.
*/
function embedHash(entry) {
return sha256Hex(buildSearchText(entry));
}
/**
* Insert or update one canonical entry and all synchronously derived search
* state. Returns the stable row id.
*
* The entries row, FTS projection, declared links, and stale-vector
* invalidation commit as one SQLite transaction. Callers therefore cannot
* publish an entry and forget a second FTS or links maintenance step.
*/
export function upsertEntry(db, filePath, entry, provenance, contentHash) {
// Hot path during indexing — cache prepared statements per database
// connection so we don't pay the SQL parse/compile cost on every call.
const stmts = getUpsertStmts(db);
// Phase 5A / Advantage D5: surface derived memory parent ref into the
// dedicated `derived_from` column so retrieval-time lookup (parent→child)
// does not have to scan + JSON-decode every memory row.
const derivedFrom = typeof entry.derivedFrom === "string" && entry.derivedFrom.trim() ? entry.derivedFrom.trim() : null;
const hash = embedHash(entry);
// `content_hash` is optional; an upsert that omits it preserves the scan
// writer's current value.
const apply = () => {
const previous = stmts.findByItemRef.get(provenance.itemRef);
const result = stmts.upsert.get(provenance.itemRef, provenance.bundleId, provenance.componentId, provenance.conceptId, provenance.adapterId, entry.type, filePath, contentHash ?? null, JSON.stringify(entry), derivedFrom, hash);
if (!result)
throw new Error("upsertEntry: item_ref not found after upsert");
if (previous?.id === result.id && previous.embed_hash !== hash)
deleteEntryVectors(db, result.id);
replaceFtsEntry(db, result.id, entry, hasMarkdownFragmentContent(entry) ? (getMarkdownFragmentContent(entry) ?? null) : undefined);
replaceEntryLinks(db, result.id, entry, provenance);
return result.id;
};
// Always enter the driver's transaction wrapper. Both supported SQLite
// drivers lower a transaction opened inside another transaction to a
// savepoint, so a caller that catches this mutation's error cannot commit a
// partial entries row through its outer transaction.
return db.transaction(apply)();
}
const upsertStmtsByDb = new WeakMap();
// item_ref is the sole durable conflict target. `content_hash` COALESCEs so
// an upsert that omits it (`contentHash` is optional) cannot wipe a scan hash.
const UPSERT_SET_CLAUSE = `SET
bundle_id = excluded.bundle_id,
component_id = excluded.component_id,
concept_id = excluded.concept_id,
adapter_id = excluded.adapter_id,
type = excluded.type,
file_path = excluded.file_path,
document_json = excluded.document_json,
derived_from = excluded.derived_from,
embed_hash = excluded.embed_hash,
content_hash = COALESCE(excluded.content_hash, content_hash)`;
function getUpsertStmts(db) {
const existing = upsertStmtsByDb.get(db);
if (existing)
return existing;
const stmts = {
// RETURNING id handles ON CONFLICT DO UPDATE correctly — no second
// SELECT round-trip needed (last_insert_rowid() is unreliable for
// ON CONFLICT). Use `.get()` so a single row comes back.
upsert: db.prepare(`
INSERT INTO entries (
item_ref, bundle_id, component_id, concept_id, adapter_id, type,
file_path, content_hash, document_json, derived_from, embed_hash
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(item_ref) DO UPDATE ${UPSERT_SET_CLAUSE}
RETURNING id
`),
findByItemRef: db.prepare("SELECT id, embed_hash FROM entries WHERE item_ref = ?"),
};
upsertStmtsByDb.set(db, stmts);
return stmts;
}
/**
* Phase 5A / Advantage D5: look up the derived-memory child row whose
* `derived_from` column matches `parentRef` (the 0.9.0 conceptId, e.g.
* `"memories/claude-prefs"` — Group-C item 2 flip).
*
* Returns the most-recently-updated derived child when multiple exist (one
* parent should yield exactly one `.derived` child in practice, but the
* ordering keeps results deterministic). Returns `null` when no derived
* child has been indexed for this parent.
*/
export function getDerivedForParent(db, parentRef, bundleId) {
if (!parentRef)
return null;
const sourceScope = bundleId ? "AND bundle_id = ?" : "";
const row = db
.prepare(`SELECT ${ENTRY_COLUMNS}
FROM entries
WHERE derived_from = ?
${sourceScope}
ORDER BY id DESC
LIMIT 1`)
.get(parentRef, ...(bundleId ? [bundleId] : []));
if (!row)
return null;
return rowToIndexedEntry(row, "getDerivedForParent");
}
/**
* 03-R3: for the given derived-twin row ids, fetch each twin's BASE memory
* `beliefState`, keyed by twin id.
*
* Used by the derived-twin belief inheritance in search ranking: a `.derived`
* twin has no belief state of its own, so it inherits its base memory's
* demoting state (contradicted/superseded/…) at search time. A twin's durable
* `item_ref` is exactly its base ref plus the `.derived` suffix, so the base is
* found by stripping that suffix. Returns a map
* of twin id → base beliefState for bases that carry a non-empty state.
* Best-effort: any query error (e.g. legacy DB) yields no inheritance rather
* than failing the search.
*/
export function getBaseBeliefStatesForDerivedTwins(db, twinIds) {
const out = new Map();
if (twinIds.length === 0)
return out;
// Chunk at SQLITE_CHUNK_SIZE like the sibling bulk-by-id helpers, so a large
// `--limit` candidate set never trips SQLITE_MAX_VARIABLE_NUMBER (which would
// otherwise fall into the best-effort catch and silently disable the feature).
for (let i = 0; i < twinIds.length; i += SQLITE_CHUNK_SIZE) {
const chunk = twinIds.slice(i, i + SQLITE_CHUNK_SIZE);
const placeholders = chunk.map(() => "?").join(",");
bestEffort(() => {
const rows = db
.prepare(`SELECT twin.id AS twin_id, json_extract(base.document_json, '$.beliefState') AS belief
FROM entries twin
JOIN entries base
ON base.type = 'memory'
AND base.item_ref = substr(twin.item_ref, 1, length(twin.item_ref) - length('.derived'))
WHERE twin.id IN (${placeholders})
AND twin.item_ref LIKE '%.derived'
AND json_extract(base.document_json, '$.beliefState') IS NOT NULL`)
.all(...chunk);
for (const r of rows) {
if (typeof r.belief === "string" && r.belief.trim().length > 0)
out.set(r.twin_id, r.belief.trim());
}
}, "belief-state inheritance is best-effort");
}
return out;
}
/**
* Re-key an entries row in place for the opt-in source-maintenance script.
*
* The row id is preserved on purpose — `utility_scores` and `embeddings` are
* keyed by `entry_id`, so an UPDATE (rather than a delete + insert under the
* new `item_ref`) is what keeps the asset's accumulated usage-ranking history
* attached across a rename. (`asset_salience` / `asset_outcome` live in
* state.db keyed by `asset_ref` TEXT and are re-keyed separately by `akm mv` — see
* the state rekey helper.) `document_json.name` (and `filename`, when
* present) is patched and `embed_hash` recomputed so search reflects the new
* name. Its FTS projection and stale vector are updated in the same
* transaction as the canonical identity.
*
* Bundle-qualified `usage_events.entry_ref` rows for the old conceptId are
* rewritten to the new item ref. Without this, events keep the old
* ref, `relinkUsageEvents` finds no matching entry after the next full
* rebuild, and the utility history the re-key exists to preserve silently
* resets. DETACHED orphan events already sitting at the new ref (entry_id
* NULL — a deleted stranger's history) are deleted first, so the moved asset
* never adopts them (live asset's history wins, matching the stale-row
* eviction below).
*
* A stale row already occupying the new item ref (the caller has verified no
* FILE exists at the target, so such a row can only be a leftover for a
* deleted file) is evicted first — through {@link deleteRelatedRows}, so its
* child rows (embeddings, utility scores, usage events) go with
* it. A bare `DELETE FROM entries` would trip the non-CASCADE `embeddings`
* FK under `PRAGMA foreign_keys = ON` and roll back the whole re-key.
* The moved row keeps its id.
*
* Returns the surviving row id, or `null` when no row matches the old item ref
* (nothing indexed under the old name — the caller falls open and the next
* full `akm index` picks the file up as a fresh entry).
*/
export function rekeyEntryInPlace(db, opts) {
const oldItemRef = `${opts.sourceName}//${opts.oldRef}`;
const row = db
.prepare("SELECT id, file_path, document_json, embed_hash, type FROM entries WHERE item_ref = ?")
.get(oldItemRef);
if (!row)
return null;
const sourceRoot = path.resolve(opts.sourceRoot);
const currentPath = path.resolve(row.file_path);
if (currentPath !== sourceRoot && !currentPath.startsWith(`${sourceRoot}${path.sep}`)) {
throw new Error(`Refusing to re-key entry ${oldItemRef}: source root does not match.`);
}
// Patch the JSON payload. On corrupt document_json still re-key identity/path so
// the utility history survives; the next full index heals the JSON.
let documentJson = row.document_json;
let hash = row.embed_hash;
let document;
try {
const entry = JSON.parse(row.document_json);
entry.name = opts.newName;
if (typeof entry.filename === "string")
entry.filename = path.basename(opts.newFilePath);
if (opts.newDerivedFrom !== undefined)
entry.derivedFrom = opts.newDerivedFrom;
documentJson = JSON.stringify(entry);
hash = embedHash(entry);
document = entry;
}
catch {
/* corrupt document_json — identity/path-only re-key */
}
const expectedNewRef = conceptIdFromTypeName(row.type, opts.newName);
if (opts.newRef !== expectedNewRef) {
throw new Error(`Refusing to re-key entry ${oldItemRef}: target ref does not match the entry type and name.`);
}
const newItemRef = `${opts.sourceName}//${opts.newRef}`;
db.transaction(() => {
const stale = db.prepare("SELECT id FROM entries WHERE item_ref = ?").get(newItemRef);
if (stale && stale.id !== row.id) {
// Full child-row cleanup (embeddings, utility scores,
// usage events, FTS + dirty marks) BEFORE the entries delete: the
// `embeddings` FK is non-CASCADE and `foreign_keys = ON`, so a bare
// entries delete would throw and roll back the entire re-key; and
// without it the FK-less child rows would orphan permanently.
deleteRelatedRows(db, [{ id: stale.id }]);
db.prepare("DELETE FROM entries WHERE id = ?").run(stale.id);
}
db.prepare("UPDATE entries SET file_path = ?, document_json = ?, embed_hash = ?, item_ref = ?, concept_id = ? WHERE id = ?").run(opts.newFilePath, documentJson, hash, newItemRef, opts.newRef, row.id);
if (opts.newDerivedFrom !== undefined) {
db.prepare("UPDATE entries SET derived_from = ? WHERE id = ?").run(opts.newDerivedFrom, row.id);
}
if (row.embed_hash !== hash)
deleteEntryVectors(db, row.id);
if (document) {
replaceFtsEntry(db, row.id, document, hasMarkdownFragmentContent(document) ? (getMarkdownFragmentContent(document) ?? null) : undefined);
replaceEntryLinks(db, row.id, document, { bundleId: opts.sourceName, conceptId: opts.newRef });
}
else
deleteFtsEntries(db, [row.id]);
})();
// Re-point usage history at the new ref. Chunk-8 WI-8.3: usage_events lives in
// state.db now, so this is a SEPARATE cross-DB transaction (best-effort — the
// rename itself already committed above; on failure the next full index's
// relinkUsageEvents re-attaches by ref). See the docstring for the spellings
// and the live-asset-wins collision policy.
rewriteUsageEventRefForMove(opts);
return row.id;
}
/**
* Rewrite `usage_events.entry_ref` from `opts.oldRef` to `opts.newRef` in
* state.db. DETACHED orphan events (entry_id NULL) already sitting AT the new
* ref are evicted first so the moved asset never adopts a deleted stranger's
* history (live-asset-wins). Best-effort + guarded on state.db's existence.
*/
function rewriteUsageEventRefForMove(opts) {
// Every other failure in here throws (see the catch below) precisely because
// a move that quietly drops its usage history is a wrong answer wearing a
// success. An unreadable state.db must not be the one silent exception —
// only a state.db that was never created skips (#791).
if (isPathAbsent(getStateDbPath()))
return;
// `usage_events.entry_ref` is the fully-qualified item_ref
// (`<bundle>//<conceptId>`).
const rename = (stateDb, oldR, newR) => {
stateDb.prepare("DELETE FROM usage_events WHERE entry_id IS NULL AND entry_ref = ?").run(newR);
stateDb.prepare("UPDATE usage_events SET entry_ref = ? WHERE entry_ref = ?").run(newR, oldR);
};
try {
withStateDb((stateDb) => {
stateDb.transaction(() => {
rename(stateDb, `${opts.sourceName}//${opts.oldRef}`, `${opts.sourceName}//${opts.newRef}`);
})();
});
}
catch (error) {
const message = error instanceof Error ? error.message : String(error);
throw new Error(`Failed to rewrite usage events for move: ${message}`, { cause: error });
}
}
/**
* Rows whose `file_path` sits directly in `dirPath`. A half-open byte range
* over `idx_entries_file_path` (`[dir + sep, dir + sep + 1)`) turns the lookup
* into an index seek; the range is exact for "starts with `dir/`" but also
* admits nested subdirectories (`/a/b/c/x.md` for `/a/b`), so the dirname
* post-filter stays.
*/
function selectRowsInDirectory(db, dirPath, columns, bundleId) {
const resolvedDir = path.resolve(dirPath);
const prefix = resolvedDir + path.sep;
const upperBound = resolvedDir + String.fromCharCode(path.sep.charCodeAt(0) + 1);
const params = [prefix, upperBound];
let sql = `SELECT ${columns} FROM entries WHERE file_path >= ? AND file_path < ?`;
if (bundleId) {
sql += " AND bundle_id = ?";
params.push(bundleId);
}
const rows = db.prepare(sql).all(...params);
return rows.filter((row) => path.dirname(row.file_path) === resolvedDir);
}
function rowsInDirectory(db, dirPath, bundleId) {
return selectRowsInDirectory(db, dirPath, "id, item_ref, file_path", bundleId);
}
function deleteEntryRows(db, rows, options = {}) {
if (rows.length === 0)
return [];
deleteRelatedRows(db, rows, options);
for (let i = 0; i < rows.length; i += SQLITE_CHUNK_SIZE) {
const chunk = rows.slice(i, i + SQLITE_CHUNK_SIZE);
const placeholders = chunk.map(() => "?").join(",");
db.prepare(`DELETE FROM entries WHERE id IN (${placeholders})`).run(...chunk.map((row) => row.id));
}
return rows.map((row) => row.id);
}
export function deleteEntriesByDirAndBundle(db, dirPath, bundleId, options = {}) {
return db.transaction(() => deleteEntryRows(db, rowsInDirectory(db, dirPath, bundleId), options))();
}
/** Delete every entry and child row belonging to one canonical bundle. */
export function deleteEntriesByBundle(db, bundleId) {
db.transaction(() => {
const rows = db.prepare("SELECT id FROM entries WHERE bundle_id = ?").all(bundleId);
deleteEntryRows(db, rows);
})();
}
/**
* The `file_path` of every entry indexed under one bundle — used by
* `akm bundle rename` (D6) to find bundle CONTENT that still spells the old
* `<bundle>//` prefix (xrefs, `supersededBy`, task `uses:`), which the rename
* reports rather than rewrites.
*/
export function getFilePathsByBundle(db, bundleId) {
const rows = db.prepare("SELECT DISTINCT file_path FROM entries WHERE bundle_id = ?").all(bundleId);
return rows.map((row) => row.file_path);
}
/**
* Re-key every entry row's `bundle_id`/`item_ref` from `oldBundleId` to
* `newBundleId` in place (`akm bundle rename`, D6). Unlike
* {@link rekeyEntryInPlace} (one asset, `akm mv`), this is a bulk identity
* change with no content move: `concept_id`/`file_path`/`document_json` are
* untouched, so no FTS/vector rebuild is needed (FTS and `embeddings` key on
* the entry's row `id`, which this preserves, not on `item_ref`). Returns the
* number of rows renamed.
*/
export function renameEntriesBundleId(db, oldBundleId, newBundleId) {
return db.transaction(() => {
const result = db
.prepare("UPDATE entries SET bundle_id = ?, item_ref = ? || '//' || concept_id WHERE bundle_id = ?")
.run(newBundleId, newBundleId, oldBundleId);
return Number(result.changes);
})();
}
/**
* Diff-persist orphan delete: remove every entry under `dirPath` whose durable
* `item_ref` is not in `keepRefs`.
*
* Replaces the old per-dir `deleteEntriesByDir` + full re-insert: the caller
* upserts the current file set first (ON CONFLICT preserving `entries.id`, so
* embeddings / utility / usage stay attached to unchanged rows), then calls this
* to prune only the departed rows. The net row-state for the directory is identical
* to delete-then-reinsert; the win is that unchanged rows keep their id.
*
* Both directory- and bundle-scoped so overlapping physical roots cannot
* prune one another's rows.
*/
export function deleteEntriesByDirExceptRefs(db, dirPath, bundleId, keepRefs, options = {}) {
return db.transaction(() => {
const doomed = rowsInDirectory(db, dirPath, bundleId).filter((row) => !keepRefs.has(row.item_ref));
return deleteEntryRows(db, doomed, options);
})();
}
function deleteRelatedRows(db, ids, options = {}) {
if (ids.length === 0)
return;
const numericIds = ids.map((r) => r.id);
// FTS and declared links are part of the canonical mutation boundary, not a
// caller-maintained dirty queue. Delete them before the parent row inside
// this transaction.
deleteFtsEntries(db, numericIds);
deleteEntryLinks(db, numericIds);
// Process in chunks to stay within SQLITE_MAX_VARIABLE_NUMBER
for (let i = 0; i < numericIds.length; i += SQLITE_CHUNK_SIZE) {
const chunk = numericIds.slice(i, i + SQLITE_CHUNK_SIZE);
const placeholders = chunk.map(() => "?").join(",");
bestEffort(() => db.prepare(`DELETE FROM embeddings WHERE id IN (${placeholders})`).run(...chunk), "delete embeddings for entries");
// Clean up utility scores before deleting entries
bestEffort(() => db.prepare(`DELETE FROM utility_scores WHERE entry_id IN (${placeholders})`).run(...chunk), "delete utility_scores for entries");
}
// usage_events lives in state.db, outside this transaction. Index persistence
// disables this cleanup and runs it only after its index.db transaction
// commits; standalone delete callers retain the immediate behavior.
if (options.cleanupUsageEvents !== false)
deleteUsageEventsByEntryIds(numericIds);
}
export function deleteUsageEventsByEntryIds(entryIds) {
if (entryIds.length === 0 || !fs.existsSync(getStateDbPath()))
return;
bestEffort(() => {
withStateDb((stateDb) => {
for (let i = 0; i < entryIds.length; i += SQLITE_CHUNK_SIZE) {
const chunk = entryIds.slice(i, i + SQLITE_CHUNK_SIZE);
const placeholders = chunk.map(() => "?").join(",");
stateDb.prepare(`DELETE FROM usage_events WHERE entry_id IN (${placeholders})`).run(...chunk);
}
});
}, "delete usage_events (state.db) for entries");
}
/**
* Delete entries by their primary key IDs, along with all related rows
* (embeddings, entries_fts, utility scores, usage_events).
*
* Used by explicit `--clean` reconciliation before embeddings and final
* verification to remove stale entries whose source files no longer exist.
*/
export function deleteEntriesByIds(db, ids) {
if (ids.length === 0)
return;
db.transaction(() => {
const idObjs = ids.map((id) => ({ id }));
deleteRelatedRows(db, idObjs);
for (let i = 0; i < ids.length; i += SQLITE_CHUNK_SIZE) {
const chunk = ids.slice(i, i + SQLITE_CHUNK_SIZE);
const placeholders = chunk.map(() => "?").join(",");
db.prepare(`DELETE FROM entries WHERE id IN (${placeholders})`).run(...chunk);
}
})();
}
// ── All entries ─────────────────────────────────────────────────────────────
function parseEntryRows(rows, context) {
const entries = [];
for (const row of rows) {
const mapped = rowToIndexedEntry(row, context);
if (mapped)
entries.push(mapped);
}
return entries;
}
export function getAllEntries(db, entryType, excludeTypes) {
let sql;
let params;
// #627 — exclude-type clause applies only on the untyped ('any') path. Empty
// list skips the clause (never `NOT IN ()`).
const excludes = excludeTypes && excludeTypes.length > 0 ? excludeTypes : [];
if (entryType && entryType !== "any") {
sql = `SELECT ${ENTRY_COLUMNS} FROM entries WHERE type = ?`;
params = [entryType];
}
else if (excludes.length > 0) {
sql = `SELECT ${ENTRY_COLUMNS} FROM entries WHERE type NOT IN (${excludes.map(() => "?").join(", ")})`;
params = [...excludes];
}
else {
sql = `SELECT ${ENTRY_COLUMNS} FROM entries`;
params = [];
}
const rows = db.prepare(sql).all(...params);
return parseEntryRows(rows, "getAllEntries");
}
/**
* Resolve a single `entries.id` from a new-grammar `[bundle//]conceptId` ref,
* keying on the canonical stored `item_ref` (ref-grammar decision D-R1/D-R4).
* The optional `bundleId` scopes the match to one indexed bundle.
*/
export function findEntryIdByRef(db, ref, bundleId) {
return findEntryIdByBundleRef(db, ref, bundleId);
}
/** `name` plus its `.md`-toggled sibling — the markdown ext-keep/strip ambiguity. */
function withMdVariants(name) {
return name.endsWith(".md") ? [name, name.slice(0, -3)] : [name, `${name}.md`];
}
/**
* Current (`[bundle//]conceptId`) id lookup: match `item_ref` exactly when
* bundle-qualified or by `//conceptId` suffix when short.
*/
function findEntryIdByBundleRef(db, ref, bundleId) {
const parsed = parseBundleRef(ref);
const conceptVariants = withMdVariants(parsed.conceptId);
// item_ref is the canonical stored spelling post-flip.
for (const conceptId of conceptVariants) {
const id = matchIdByItemRef(db, parsed.bundle, conceptId, bundleId);
if (id !== undefined)
return id;
}
return undefined;
}
/**
* Match a row `id` by `item_ref`: exact `bundle//conceptId` when `bundle` is
* known, else the `//conceptId` SUFFIX (short ref — any bundle). The suffix
* uses `substr(...) = ...` (never LIKE) so `_`/`%` in a conceptId are literal,
* and includes the `//` boundary so a segment split never false-matches.
*
* ── Deterministic winner (`ORDER BY id ASC`) ──
*
* A SHORT ref can match one concept id across MULTIPLE bundles (e.g. the same
* `knowledge/guide` in the primary stash and an installed source). The bare
* `LIMIT 1` this replaced picked whichever row SQLite happened to visit first —
* a nondeterministic winner when two bundles share a conceptId. We now impose a
* total order (`ORDER BY id ASC`) so the winner is STABLE across runs.
*
* `id ASC` is also the SENSIBLE choice, not just a stable tiebreak: a full index
* walks its sources in installation-precedence order (`resolveSourceEntries`,
* primary stash first — the same order `resolveSourcesForOrigin` calls "local"),
* so the highest-precedence source's rows carry the LOWEST ids. Ascending id
* therefore prefers the primary/highest-precedence bundle — mirroring what a
* precedence-ordered scan would pick — while staying a pure config-free leaf
* (true installation-priority resolution against an injected bundle list is
* `resolveRef`'s job; this DB helper takes no config handle). The exact-bundle
* arm is single-row under the UNIQUE `item_ref` index.
*/
function matchIdByItemRef(db, bundle, conceptId, bundleId) {
const scope = bundleId ? "AND bundle_id = ?" : "";
if (bundle !== undefined) {
const itemRef = `${bundle}//${conceptId}`;
const row = db
.prepare(`SELECT id FROM entries WHERE item_ref = ? ${scope} ORDER BY id ASC LIMIT 1`)
.get(itemRef, ...(bundleId ? [bundleId] : []));
return row?.id;
}
const suffix = `//${conceptId}`;
const row = db
.prepare(`SELECT id FROM entries
WHERE substr(item_ref, length(item_ref) - length(?) + 1) = ?
${scope}
ORDER BY id ASC
LIMIT 1`)
.get(suffix, suffix, ...(bundleId ? [bundleId] : []));
return row?.id;
}
export function getEntryCount(db) {
const row = db.prepare("SELECT COUNT(*) AS cnt FROM entries").get();
return row.cnt;
}
/**
* Per-asset-type entry counts (keyed by `type`, e.g. "skill",
* "knowledge", "memory"). Used by `akm info` to break down the aggregate
* `indexStats.entryCount` (R-057).
*/
export function getEntryCountByType(db) {
const rows = db.prepare("SELECT type, COUNT(*) AS cnt FROM entries GROUP BY type").all();
const out = {};
for (const row of rows) {
out[row.type] = row.cnt;
}
return out;
}
export function getEmbeddableEntryCount(db) {
return getEntryCount(db);
}
export function getEntryById(db, id) {
const row = db
.prepare("SELECT file_path, document_json, item_ref, bundle_id, component_id, concept_id, adapter_id FROM entries WHERE id = ?")
.get(id);
if (!row)
return undefined;
// Guard against corrupt JSON
let entry;
try {
entry = JSON.parse(row.document_json);
}
catch {
warn(`[db] getEntryById: skipping entry id=${id} — corrupt document_json`);
return undefined;
}
return {
filePath: row.file_path,
entry,
itemRef: row.item_ref,
bundleId: row.bundle_id,
componentId: row.component_id,
conceptId: row.concept_id,
adapterId: row.adapter_id,
};
}
/** The `item_ref` and `type` of every id that has an entry row. */
export function getEntryRefsAndTypes(db, ids) {
const out = new Map();
for (let i = 0; i < ids.length; i += SQLITE_CHUNK_SIZE) {
const chunk = ids.slice(i, i + SQLITE_CHUNK_SIZE);
const placeholders = chunk.map(() => "?").join(",");
const rows = db
.prepare(`SELECT id, item_ref, type FROM entries WHERE id IN (${placeholders})`)
.all(...chunk);
for (const row of rows)
out.set(row.id, { itemRef: row.item_ref, type: row.type });
}
return out;
}
export function getEntriesByDir(db, dirPath) {
return parseEntryRows(selectRowsInDirectory(db, dirPath, ENTRY_COLUMNS), "getEntriesByDir");
}
/** Return every directory previously indexed for one canonical bundle. */
export function getIndexedDirPathsByBundleId(db, bundleId) {
const rows = db.prepare("SELECT file_path FROM entries WHERE bundle_id = ?").all(bundleId);
return [...new Set(rows.map((row) => path.dirname(row.file_path)))];
}
/** Return every persisted bundle owner for one physical directory. */
export function getIndexedBundleIdsByDir(db, dirPath) {
const rows = selectRowsInDirectory(db, dirPath, "bundle_id, file_path");
return [...new Set(rows.map((row) => row.bundle_id))];
}
/**
* Resolve a single `entries.id` by exact `file_path` (the canonical on-disk
* path), or `undefined` if no row matches.
*
* Lifted verbatim (WS5) from the inline `SELECT id FROM entries WHERE
* file_path = ? LIMIT 1` in commands/search.ts so all `entries` SQL lives in
* this module. The result is a plain number materialised before return —
* nothing lazy crosses a connection boundary.
*/
export function getEntryIdByFilePath(db, filePath) {
const row = db.prepare("SELECT id FROM entries WHERE file_path = ? LIMIT 1").get(filePath);
return row?.id;
}
/**
* Set of every non-empty `entries.file_path` currently indexed (across all
* stashes/sources). Used by staleness detection to spot files that exist on
* disk but were never indexed — a clock-independent signal for newly-added
* assets that an mtime-vs-builtAt comparison can miss when the two clocks
* (filesystem vs wall-clock) are skewed within the same millisecond.
*/
export function getIndexedFilePaths(db) {
const rows = db
.prepare("SELECT DISTINCT file_path FROM entries WHERE file_path IS NOT NULL AND file_path <> ''")
.all();
return new Set(rows.map((r) => r.file_path));
}
/**
* `entries.file_path -> content_hash` for every currently-indexed file that
* carries a stored hash. Used by per-file staleness detection
* (`hasNewerIndexableFiles`, R6) to tell "edited after the last build" apart
* from "already re-indexed with this exact content after the last build" —
* `indexWrittenAssets` upserts a fresh row (and its `content_hash`) without
* bumping `builtAt`, so mtime alone cannot distinguish the two cases.
*/
export function getIndexedFileHashes(db) {
const rows = db
.prepare(`SELECT file_path, content_hash FROM entries
WHERE file_path IS NOT NULL AND file_path <> ''
AND content_hash IS NOT NULL`)
.all();
return new Map(rows.map((r) => [r.file_path, r.content_hash]));
}
/**
* Resolve a single `entries.file_path` by primary key, or `undefined` if no
* row matches.
*
* Lifted verbatim (WS5) from the inline `SELECT file_path FROM entries WHERE
* id = ?` in commands/feedback-cli.ts. Unlike {@link getEntryById}, this does
* NOT parse `document_json`, so a row with corrupt JSON still yields its path —
* preserving feedback-cli's pre-extraction behaviour byte-for-byte.
*/
export function getEntryFilePathById(db, id) {
const row = db.prepare("SELECT file_path FROM entries WHERE id = ?").get(id);
return row?.file_path;
}
// ── Indexer-phase helpers (moved from indexer.ts) ────────────────────────────
/**
* Return distinct zero-result search queries from the `usage_events` table
* within the given lookback window.
*
* Reads from `usage_events` (event_type = 'search') where the metadata JSON
* blob contains `resultCount = 0`. The `search_events` table never existed;
* all errors are caught and an empty array is returned so callers never need
* to guard against DB schema differences.
*/
export function getZeroResultSearches(db, sinceDays = 30) {
const since = new Date(Date.now() - sinceDays * 24 * 60 * 60 * 1000).toISOString();
try {
const rows = db
.prepare(`SELECT DISTINCT json_extract(metadata, '$.query') AS query
FROM usage_events
WHERE event_type = 'search'
AND created_at >= ?
AND json_extract(metadata, '$.resultCount') = 0
ORDER BY created_at DESC LIMIT 20`)
.all(since);
return rows.map((r) => r.query).filter((q) => q !== null);
}
catch {
return []; // table may not exist in older DBs
}
}
/**
* Look up an entry `id` by a new-grammar `[bundle//]conceptId` ref, resolving
* against the canonical stored `item_ref`. Returns `{ id }` or `null`.
*/
export function getEntryByRef(db, ref) {
const id = findEntryIdByRef(db, ref);
return id === undefined ? null : { id };
}
/** Build a {@link LiveRefSnapshot} from the current `entries` table. */
export function getLiveRefSnapshot(db) {
const rows = db.prepare("SELECT item_ref FROM entries").all();
const itemRefs = new Set();
const conceptIds = new Set();
for (const { item_ref } of rows) {
itemRefs.add(item_ref);
const boundary = item_ref.indexOf("//");
if (boundary >= 0)
conceptIds.add(item_ref.slice(boundary + 2));
}
return { itemRefs, conceptIds };
}
/**
* `getEntryByRef`'s liveness check (bundle-qualified exact match, or bare
* `//conceptId` suffix match across bundles, each tried with the `.md`
* toggle {@link withMdVariants} applies) against a prebuilt
* {@link LiveRefSnapshot} instead of the database — no query per call.
*/
export function isRefLiveInSnapshot(snapshot, ref) {
const parsed = parseBundleRef(ref);
const conceptVariants = withMdVariants(parsed.conceptId);
if (parsed.bundle !== undefined) {
return conceptVariants.some((conceptId) => snapshot.itemRefs.has(`${parsed.bundle}//${conceptId}`));
}
return conceptVariants.some((conceptId) => snapshot.conceptIds.has(conceptId));
}
/**
* The fully-qualified `item_ref` (`<bundle>//<conceptId>`, the durable stored
* spelling — spec §11.1 D-R3) for an entry `id`, or `null` when the row is gone
* The usage-event / salience / feedback writers derive the durable
* key from this so a stored key is always the resolved entry's canonical ref,
* never raw input (D-R3: durable keys are never derived from input).
*/
export function getItemRefById(db, id) {
const row = db.prepare("SELECT item_ref FROM entries WHERE id = ?").get(id);
return row?.item_ref ?? null;
}
/**
* Resolve a `usage_events.entry_ref` to its live `entries.id`. `entry_ref` is
* the fully-qualified `bundle//conceptId` `item_ref` spelling, so this keys
* directly on the globally-unique `item_ref`. Bare durable refs are invalid and
* remain detached.
*/
function resolveUsageEventEntryId(db, ref) {
if (parseBundleRef(ref).bundle === undefined)
return undefined;
return findEntryIdByRef(db, ref);
}
/**
* Re-link detached usage_events to their current entry_ids via entry_ref.
*
* After a full rebuild, entry IDs change. This restores each event's link
* using the stable `entry_ref` column so usage history survives a reindex.
*
* Cross-DB (Chunk-8 WI-8.3): `usage_events` lives in `stateDb` while `entries`
* lives in `indexDb`. The stale-id null-out (formerly a single-DB
* `NOT IN (SELECT id FROM entries)`) is done in two bounded passes — the set of
* distinct linked entry_ids in usage_events is small — and the re-resolution
* reads `entries` from `indexDb`.
*/
export function relinkUsageEvents(indexDb, stateDb) {
bestEffort(() => {
// Step 1: null out stale entry_ids (entry was deleted, re-keyed, etc).
// Leaving them in place would let `recomputeUtilityScores` aggregate by an
// entry_id that no longer exists in `entries`, then trip the FK constraint
// on the utility_scores INSERT and roll back the entire finalize
// transaction. Nulled rows can be re-resolved by step 2 below; events whose
// entry is permanently gone simply stay null and age out via retention.
const linkedRows = stateDb
.prepare("SELECT DISTINCT entry_id AS id, entry_ref AS ref FROM usage_events WHERE entry_id IS NOT NULL")
.all();
const entryIdentity = indexDb.prepare("SELECT item_ref AS itemRef FROM entries WHERE id = ?");
const staleLinks = linkedRows.filter(({ id, ref }) => {
const live = entryIdentity.get(id);
return live == null || (ref !== null && live.itemRef !== ref);
});
if (staleLinks.length > 0) {
const nullOut = stateDb.prepare("UPDATE usage_events SET entry_id = NULL WHERE entry_id = ? AND entry_ref IS ?");
const nullTx = stateDb.transaction(() => {
for (const { id, ref } of staleLinks)
nullOut.run(id, ref);
});
nullTx();
}
// Step 2: re-resolve each fully-qualified ref. Bare rows are not current
// durable identities and remain detached.
const refs = stateDb
.prepare("SELECT DISTINCT entry_ref AS ref FROM usage_events WHERE entry_id IS NULL AND entry_ref IS NOT NULL")
.all();
const update = stateDb.prepare("UPDATE usage_events SET entry_id = ? WHERE entry_ref = ? AND entry_id IS NULL");
const relinkTx = stateDb.transaction(() => {
for (const { ref } of refs) {
let id;
try {
id = resolveUsageEventEntryId(indexDb, ref);
}
catch (err) {
if (err instanceof Error && err.name === "UsageError")
continue;
throw err;
}
if (id !== undefined)
update.run(id, ref);
}
});
relinkTx();
}, "usage_events table may not exist yet during entry_id re-resolution");
}