open-memex/dist/retrieve/search.js

Local-first memory layer and protocol for AI coding agents. Markdown source of truth, SQLite FTS5 index, zero cloud.

import { db } from "../store/db.js";
import { toFtsQuery } from "./query.js";
function toHit(r) {
    const aliases = r.aliases ? r.aliases.split(",").map((a) => a.trim()).filter(Boolean) : [];
    let snippet = r.snippet ?? "";
    // D61: the snippet highlights matches in the body column only. When the
    // match came through an alias (body shows no highlight), say so — the
    // hit otherwise looks unrelated to the query.
    if (aliases.length > 0 && !/\[[^\]]+\]/.test(snippet)) {
        snippet += ` (aka: ${aliases.join(", ")})`;
    }
    return {
        id: r.id,
        scope_key: r.scope_key,
        project_name: r.project_name,
        type: r.type,
        tags: r.tags ? r.tags.split(",").filter(Boolean) : [],
        aliases,
        snippet,
        score: r.score,
        updated_at: r.updated_at,
        status: r.status,
        review_state: r.review_state ?? "draft",
        source: r.source ?? "",
        created_at: r.created_at ?? r.updated_at ?? 0,
    };
}
/**
 * D30: review-state ranking tiers for shared (project) memories.
 * Reviewed knowledge (approved/published) outranks unreviewed outbox drafts;
 * personal memories are always `draft` by design and are NOT demoted.
 * Stable: order within a tier keeps the incoming score/recency order.
 */
function reviewTier(r) {
    if (r.scope_key === "personal")
        return 1;
    if (r.review_state === "approved" || r.review_state === "published")
        return 0;
    if (r.review_state === "draft" || r.review_state === "rejected")
        return 2;
    return 1; // proposed and anything unexpected
}
/** D30: `[draft]` / `[proposed]` / … marker for project memories.
 *  Personal memories stay unmarked (always draft — the marker would be noise). */
export function hitStateLabel(h) {
    if (h.scope_key === "personal")
        return "";
    return ` [${h.review_state || "draft"}]`;
}
/**
 * Lifecycle-aware post-processing (§3.3):
 * - retracted / archived are excluded from retrieval (kept for audit);
 * - a superseded memory resolves to the newest of its chain (cycle-safe);
 * - D30: approved/published project memories rank first, project
 *   drafts/rejected rank after unreviewed content;
 * - deprecated stays visible as a warning but ranks after active.
 */
function resolveVisible(rows, limit, stats) {
    const byId = new Map(rows.map((r) => [r.id, r]));
    const fullRow = (id) => {
        const cached = byId.get(id);
        if (cached)
            return cached;
        const r = db()
            .prepare(`SELECT id, scope_key, project_name, type, tags, updated_at, status,
                superseded_by, review_state, source, created_at, substr(content, 1, 240) AS snippet
         FROM memories WHERE id = ?`)
            .get(id);
        if (!r)
            return undefined;
        const full = { ...r, score: 0 };
        byId.set(id, full);
        return full;
    };
    const seen = new Set();
    const tiers = [[], [], [], []];
    const deprecated = [];
    for (const r of rows) {
        if (r.status === "retracted" || r.status === "archived") {
            if (stats)
                stats.hiddenExcluded++;
            continue;
        }
        let target = r;
        if (r.status === "superseded") {
            if (stats)
                stats.hiddenSuperseded++;
            let cur = r;
            const chain = new Set([r.id]);
            while (cur.status === "superseded" && cur.superseded_by) {
                if (chain.has(cur.superseded_by))
                    break; // cycle guard
                chain.add(cur.superseded_by);
                const nxt = fullRow(cur.superseded_by);
                if (!nxt)
                    break;
                cur = nxt;
            }
            if (cur.status === "retracted" || cur.status === "archived")
                continue;
            target = cur;
        }
        if (seen.has(target.id))
            continue;
        seen.add(target.id);
        const hit = toHit(target);
        if (target.status === "deprecated")
            deprecated.push(hit);
        else
            tiers[reviewTier(target)].push(hit);
    }
    return [...tiers[0], ...tiers[1], ...tiers[2], ...deprecated].slice(0, limit);
}
/**
 * FTS5 query construction lives in query.ts (pure, unit-tested): free-text
 * questions are split into content terms (OR-ed; bm25 ranks), with
 * function/question words filtered so they can't dilute the ranking.
 */
export function search(query, opts = {}) {
    const q = toFtsQuery(query);
    if (opts.stats) {
        opts.stats.ftsQuery = q;
        opts.stats.candidates = 0;
        opts.stats.hiddenSuperseded = 0;
        opts.stats.hiddenExcluded = 0;
    }
    if (!q)
        return [];
    const limit = Math.max(1, Math.min(opts.limit ?? 8, 50));
    // Over-fetch: lifecycle filtering (chain resolution, exclusions) happens
    // after the FTS query, so candidates must survive it.
    const fetchLimit = Math.min(limit * 3 + 10, 150);
    const scopeFilter = opts.scopeKeys && opts.scopeKeys.length > 0
        ? ` AND m.scope_key IN (${opts.scopeKeys.map(() => "?").join(",")})`
        : "";
    const typeFilter = opts.type ? ` AND m.type = ?` : "";
    const sql = `
    SELECT m.id, m.scope_key, m.project_name, m.type, m.tags, m.aliases, m.updated_at,
           m.status, m.superseded_by, m.review_state, m.source, m.created_at,
           snippet(memories_fts, 0, '[', ']', ' ... ', 12) AS snippet,
           bm25(memories_fts) AS score
    FROM memories_fts
    JOIN memories m ON m.rowid = memories_fts.rowid
    WHERE memories_fts MATCH ?${scopeFilter}${typeFilter}
    ORDER BY score ASC
    LIMIT ?
  `;
    const params = [q];
    if (opts.scopeKeys && opts.scopeKeys.length > 0)
        params.push(...opts.scopeKeys);
    if (opts.type)
        params.push(opts.type);
    params.push(fetchLimit);
    const rows = db()
        .prepare(sql)
        .all(...params);
    // FTS5 bm25: lower = better; invert for intuition.
    const raw = rows.map((r) => ({ ...r, score: -r.score }));
    if (opts.stats)
        opts.stats.candidates = raw.length;
    return resolveVisible(raw, limit, opts.stats);
}
export function list(scopeKey, opts = {}) {
    const limit = Math.max(1, Math.min(opts.limit ?? 20, 100));
    const fetchLimit = Math.min(limit * 3 + 10, 150);
    const typeFilter = opts.type ? ` AND type = ?` : "";
    const sql = `
    SELECT id, scope_key, project_name, type, tags, updated_at, status,
           superseded_by, review_state, source, created_at, substr(content, 1, 240) AS snippet
    FROM memories
    WHERE scope_key = ?${typeFilter}
    ORDER BY updated_at DESC
    LIMIT ?
  `;
    const params = [scopeKey];
    if (opts.type)
        params.push(opts.type);
    params.push(fetchLimit);
    const rows = db()
        .prepare(sql)
        .all(...params);
    const raw = rows.map((r) => ({ ...r, score: 1 }));
    if (opts.include === "all") {
        // D68: the audit view — every row as stored (superseded versions,
        // retracted, archived), no chain collapsing, capped at limit.
        return raw.slice(0, limit).map(toHit);
    }
    return resolveVisible(raw, limit);
}
/**
 * True totals for the inventory's truncation disclosure (D68). Under
 * `active` the visible set is the newest of each chain, which in a
 * consistent store is exactly the rows whose status is active/deprecated.
 *
 * D70: one deliberate exception — a `superseded` row whose chain target is
 * missing (a dangling pointer, which the read path self-heals with a
 * warning) is returned as a visible hit by resolveVisible but is not
 * counted here, so the total can read one low in that state. Not worth
 * widening the query for a self-healing case; the disclosure stays honest
 * in every state a user can reach deliberately.
 */
export function countList(scopeKey, opts = {}) {
    const typeFilter = opts.type ? ` AND type = ?` : "";
    const statusFilter = opts.include === "all" ? "" : ` AND status IN ('active', 'deprecated')`;
    const params = [scopeKey];
    if (opts.type)
        params.push(opts.type);
    const row = db()
        .prepare(`SELECT COUNT(*) AS n FROM memories WHERE scope_key = ?${statusFilter}${typeFilter}`)
        .get(...params);
    return row.n;
}