akm-cli/dist/commands/improve/consolidate/continuity-check.js

akm (Agent Knowledge Manager) — a portable, local-first capability library for AI agents. Discover, load, share, and improve reusable skills, scripts, workflows, and knowledge across any shell-capable coding agent, including Claude Code, OpenCode, and Cur

// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
import { searchLocal } from "../../../indexer/search/db-search.js";
import { stripFrontmatterBody } from "../content-hash.js";
import { stripBundle } from "../ledger.js";
import { loadRetrievalQueries } from "../retrieval-gate.js";
/** At most this many of the retired asset's most recent queries are replayed (plan §5.4, §7). */
export const CONTINUITY_MAX_QUERIES = 5;
/** "Top 10" per the plan's rule — both the pass/fail cutoff and the search `limit`. */
export const CONTINUITY_TOP_N = 10;
/**
 * Builds the real search call the continuity check uses, stateful across
 * every query asked of ONE instance (S2): the first time a query falls back
 * to keyword-only ranking (`mode: "fts-fallback"` — most often a down or
 * unreachable embedding endpoint), every later call through THIS instance
 * forces `semanticSearchMode: "off"` instead of attempting semantic search
 * again, so a dead endpoint costs one failed attempt per run, not one per
 * remaining query (a hanging endpoint at ~3s/query, 300 proposals x 5
 * queries, would otherwise cost on the order of an hour). Construct exactly
 * one instance per pair-pass run and reuse it for every proposal judged, so
 * the throttle covers the whole run, not just one proposal's own queries —
 * and every call made once it has switched, across every remaining proposal
 * in the run, reports `forcedKeywordOnly: true`, not just the one call that
 * discovered the fallback (round-3 review: the first fix only marked THAT
 * call unverified, so a second proposal checked while the endpoint was
 * still down came back with a clean, `mode: "keyword"` — and therefore
 * bulk-acceptable — result).
 */
export function createContinuitySearch(stashDir, config) {
    const base = {
        searchType: "any",
        limit: CONTINUITY_TOP_N,
        stashDir,
        sources: [{ path: stashDir, isDefault: true }],
    };
    let keywordOnly = false;
    return async (query) => {
        const forcedKeywordOnly = keywordOnly;
        const callConfig = keywordOnly ? { ...config, semanticSearchMode: "off" } : config;
        const result = await searchLocal({ ...base, query, config: callConfig });
        if (result.mode === "fts-fallback")
            keywordOnly = true;
        return { hits: result.hits, mode: result.mode, forcedKeywordOnly };
    };
}
/** 1-indexed position of `conceptId` in `hits`, or `undefined` if it is not among them. */
function rankOf(hits, conceptId) {
    const index = hits.findIndex((hit) => stripBundle(hit.ref) === conceptId);
    return index === -1 ? undefined : index + 1;
}
/** Body only, whitespace collapsed — the same shape `db-search.ts`'s own content-dedupe compares (S3b). */
function normalizedBody(raw) {
    return stripFrontmatterBody(raw).replace(/\s+/g, " ").trim();
}
/**
 * Replay `retiredRef`'s own past queries and check that `successorRef` ranks
 * in the top {@link CONTINUITY_TOP_N} for every one where the retired asset
 * did. Returns `undefined` when there is nothing to flag: the two bodies are
 * content-identical, no recorded queries, or every query ran on the real
 * ranking and either the retired asset never ranked top 10 for it, or the
 * survivor always did too. Never throws.
 *
 * S3: two fixes against false flags measured on a real night-1 admission
 * (300 pairs, 6 flags, half spurious):
 *  - queries are the SAME cleaned set `loadRetrievalQueries` replays for the
 *    retrieval regression gate (`../retrieval-gate.ts`) — stash-README
 *    boilerplate, harness/tool envelopes, pastes over 2,000 characters, and
 *    duplicates are dropped before replay, not just capped at 5 raw entries;
 *  - when the retired and successor bodies normalize identical, the check
 *    never runs at all: search's own content-dedupe (`db-search.ts`) already
 *    hides the successor behind the retired asset for every such query, so a
 *    "successor missing from the top 10" finding here would not be a real
 *    risk, just that dedupe working as designed.
 *
 * S2: a query that never ran (the search call threw), ran on the
 * keyword-only fallback (`mode: "fts-fallback"` — the real ranking was
 * attempted and failed, most often a down or unreachable embedding
 * endpoint), or ran after the shared search instance had already switched
 * to forced keyword-only because an EARLIER query in the same run fell back
 * (`forcedKeywordOnly`) is "unverified" — it is dropped from the rank
 * comparison below (its hits cannot be trusted as "the ranking a user
 * actually gets"), but unlike a query the retired asset simply did not rank
 * for, it can never by itself lead to a silent `undefined` — at least one
 * unverified query always produces a `continuityRisk`, so a dead endpoint
 * reads as "risk unknown" for every proposal it touches that run, never as
 * "no risk found" for the ones checked after the first failure.
 */
export async function checkRetirementContinuity(args) {
    // S3b: identical bodies — search's own content-dedupe already hides the
    // successor for every query that would rank the retired asset, so there is
    // no real risk here to check for.
    if (normalizedBody(args.retiredRaw) === normalizedBody(args.successorRaw))
        return undefined;
    // S3a: the SAME cleaned queries the retrieval regression gate replays —
    // boilerplate, envelopes, pastes and duplicates dropped before replay.
    const queries = loadRetrievalQueries(args.ledgerAccess, args.retiredRef).slice(0, CONTINUITY_MAX_QUERIES);
    if (queries.length === 0)
        return undefined; // no queries recorded: no check, no flag
    const search = args.search ?? createContinuitySearch(args.stashDir, args.config);
    // N2: no rank-change-report abstraction — a query only ever needs "did the
    // retired asset rank top 10, and if so, did the successor too?", and
    // `rankOf` (search itself returning at most CONTINUITY_TOP_N hits) already
    // answers both directly.
    const ranks = [];
    let unverifiedQueries = 0;
    for (const query of queries) {
        let hits;
        try {
            const result = await search(query);
            if (result.mode === "fts-fallback" || result.forcedKeywordOnly) {
                unverifiedQueries++; // S2: never silently compare keyword-only ranks
                continue;
            }
            hits = result.hits;
        }
        catch {
            unverifiedQueries++; // S2: a query that never ran cannot be "no risk"
            continue;
        }
        const retiredRank = rankOf(hits, args.retiredRef);
        if (retiredRank === undefined)
            continue; // the retired asset itself did not rank top 10 here — nothing to protect
        const successorRank = rankOf(hits, args.successorRef);
        if (successorRank === undefined)
            ranks.push({ query, retiredRank, successorRank: null });
    }
    // Every query verified, and either the retired asset never ranked top 10
    // for any of them, or the survivor always did too: nothing to flag.
    if (unverifiedQueries === 0 && ranks.length === 0)
        return undefined;
    return {
        failingQueries: ranks.length,
        ranks,
        ...(unverifiedQueries > 0 ? { unverifiedQueries } : {}),
    };
}