// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
/**
 * The improve preparation stage: consolidation and session extraction (which
 * run before the loop), memory cleanup, structural validation, and candidate
 * selection for the reflect/distill loop.
 *
 * Candidate selection reads the improve ledger plus one set of signals: a ref
 * is eligible for a source when feedback newer than its last attempt landed and
 * no ledger window holds it, and reflect reads only negative feedback. Refs
 * without recent feedback can still be picked by the fallback lanes (proactive
 * maintenance, high salience), which only score them; the survivors are ranked
 * by salience, checked on disk and capped. A plan-only run evaluates the same
 * selectors against read snapshots and writes nothing.
 */
import fs from "node:fs";
import path from "node:path";
import { parseFrontmatter } from "../../core/asset/frontmatter.js";
import { daysToMs } from "../../core/common.js";
import { loadConfig } from "../../core/config/config.js";
import { ConfigError, rethrowIfTestIsolationError } from "../../core/errors.js";
import { appendEvent, readEvents } from "../../core/events.js";
import { withStateDb } from "../../core/state-db.js";
import { info, warn } from "../../core/warn.js";
import { countUsageEventsByType, USAGE_EVENT_RETENTION_DAYS } from "../../indexer/usage/usage-events.js";
import { runnerLlmConnection } from "../../integrations/agent/runner.js";
import { getAvailableHarnesses } from "../../integrations/session-logs/index.js";
import { getZeroResultSearches } from "../../storage/repositories/index-entries-repository.js";
import { getRetrievalCounts } from "../../storage/repositories/index-utility-repository.js";
import { listStateProposals } from "../../storage/repositories/proposals-repository.js";
import { akmLint } from "../lint/index.js";
import { runSchemaRepairPass } from "../sources/schema-repair.js";
import { isAutonomyLaneAllowed } from "./autonomy-gate.js";
import { akmConsolidate, inspectConsolidationPool, loadExistingKnowledgeBodyHashes, makeConsolidateResult, } from "./consolidate.js";
import { computeSafeChunkSize, DEFAULT_CONTEXT_LENGTH_TOKENS } from "./consolidate/chunking.js";
import { assetTypeOf, buildUtilityMap, dedupeRefs, findAssetFilePath, isDistillCandidateRef, isLessonCandidate, resolveImproveScope, withIndexDb, } from "./eligibility.js";
import { akmExtract, countNewExtractCandidates } from "./extract.js";
import { computeValenceScore } from "./feedback-valence.js";
import { isLedgerBlocked, lastAttemptByRef, ledgerRowFor, loadLedgerSnapshot, stateKey, } from "./ledger.js";
import { applyMemoryCleanup } from "./memory/memory-improve.js";
import { getAllAssetOutcomes, getAssetOutcome, getOutcomeScoresByRef, OUTCOME_SCORE_MAX, outcomeScoreToSalience, projectAssetOutcome, updateAssetOutcome, } from "./outcome-loop.js";
import { projectMemoryCleanup, selectEffectiveImproveRefs } from "./planner.js";
import { DEFAULT_DUE_DAYS, DEFAULT_MAX_PER_RUN, selectProactiveMaintenanceRefs } from "./proactive-maintenance.js";
import { isInRetrievalScope, loadRetrievalScope } from "./retrieval-scope.js";
import { computeSalience, getAssetSalience, getLastUseMsByRef, isContentEncodingRow, SALIENCE_NO_OP_DAMPEN_FACTOR, SALIENCE_NO_OP_DAMPEN_THRESHOLD, upsertAssetSalience, } from "./salience.js";
import { attributeStage, errMessage } from "./stage.js";
/** The candidate's durable state key (salience, outcome, ledger). */
const keyOf = (r) => stateKey(r.ref, r.itemRef);
/**
 * Run `fn` against the run's state.db (its long-lived handle when there is
 * one). A plan-only run without a handle reads nothing. Best-effort.
 */
function withRunState(eventsCtx, persist, fn) {
    if (!persist && !eventsCtx?.db)
        return undefined;
    try {
        return withStateDb(fn, { path: eventsCtx?.dbPath, borrowed: eventsCtx?.db });
    }
    catch (err) {
        rethrowIfTestIsolationError(err);
        return undefined;
    }
}
function fileSize(filePath) {
    if (!filePath)
        return undefined;
    try {
        return fs.statSync(filePath).size;
    }
    catch {
        return undefined;
    }
}
/** `{ key: value }` for each key `source` defines. */
export function pickDefined(source, keys) {
    const out = {};
    for (const key of keys)
        if (source?.[key] !== undefined)
            out[key] = source[key];
    return out;
}
export const CONSOLIDATION_CONFIG_KEYS = ["enabled", "minPoolSize", "limit", "maxChunkSize"];
/** Emit an aggregate `improve_skipped` row (never one per ref). */
export function recordImproveSkip(eventsCtx, ref, metadata) {
    appendEvent({ eventType: "improve_skipped", ref, metadata }, eventsCtx);
}
/** Per-originator rolling error windows (3 each) shown to later prompts as patterns to avoid. */
export function pushRecentError(recentErrors, originator, msg) {
    const window = recentErrors[originator] ?? [];
    window.push(msg);
    if (window.length > 3)
        window.shift();
    recentErrors[originator] = window;
}
// ── Consolidation ────────────────────────────────────────────────────────────
/**
 * The consolidation gates and pool, with no model call: the profile toggle,
 * `minPoolSize` (not for a named strategy or ref scope, nor once the pool is
 * over the 100-memory volume trigger), and the ledger delta (every memory
 * judged recently and unchanged since means nothing to do).
 */
function planConsolidationPass(args) {
    const { options, primaryStashDir, memorySummary, resolvedPlan } = args;
    const processConfig = args.improveProfile?.processes?.consolidate;
    const volumeTriggered = memorySummary.eligible > 100 && resolvedPlan.processes.consolidate.runner !== null;
    const minPoolSize = typeof processConfig?.minPoolSize === "number" ? processConfig.minPoolSize : 0;
    const eligiblePoolSize = typeof memorySummary.eligible === "number" ? memorySummary.eligible : 0;
    const userNamed = options.strategy !== undefined || resolveImproveScope(options.scope).mode === "ref";
    const poolBelowMinSize = !volumeTriggered && !userNamed && minPoolSize > 0 && eligiblePoolSize < minPoolSize;
    const pool = primaryStashDir
        ? inspectConsolidationPool({
            config: options.config,
            stashDir: options.stashDir,
            writeTarget: options.writeTarget,
            target: options.target,
            limit: processConfig?.limit,
            maxChunkSize: processConfig?.maxChunkSize,
        }, primaryStashDir, [], args.existingKnowledgeBodyHashes ?? loadExistingKnowledgeBodyHashes(primaryStashDir), { readOnly: args.eventsCtx?.readOnly === true })
        : { poolSize: 0, candidatePoolSize: 0, judgedUnchanged: 0 };
    // A credential-unavailable engine still resolved its context length.
    const unavailable = resolvedPlan.engineUnavailable.find((item) => item.process === "consolidate");
    const chunkSize = computeSafeChunkSize((resolvedPlan.processes.consolidate.runner &&
        runnerLlmConnection(resolvedPlan.processes.consolidate.runner)?.contextLength) ??
        unavailable?.contextLength ??
        DEFAULT_CONTEXT_LENGTH_TOKENS, 500, processConfig?.maxChunkSize);
    const profilePassed = processConfig?.enabled !== false;
    const deltaPassed = pool.candidatePoolSize > 0 || pool.judgedUnchanged === 0;
    const wouldRun = profilePassed && !poolBelowMinSize && deltaPassed && pool.candidatePoolSize > 0;
    const belowMin = `pool ${eligiblePoolSize} is below minPoolSize ${minPoolSize}`;
    const unchanged = "every memory was judged recently and is unchanged since";
    return {
        poolBelowMinSize,
        eligiblePoolSize,
        minPoolSize,
        plan: {
            configured: pickDefined(processConfig, CONSOLIDATION_CONFIG_KEYS),
            effective: {
                enabled: profilePassed,
                minPoolSize,
                ...(processConfig?.limit !== undefined ? { limit: processConfig.limit } : {}),
                chunkSize,
            },
            poolSize: pool.poolSize,
            candidatePoolSize: pool.candidatePoolSize,
            gates: {
                profile: {
                    passed: profilePassed,
                    reason: profilePassed ? "consolidation enabled" : "disabled by improve profile",
                },
                minimumPool: {
                    passed: !poolBelowMinSize,
                    reason: poolBelowMinSize ? belowMin : `pool satisfies minPoolSize ${minPoolSize}`,
                },
                delta: {
                    passed: deltaPassed,
                    reason: !deltaPassed
                        ? unchanged
                        : pool.judgedUnchanged > 0
                            ? `${pool.judgedUnchanged} recently judged, unchanged memories skipped`
                            : "no memory was judged recently",
                },
            },
            wouldRun,
            reason: !profilePassed
                ? "disabled by improve profile"
                : poolBelowMinSize
                    ? belowMin
                    : !deltaPassed
                        ? unchanged
                        : pool.candidatePoolSize === 0
                            ? "candidate pool is empty after narrowing"
                            : "all consolidation gates pass",
            estimatedChunks: wouldRun ? Math.ceil(pool.candidatePoolSize / chunkSize) : 0,
        },
    };
}
async function runConsolidationPass(args) {
    const { options, primaryStashDir, improveProfile, resolvedPlan, eventsCtx } = args;
    // Walked once and shared with the live pass.
    const existingKnowledgeBodyHashes = primaryStashDir ? loadExistingKnowledgeBodyHashes(primaryStashDir) : undefined;
    const planned = planConsolidationPass({ ...args, existingKnowledgeBodyHashes });
    const processConfig = improveProfile?.processes?.consolidate;
    let consolidation = makeConsolidateResult({ target: "", durationMs: 0 });
    if (!planned.plan.gates.profile.passed) {
        info("[improve] consolidation skipped (disabled by improve profile)");
    }
    else if (planned.poolBelowMinSize) {
        recordImproveSkip(eventsCtx, "memories/_consolidation", {
            reason: "pool_below_min_size",
            poolSize: planned.eligiblePoolSize,
            minPoolSize: planned.minPoolSize,
        });
        info(`[improve] consolidation skipped (pool ${planned.eligiblePoolSize} < minPoolSize ${planned.minPoolSize})`);
    }
    else if (!planned.plan.gates.delta.passed) {
        recordImproveSkip(eventsCtx, "memories/_consolidation", { reason: "consolidation_no_memory_updates" });
        info("[improve] consolidation skipped (every memory was judged recently and is unchanged)");
    }
    else {
        consolidation = await attributeStage(resolvedPlan, "consolidate", () => akmConsolidate({
            target: options.target,
            ...(options.writeTarget ? { writeTarget: options.writeTarget } : {}),
            config: options.config ?? loadConfig(),
            dryRun: options.dryRun ?? false,
            stashDir: options.stashDir,
            improveProfile,
            llmRunner: resolvedPlan.processes.consolidate.runner,
            existingKnowledgeBodyHashes,
            sourceRun: `consolidate-${Date.now()}`,
            limit: processConfig?.limit,
            maxChunkSize: processConfig?.maxChunkSize,
            signal: args.budgetSignal,
            p90ChunkSecondsDefault: processConfig?.p90ChunkSecondsDefault,
            // Its read-only proposal lookups go through the run's own state.db handle.
            ...(eventsCtx?.db ? { proposalsCtx: { db: eventsCtx.db } } : {}),
        }));
    }
    return { consolidation, plan: planned.plan };
}
/** The extract gates, evaluated once for both the live pass and the dry-run report. */
function inspectExtractPass(args, readOnly) {
    const { options, improveProfile, resolvedPlan, eventsCtx } = args;
    const enabled = resolvedPlan.processes.extract.enabled;
    const hasRunner = resolvedPlan.processes.extract.runner?.engine !== undefined;
    const availableHarnesses = (options.extractHarnesses ?? getAvailableHarnesses()).filter((h) => h.isAvailable());
    const configured = improveProfile.processes?.extract?.minNewSessions;
    const minNewSessions = typeof configured === "number" ? configured : 0;
    let newCandidateCount;
    if (enabled && hasRunner && availableHarnesses.length > 0 && minNewSessions > 0) {
        const defaultSince = improveProfile.processes?.extract?.defaultSince;
        newCandidateCount = (options.extractCandidateCountFn ?? countNewExtractCandidates)(options.config ?? loadConfig(), {
            harnesses: availableHarnesses,
            improveProfile,
            ...(defaultSince ? { since: defaultSince } : {}),
            ...(eventsCtx?.db ? { stateDb: eventsCtx.db } : {}),
            ...(!readOnly && eventsCtx?.dbPath ? { stateDbPath: eventsCtx.dbPath } : {}),
            ...(readOnly ? { readOnly: true } : {}),
        });
    }
    const belowMinNewSessions = minNewSessions > 0 && newCandidateCount !== undefined && newCandidateCount < minNewSessions;
    const count = `${newCandidateCount ?? 0} new sessions`;
    return {
        availableHarnesses,
        minNewSessions,
        ...(newCandidateCount !== undefined ? { newCandidateCount } : {}),
        belowMinNewSessions,
        wouldRun: enabled && hasRunner && availableHarnesses.length > 0 && !belowMinNewSessions,
        reason: !enabled
            ? "disabled"
            : !hasRunner
                ? "enabled but no runner is resolved"
                : availableHarnesses.length === 0
                    ? "enabled but no session-log harness is available"
                    : belowMinNewSessions
                        ? `${count} is below minNewSessions ${minNewSessions}`
                        : minNewSessions > 0
                            ? `${count} satisfies minNewSessions ${minNewSessions}`
                            : `enabled with ${availableHarnesses.length} available session-log harness(es); minNewSessions is disabled`,
    };
}
/**
 * One `akmExtract` per available harness under the strategy's frozen plan. A
 * harness that throws is a warning; the `minNewSessions` gate skips the whole
 * pass with no model call.
 */
async function runSessionExtractPass(args, plan) {
    const { options, primaryStashDir, resolvedPlan, eventsCtx } = args;
    const warnings = [];
    if (!resolvedPlan.processes.extract.enabled)
        return { warnings };
    const runner = resolvedPlan.processes.extract.runner;
    if (!runner?.engine) {
        throw new ConfigError("Resolved improve plan has no runner for enabled extract process.", "LLM_NOT_CONFIGURED");
    }
    const config = options.config ?? loadConfig();
    const extractPlan = Object.freeze({
        strategy: resolvedPlan.strategy.name,
        engine: runner.engine,
        enabled: true,
        process: resolvedPlan.processes.extract.config,
        runner,
        timeoutMs: runner.timeoutMs === undefined ? 600_000 : runner.timeoutMs,
        embeddingConfig: Object.freeze(structuredClone(config.embedding)),
        ...(resolvedPlan.processes.extract.notices?.length ? { notices: resolvedPlan.processes.extract.notices } : {}),
    });
    if (plan.belowMinNewSessions) {
        recordImproveSkip(eventsCtx, "memories/_extract", {
            reason: "below_min_new_sessions",
            newSessions: plan.newCandidateCount ?? 0,
            minNewSessions: plan.minNewSessions,
        });
        info(`[improve] extract skipped (new sessions ${plan.newCandidateCount ?? 0} < minNewSessions ${plan.minNewSessions})`);
    }
    if (!plan.wouldRun)
        return { warnings };
    const extractResults = [];
    for (const harness of plan.availableHarnesses) {
        try {
            extractResults.push(await attributeStage(resolvedPlan, "extract", () => akmExtract({
                type: harness.name,
                ...(primaryStashDir !== undefined ? { stashDir: primaryStashDir } : {}),
                config,
                resolvedPlan: extractPlan,
                dryRun: options.dryRun ?? false,
                signal: args.budgetSignal,
                ...(options.extractHarnesses ? { harnesses: options.extractHarnesses } : {}),
                ...(eventsCtx?.dbPath ? { stateDbPath: eventsCtx.dbPath } : {}),
                eventsCtx,
            })));
        }
        catch (err) {
            warnings.push(`extract(${harness.name}) failed: ${errMessage(err)}`);
        }
    }
    // Every harness threw: no `extract` field rather than a misleadingly empty one.
    return { ...(extractResults.length > 0 ? { extractResults } : {}), warnings };
}
// ── Validation ───────────────────────────────────────────────────────────────
/**
 * Structural validation (file on disk, lesson description) with optional LLM
 * schema repair. A repair is advisory: a ref leaves the failure set only when a
 * fresh read of the live asset passes.
 */
export async function runValidationAndRepairPass(args) {
    const { postCleanupRefs, options, resolvedPlan, repairValidationFailures } = args;
    const validate = async (candidate) => {
        try {
            const filePath = candidate.filePath && fs.existsSync(candidate.filePath)
                ? candidate.filePath
                : await findAssetFilePath(candidate.ref, options.stashDir);
            if (!filePath)
                return "file not found on disk";
            if (path.extname(filePath).toLowerCase() !== ".md")
                return undefined;
            if (isLessonCandidate(candidate.ref) && !parseFrontmatter(fs.readFileSync(filePath, "utf8")).data.description) {
                return "missing description";
            }
            return undefined;
        }
        catch (error) {
            return String(error);
        }
    };
    const validationFailures = [];
    for (const candidate of postCleanupRefs) {
        const reason = await validate(candidate);
        if (reason)
            validationFailures.push({ ref: candidate.ref, reason });
    }
    if (validationFailures.length > 0) {
        info(`[improve] ${validationFailures.length} assets have validation issues${repairValidationFailures ? " (will attempt schema repair)" : ""}:`);
        for (const f of validationFailures)
            info(`  ${f.ref}: ${f.reason}`);
    }
    let schemaRepairs = [];
    const repaired = new Set();
    const runner = resolvedPlan.processes.validation.runner;
    if (repairValidationFailures && validationFailures.length > 0 && runner) {
        const result = await attributeStage(resolvedPlan, "validation", () => (args.schemaRepairFn ?? runSchemaRepairPass)(validationFailures, {
            startMs: args.startMs,
            budgetMs: args.budgetMs,
            llmRunner: runner,
            // The resolved source path, not the raw `--stash-dir` flag.
            stashDir: args.primaryStashDir,
            findFilePath: findAssetFilePath,
            isLessonCandidateFn: isLessonCandidate,
        }));
        schemaRepairs = result.repairs;
        const byRef = new Map(postCleanupRefs.map((candidate) => [candidate.ref, candidate]));
        for (const { ref } of validationFailures) {
            const candidate = byRef.get(ref);
            if (candidate && !(await validate(candidate)))
                repaired.add(ref);
        }
    }
    const validationFailureRefs = new Set(validationFailures.filter((f) => !repaired.has(f.ref)).map((f) => f.ref));
    if (repaired.size > 0) {
        info(`[improve] schema repair fixed ${repaired.size}/${validationFailures.length} validation failures; ${validationFailureRefs.size} remain`);
    }
    return { validationFailures, validationFailureRefs, schemaRepairs };
}
export async function runImprovePreparationStage(args) {
    const { scope, options, plannedRefs, memoryCleanupPlan, primaryStashDir, eventsCtx, resolvedPlan } = args;
    const planOnly = args.planOnly ?? options.dryRun === true;
    const persist = !planOnly;
    const actions = [];
    const cleanupWarnings = [...(args.initialCleanupWarnings ?? [])];
    const memoryIndexHealth = assessMemoryIndex(primaryStashDir, cleanupWarnings);
    // Consolidation precedes extract, so it only judges memories from earlier runs.
    const consolidationPass = planOnly
        ? {
            consolidation: makeConsolidateResult({
                dryRun: true,
                previewOnly: true,
                target: options.target ?? options.stashDir ?? "",
                durationMs: 0,
            }),
            plan: planConsolidationPass(args).plan,
        }
        : await runConsolidationPass(args);
    const extractPlan = inspectExtractPass(args, planOnly);
    const extractPass = planOnly ? { warnings: [] } : await runSessionExtractPass(args, extractPlan);
    cleanupWarnings.push(...extractPass.warnings);
    if (persist) {
        appendEvent({
            eventType: "improve_invoked",
            ref: scope.mode === "ref" ? scope.value : `improve:${scope.mode}:${scope.value ?? "all"}`,
            metadata: {
                strategy: args.strategyName,
                scope,
                dryRun: options.dryRun ?? false,
                eligibleCount: plannedRefs.length,
            },
        }, eventsCtx);
    }
    // Memory cleanup: archive redundant derived memories (autonomy-gated).
    const allowCleanup = isAutonomyLaneAllowed("memoryCleanup", options.config ?? loadConfig());
    let appliedCleanup;
    if (persist) {
        try {
            appliedCleanup =
                primaryStashDir && memoryCleanupPlan && allowCleanup
                    ? applyMemoryCleanup(primaryStashDir, memoryCleanupPlan)
                    : undefined;
        }
        catch (err) {
            cleanupWarnings.push(`applyMemoryCleanup failed: ${errMessage(err)}`);
        }
    }
    const cleanup = planOnly
        ? projectMemoryCleanup({
            mode: "estimate",
            plannedRefs,
            candidateRefs: memoryCleanupPlan?.pruneCandidates.map((candidate) => candidate.ref) ?? [],
            allowApply: allowCleanup,
        })
        : projectMemoryCleanup({
            mode: "execution",
            plannedRefs,
            archivedRefs: appliedCleanup?.archived.map((record) => record.ref) ?? [],
            allowApply: allowCleanup,
        });
    if (appliedCleanup) {
        for (const candidate of memoryCleanupPlan?.pruneCandidates ?? []) {
            if (!appliedCleanup.archived.some((record) => record.ref === candidate.ref))
                continue;
            actions.push({
                ref: candidate.ref,
                mode: "memory-prune",
                result: { ok: true, pruned: true, reason: candidate.reason },
            });
        }
        if ((appliedCleanup.archived.length > 0 || appliedCleanup.beliefStateTransitions.length > 0) && primaryStashDir) {
            try {
                await args.reindexFn({ stashDir: primaryStashDir, signal: args.budgetSignal });
            }
            catch (err) {
                cleanupWarnings.push(`reindex after cleanup failed: ${errMessage(err)}`);
            }
        }
    }
    const { postCleanupRefs } = cleanup;
    const { validationFailures, validationFailureRefs, schemaRepairs } = await runValidationAndRepairPass({
        postCleanupRefs,
        options,
        startMs: args.startMs,
        budgetMs: args.budgetMs,
        primaryStashDir,
        resolvedPlan,
        repairValidationFailures: persist && resolvedPlan.processes.validation.enabled && options.repairValidationFailures !== false,
    });
    let lintSummary;
    if (primaryStashDir) {
        try {
            const lintResult = await akmLint({ fix: false, dir: primaryStashDir });
            lintSummary = { fixed: lintResult.summary.fixed, flagged: lintResult.summary.flagged };
        }
        catch {
            // lint never blocks improve
        }
    }
    // Schema-repair errors get their own window; they are never shown to reflect.
    const recentErrors = {};
    for (const repair of schemaRepairs) {
        if (repair.outcome !== "error")
            continue;
        pushRecentError(recentErrors, "schema-repair", repair.error ?? `schema repair error: ${repair.reason}`);
    }
    const selection = await selectLoopCandidates(args, postCleanupRefs, validationFailureRefs, actions, persist);
    return {
        actions,
        cleanupWarnings,
        appliedCleanup,
        memoryIndexHealth,
        extract: extractPass.extractResults,
        actionableRefs: selection.actionableRefs,
        signalBearingSet: selection.signalBearingSet,
        validationFailures,
        schemaRepairs,
        lintSummary,
        loopRefs: selection.loopRefs,
        distillCooledRefs: selection.distillCooledRefs,
        distillOnlyRefs: selection.distillOnlyRefs,
        coverageGaps: selection.coverageGaps,
        recentErrors,
        consolidation: consolidationPass.consolidation,
        ...(selection.proactive.proactiveMaintenanceSummary
            ? { proactiveMaintenance: selection.proactive.proactiveMaintenanceSummary }
            : {}),
        planning: {
            gates: [
                cleanup.gate,
                { name: "validation", removed: validationFailureRefs.size, reason: "structural validation failures" },
                ...selection.gates,
            ],
            ...(selection.proactive.proactivePlan ? { proactive: selection.proactive.proactivePlan } : {}),
            consolidation: consolidationPass.plan,
            extract: { wouldRun: extractPlan.wouldRun, reason: extractPlan.reason },
        },
    };
}
/** MEMORY.md line budget: warn at 180 of 200 lines. */
function assessMemoryIndex(primaryStashDir, warnings) {
    if (!primaryStashDir)
        return undefined;
    const memoryMdPath = path.join(primaryStashDir, "memories", "MEMORY.md");
    if (!fs.existsSync(memoryMdPath))
        return undefined;
    try {
        const lineCount = fs.readFileSync(memoryMdPath, "utf8").split("\n").length;
        if (lineCount >= 180) {
            warnings.push(`MEMORY.md has ${lineCount} lines (budget: 200). Consolidation strongly recommended.`);
        }
        return { lineCount, overBudget: lineCount >= 180 };
    }
    catch {
        return undefined;
    }
}
// ── Candidate selection ──────────────────────────────────────────────────────
const FEEDBACK_SIGNAL_WINDOW_DAYS = 30;
/** Feedback that counts as a signal carries a signal or a note (a bare `akm feedback` does not). */
function isSignalEvent(metadata) {
    const meta = metadata;
    return meta !== undefined && (typeof meta.signal === "string" || typeof meta.note === "string");
}
/** One read of the feedback events and the ledger's reflect/distill rows. */
export function buildSnapshotManifest(args) {
    const { eventsCtx, stashDir } = args;
    const feedbackSinceCutoff = new Date(Date.now() - daysToMs(FEEDBACK_SIGNAL_WINDOW_DAYS)).toISOString();
    const candidates = args.postCleanupRefs.filter((r) => !args.validationFailureRefs.has(r.ref));
    const refByKey = new Map(candidates.map((r) => [keyOf(r), r.ref]));
    const latestFeedbackTs = new Map();
    const latestNegativeTs = new Map();
    const feedback = new Map(candidates.map((r) => [r.ref, { hasSignal: false, positive: 0, negative: 0 }]));
    if (candidates.length > 0) {
        for (const e of readEvents({ type: "feedback" }, eventsCtx).events) {
            const ref = e.ref ? refByKey.get(e.ref) : undefined;
            const entry = ref ? feedback.get(ref) : undefined;
            if (!ref || !entry)
                continue;
            const ts = e.ts ?? "";
            const signal = e.metadata?.signal;
            if (ts >= feedbackSinceCutoff && isSignalEvent(e.metadata)) {
                entry.hasSignal = true;
                if (ts > (latestFeedbackTs.get(ref) ?? ""))
                    latestFeedbackTs.set(ref, ts);
                if (signal === "negative" && ts > (latestNegativeTs.get(ref) ?? ""))
                    latestNegativeTs.set(ref, ts);
            }
            if (signal === "positive")
                entry.positive++;
            else if (signal === "negative")
                entry.negative++;
        }
    }
    const ledger = stashDir
        ? loadLedgerSnapshot({ eventsCtx, ...(args.readOnly ? { readOnly: true } : {}) }, stashDir, ["reflect", "distill"])
        : new Map();
    return {
        feedbackSinceCutoff,
        nowIso: new Date().toISOString(),
        latestFeedbackTs,
        latestNegativeTs,
        ledger,
        lastReflectAttemptAt: lastAttemptByRef(ledger, "reflect", candidates),
        lastDistillAttemptAt: lastAttemptByRef(ledger, "distill", candidates),
        feedback,
    };
}
/**
 * Partition the post-cleanup refs against the ledger:
 *  - eligibleRefs: reflect's signal delta passes, which only fresh negative
 *    feedback can do (distill may still be cooled);
 *  - distillOnlyRefs: only distill's passes (any signal, a positive or a note
 *    included), on a distill candidate;
 *  - noFeedbackPool: no recent feedback and no reflect window, left to the
 *    fallback lanes, which only score them;
 *  - fullySkippedCount: feedback on record but nothing new, or a live window.
 * An explicit `--scope <ref>` bypasses every gate.
 */
export function partitionBySignalDelta(args) {
    const { postCleanupRefs, validationFailureRefs } = args;
    const { latestFeedbackTs, latestNegativeTs, ledger, nowIso } = args.snapshot;
    // Newer feedback lifts a revisit window, never a rejection. Reflect reads negative feedback only.
    const deltaPasses = (candidate, source) => {
        const feedbackAt = (source === "reflect" ? latestNegativeTs : latestFeedbackTs).get(candidate.ref);
        if (!feedbackAt)
            return false;
        const row = ledgerRowFor(ledger, source, candidate.ref, candidate.itemRef);
        return feedbackAt > (row?.lastAttemptAt ?? "") && !isLedgerBlocked(row, nowIso, feedbackAt);
    };
    const out = {
        distillCooledRefs: new Set(),
        preCooldownCount: postCleanupRefs.length,
        eligibleRefs: [],
        distillOnlyRefs: [],
        noFeedbackPool: [],
        fullySkippedCount: 0,
    };
    for (const r of postCleanupRefs) {
        if (validationFailureRefs.has(r.ref))
            continue;
        if (args.scope.mode === "ref") {
            out.eligibleRefs.push(r);
            continue;
        }
        const reflectOk = deltaPasses(r, "reflect");
        const distillOk = deltaPasses(r, "distill");
        if (reflectOk) {
            if (!distillOk)
                out.distillCooledRefs.add(r.ref);
            out.eligibleRefs.push(r);
        }
        else if (distillOk && isDistillCandidateRef(r.ref, args.options.stashDir)) {
            out.distillOnlyRefs.push(r);
        }
        else if (!latestFeedbackTs.has(r.ref) &&
            !isLedgerBlocked(ledgerRowFor(ledger, "reflect", r.ref, r.itemRef), nowIso)) {
            out.noFeedbackPool.push(r);
        }
        else {
            out.fullySkippedCount++;
        }
    }
    return out;
}
/**
 * Pick the loop's refs: signal delta, the fallback lanes (unless
 * `--require-feedback-signal`; they score, never plan), lane attribution,
 * salience, the no-op-dampened ranking, the disk check and the limit.
 */
async function selectLoopCandidates(args, postCleanupRefs, validationFailureRefs, actions, persist) {
    const { scope, options, primaryStashDir, eventsCtx, improveProfile } = args;
    const snapshot = buildSnapshotManifest({
        postCleanupRefs,
        validationFailureRefs,
        eventsCtx,
        stashDir: primaryStashDir ?? options.stashDir,
        readOnly: !persist,
    });
    const partition = partitionBySignalDelta({ scope, options, postCleanupRefs, validationFailureRefs, snapshot });
    const processableRefs = [...partition.eligibleRefs, ...partition.distillOnlyRefs];
    const signalFiltered = processableRefs.filter((c) => snapshot.feedback.get(c.ref)?.hasSignal === true);
    const signalBearingSet = new Set(signalFiltered.map((r) => r.ref));
    // The fallback lanes (proactive, high salience) have no usage evidence of
    // their own: they pick only what retrieval returned or new material improve
    // never processed (#986). Evaluated once per candidate.
    const fallbackEligible = postCleanupRefs.filter((c) => !validationFailureRefs.has(c.ref));
    const allowFallbacks = options.requireFeedbackSignal !== true;
    const retrievalScope = scope.mode === "ref" || !allowFallbacks
        ? undefined
        : loadRetrievalScope({ eventsCtx, ...(persist ? {} : { readOnly: true }) }, primaryStashDir ?? options.stashDir);
    const unscoped = new Set(fallbackEligible.filter((c) => !isInRetrievalScope(retrievalScope, c.ref, c.filePath)).map((c) => c.ref));
    const noFeedbackPool = dedupeRefs([
        ...processableRefs.filter((r) => !signalBearingSet.has(r.ref)),
        ...partition.noFeedbackPool,
    ]);
    const noFeedbackCandidates = noFeedbackPool.filter((r) => !unscoped.has(r.ref));
    // Only a ref no fallback lane may pick anymore is charged to the retrieval gate.
    const outOfScope = new Set(noFeedbackPool.filter((r) => unscoped.has(r.ref)).map((r) => r.ref));
    const retrieval = fetchRetrievalSignals(options, signalFiltered, noFeedbackCandidates, eventsCtx, persist);
    const proactive = allowFallbacks
        ? selectProactiveMaintenanceLane(args, noFeedbackCandidates, snapshot, retrieval, persist)
        : { proactiveRefs: [] };
    const highSalienceRefs = allowFallbacks
        ? selectHighSalienceLane(options, improveProfile, eventsCtx, noFeedbackCandidates.filter((r) => !proactive.proactiveRefs.some((p) => p.ref === r.ref)), snapshot.lastReflectAttemptAt, persist)
        : [];
    // An explicit ref scope always acts on its ref; otherwise only feedback plans the loop. The fallback lanes'
    // picks are scored with it, never planned: a rewrite needs negative feedback.
    const signalAndRetrievalRefs = dedupeRefs([...signalFiltered, ...proactive.proactiveRefs, ...highSalienceRefs]);
    const mergedRefs = scope.mode === "ref" ? processableRefs : signalFiltered;
    const scoredRefs = scope.mode === "ref" ? processableRefs : signalAndRetrievalRefs;
    // Lane attribution: signal-delta, and an explicit ref scope over it.
    const sourceByRef = new Map();
    for (const r of signalFiltered)
        sourceByRef.set(r.ref, "signal-delta");
    if (scope.mode === "ref")
        for (const r of processableRefs)
            sourceByRef.set(r.ref, "scope");
    for (const r of mergedRefs)
        r.eligibilitySource = sourceByRef.get(r.ref) ?? "unknown";
    const salienceMap = scoreSalience(args, scoredRefs, snapshot.feedback, retrieval.retrievalCounts, persist);
    // Rank by salience; a ref skipped as a no-op repeatedly sorts lower (its stored rank is untouched).
    const noOps = new Map();
    withRunState(eventsCtx, persist, (db) => {
        for (const r of mergedRefs)
            noOps.set(r.ref, getAssetSalience(db, keyOf(r))?.consecutive_no_ops ?? 0);
    });
    const effectiveScore = (ref) => {
        const rank = salienceMap.get(ref)?.rankScore ?? 0;
        return (noOps.get(ref) ?? 0) >= SALIENCE_NO_OP_DAMPEN_THRESHOLD ? rank * SALIENCE_NO_OP_DAMPEN_FACTOR : rank;
    };
    const sorted = [...mergedRefs].sort((a, b) => effectiveScore(b.ref) - effectiveScore(a.ref) || (a.ref < b.ref ? -1 : a.ref > b.ref ? 1 : 0));
    const coverageGaps = withIndexDb(!persist, getZeroResultSearches) ?? [];
    const { actionableRefs, missing } = await dropRefsMissingOnDisk(sorted, options, eventsCtx, persist);
    const selection = selectEffectiveImproveRefs({
        rankedRefs: actionableRefs,
        distillOnlyRefs: partition.distillOnlyRefs,
        limit: options.limit,
    });
    if (signalAndRetrievalRefs.length > 0) {
        info(`[improve] ${signalAndRetrievalRefs.length} refs with usage signals (${signalFiltered.length} feedback)`);
    }
    if (validationFailureRefs.size > 0)
        info(`[improve] ${validationFailureRefs.size} with validation failures excluded`);
    if (persist && missing.length > 0)
        info(`[improve] ${missing.length} candidates dropped — file not on disk`);
    const deferred = actionableRefs.length - selection.loopRefs.length;
    info(`[improve] ${actionableRefs.length} actionable; ${selection.loopRefs.length} will be processed` +
        (options.limit && deferred > 0 ? ` (--limit ${options.limit} applied; ${deferred} deferred)` : ""));
    // Skip observability: every candidate that did not survive into the loop pool is reported skipped.
    const survivors = new Set(sorted.map((c) => c.ref));
    const skipped = fallbackEligible.filter((c) => !survivors.has(c.ref));
    const retrievalSkipped = skipped.filter((c) => outOfScope.has(c.ref));
    const signalSkipped = skipped.filter((c) => !outOfScope.has(c.ref));
    for (const ref of partition.distillCooledRefs) {
        actions.push({ ref, mode: "distill-skipped", result: { ok: true, reason: "distill signal-delta" } });
        if (persist)
            recordImproveSkip(eventsCtx, ref, { reason: "distill_no_new_signal" });
    }
    for (const candidate of skipped) {
        actions.push({
            ref: candidate.ref,
            mode: "distill-skipped",
            result: {
                ok: true,
                reason: outOfScope.has(candidate.ref)
                    ? "not retrieved inside the usage window"
                    : "no new signal since last proposal",
            },
        });
    }
    if (persist && signalSkipped.length > 0) {
        recordImproveSkip(eventsCtx, undefined, { reason: "no_new_signal", count: signalSkipped.length });
    }
    if (persist && retrievalSkipped.length > 0) {
        recordImproveSkip(eventsCtx, undefined, { reason: "not_retrieved", count: retrievalSkipped.length });
    }
    const blocked = signalSkipped.length + partition.distillOnlyRefs.length;
    if (blocked > 0) {
        info(`[improve] ${blocked} of ${partition.preCooldownCount} indexed refs blocked by reflect signal-delta ` +
            `(${signalSkipped.length} fully skipped, ${partition.distillOnlyRefs.length} routed to distill-only)`);
    }
    if (retrievalSkipped.length > 0) {
        info(`[improve] ${retrievalSkipped.length} refs left out: not retrieved in the last ${USAGE_EVENT_RETENTION_DAYS} days, and not new material`);
    }
    const gates = [
        {
            name: "retrieval",
            removed: retrievalSkipped.length,
            reason: `no feedback, and neither returned by search, curate or show in the last ${USAGE_EVENT_RETENTION_DAYS} days nor new material improve never processed`,
        },
        {
            name: "signal",
            removed: signalSkipped.length,
            reason: "no fresh negative feedback (or, for distill, any signal) since the last attempt, or a ledger window holds it",
        },
        { name: "disk", removed: missing.length, reason: "backing asset is absent on disk" },
        { name: "limit", removed: selection.limitRemoved, reason: "deferred by the effective run limit" },
    ];
    return {
        actionableRefs,
        loopRefs: selection.loopRefs,
        distillOnlyRefs: selection.distillOnlyRefs,
        distillCooledRefs: partition.distillCooledRefs,
        signalBearingSet,
        coverageGaps,
        gates,
        proactive,
    };
}
/** Retrieval counts for every candidate, and last-use times for the zero-feedback pool. */
function fetchRetrievalSignals(options, signalFiltered, noFeedbackCandidates, eventsCtx, persist) {
    const out = { retrievalCounts: new Map(), lastUseMs: new Map() };
    withIndexDb(!persist, (indexDb) => {
        // usage_events live in state.db, entries in index.db.
        withRunState(eventsCtx, persist, (stateDb) => {
            if (countUsageEventsByType(stateDb, "show") === 0) {
                warn("Warning: show events not yet in usage_events — the retrieval scope will match only search-retrieved assets.");
            }
            const refs = [...new Set([...signalFiltered, ...noFeedbackCandidates].map((r) => r.ref))];
            out.retrievalCounts = getRetrievalCounts(indexDb, stateDb, refs, { sourceName: options.sourceName });
        });
        out.lastUseMs = getLastUseMsByRef(indexDb, noFeedbackCandidates);
    });
    return out;
}
/**
 * Proactive maintenance (default off, whole-stash/type runs): pick stable
 * assets due for a revisit. The picks are scored and never planned: improve
 * does not rewrite on a proactive cadence. The due gate doubles as the
 * rotation cooldown: a freshly reflected asset waits `dueDays` before it is
 * picked again.
 */
function selectProactiveMaintenanceLane(args, candidates, snapshot, retrieval, persist) {
    if (args.scope.mode === "ref" || !args.resolvedPlan.processes.proactiveMaintenance.enabled) {
        return { proactiveRefs: [] };
    }
    const pmCfg = args.improveProfile.processes?.proactiveMaintenance;
    const dueDays = pmCfg?.dueDays ?? DEFAULT_DUE_DAYS;
    const maxPerRun = pmCfg?.maxPerRun ?? pmCfg?.limit ?? DEFAULT_MAX_PER_RUN;
    const selection = selectProactiveMaintenanceRefs({
        candidates,
        lastReflectTs: snapshot.lastReflectAttemptAt,
        lastDistillTs: snapshot.lastDistillAttemptAt,
        retrievalCounts: retrieval.retrievalCounts,
        lastUseMs: retrieval.lastUseMs,
        sizeBytesOf: (r) => fileSize(r.filePath),
        dueDays,
        maxPerRun,
    });
    const summary = {
        selected: selection.selected.length,
        dueTotal: selection.dueTotal,
        neverReflected: selection.neverReflected,
    };
    if (persist) {
        appendEvent({
            eventType: "proactive_selected",
            ref: undefined,
            metadata: { count: summary.selected, dueTotal: summary.dueTotal, neverReflected: summary.neverReflected },
        }, args.eventsCtx);
    }
    if (summary.selected > 0) {
        info(`[improve] proactive maintenance selected ${summary.selected}/${summary.dueTotal} due refs ` +
            `(${summary.neverReflected} never reflected, dueDays=${dueDays}, maxPerRun=${maxPerRun})`);
    }
    const selectedRefs = selection.selected.map((entry) => entry.ref);
    return {
        proactiveRefs: selection.selected,
        proactiveMaintenanceSummary: { ...summary, selectedRefs },
        proactivePlan: {
            configured: pickDefined(pmCfg, ["dueDays", "maxPerRun", "limit"]),
            effective: { dueDays, maxPerRun },
            candidatePool: candidates.length,
            ...summary,
            selectedRefs,
        },
    };
}
/**
 * High salience: zero-feedback refs whose content-derived encoding score (not
 * a per-type stub) reaches `salienceThreshold` and that were never reflected,
 * top-N by score, capped at 10% of the effective limit. The picks are scored,
 * never planned.
 */
function selectHighSalienceLane(options, improveProfile, eventsCtx, candidates, lastReflectAttemptAt, persist) {
    const threshold = (options.config ?? loadConfig()).improve?.salience?.salienceThreshold ?? 0.75;
    const effectiveLimit = options.limit ?? improveProfile?.processes?.reflect?.limit ?? improveProfile.limit ?? 10;
    const selected = withRunState(eventsCtx, persist, (db) => candidates
        .flatMap((r) => {
        const row = getAssetSalience(db, keyOf(r));
        return row &&
            isContentEncodingRow(row) &&
            row.encoding_salience >= threshold &&
            !lastReflectAttemptAt.has(r.ref)
            ? [{ ref: r, score: row.encoding_salience }]
            : [];
    })
        .sort((a, b) => b.score - a.score)
        .slice(0, Math.max(1, Math.floor(effectiveLimit * 0.1)))
        .map((q) => q.ref)) ?? [];
    if (selected.length > 0) {
        info(`[improve] high-salience lane admitted ${selected.length} content-scored ref(s) ` +
            `(threshold=${threshold}, requires content-derived encoding_source)`);
    }
    return selected;
}
/**
 * Score the merged refs: update `asset_outcome` (projected on a plan-only
 * run), compute each salience vector (keeping a stored content-derived
 * encoding score), then persist it.
 */
function scoreSalience(args, mergedRefs, feedback, retrievalCounts, persist) {
    const { options, eventsCtx } = args;
    const utilityMap = buildUtilityMap(mergedRefs, !persist);
    const lastUseMsByRef = withIndexDb(!persist, (db) => getLastUseMsByRef(db, mergedRefs)) ?? new Map();
    const outcomeSalience = updateOutcomeScores({
        mergedRefs,
        feedback,
        retrievalCounts,
        lastUseMsByRef,
        utilityMap,
        primaryStashDir: args.primaryStashDir,
        eventsCtx,
        persist,
    });
    const outcomeWeightEnabled = (options.config ?? loadConfig()).improve?.salience?.outcomeWeightEnabled !== false;
    const storedEncoding = new Map();
    withRunState(eventsCtx, persist, (db) => {
        for (const r of mergedRefs) {
            const row = getAssetSalience(db, keyOf(r));
            if (row && isContentEncodingRow(row))
                storedEncoding.set(r.ref, row.encoding_salience);
        }
    });
    const now = Date.now();
    const salienceMap = new Map();
    for (const r of mergedRefs) {
        const encoding = storedEncoding.get(r.ref);
        salienceMap.set(r.ref, computeSalience({
            ref: r.ref,
            type: assetTypeOf(r.ref),
            ...(encoding !== undefined ? { encodingSalience: encoding } : {}),
            retrievalFreq: retrievalCounts.get(r.ref) ?? 0,
            lastUseMs: lastUseMsByRef.get(r.ref),
            utilityScore: utilityMap.get(r.ref),
            outcomeSalience: outcomeSalience.get(r.ref),
            sizeBytes: fileSize(r.filePath),
            now,
            outcomeWeightEnabled,
        }));
    }
    if (persist) {
        withRunState(eventsCtx, persist, (db) => {
            for (const r of mergedRefs)
                upsertAssetSalience(db, keyOf(r), salienceMap.get(r.ref), now);
        });
    }
    return salienceMap;
}
/**
 * Update each ref's outcome row and return its outcome salience, normalized
 * against the stash-wide maximum. Without state.db on a plan-only run, the
 * values a live run would insert are projected instead.
 */
function updateOutcomeScores(args) {
    const { mergedRefs, feedback, eventsCtx, persist } = args;
    const now = Date.now();
    const inputsFor = (r, accepted) => {
        const fb = feedback.get(r.ref) ?? { positive: 0, negative: 0 };
        return {
            ref: keyOf(r),
            currentRetrievalCount: args.retrievalCounts.get(r.ref) ?? 0,
            lastRetrievedAt: args.lastUseMsByRef.get(r.ref) ?? 0,
            acceptedChangeCount: accepted,
            negativeFeedbackCount: fb.negative,
            valence: computeValenceScore(fb).valence,
            utilityScore: args.utilityMap.get(r.ref),
            now,
        };
    };
    const out = new Map();
    if (!persist && !eventsCtx?.db) {
        const projected = new Map(mergedRefs.map((r) => [r.ref, projectAssetOutcome(undefined, inputsFor(r, 0)).outcomeScore]));
        const max = Math.min(OUTCOME_SCORE_MAX, Math.max(0, ...projected.values()));
        for (const [ref, score] of projected)
            out.set(ref, outcomeScoreToSalience(score, max));
        return out;
    }
    withRunState(eventsCtx, persist, (db) => {
        const accepted = new Map();
        try {
            const rows = listStateProposals(db, {
                status: "accepted",
                ...(args.primaryStashDir ? { stashDir: args.primaryStashDir } : {}),
            });
            for (const p of rows)
                accepted.set(p.ref, (accepted.get(p.ref) ?? 0) + 1);
        }
        catch {
            // Accepted counts stay 0.
        }
        const raw = new Map();
        const byKey = new Map();
        for (const r of mergedRefs) {
            try {
                const inputs = inputsFor(r, accepted.get(r.ref) ?? 0);
                const result = persist
                    ? updateAssetOutcome(db, inputs)
                    : projectAssetOutcome(getAssetOutcome(db, inputs.ref), inputs);
                raw.set(r.ref, result.outcomeScore);
                byKey.set(inputs.ref, result.outcomeScore);
            }
            catch {
                // This ref keeps its stored score.
            }
        }
        // Normalize stash-wide (every row, this run's overlaid), within the writer's bound.
        let max = 0;
        try {
            const scores = new Map(getAllAssetOutcomes(db).map((row) => [row.asset_ref, row.outcome_score]));
            for (const [key, score] of byKey)
                scores.set(key, score);
            for (const score of scores.values())
                if (score > max)
                    max = score;
            max = Math.min(max, OUTCOME_SCORE_MAX);
        }
        catch {
            max = 0;
        }
        for (const [ref, score] of raw)
            out.set(ref, outcomeScoreToSalience(score, max));
        const missing = mergedRefs.filter((r) => !raw.has(r.ref));
        if (missing.length > 0) {
            const refByKey = new Map(missing.map((r) => [keyOf(r), r.ref]));
            for (const [key, score] of getOutcomeScoresByRef(db, [...refByKey.keys()])) {
                out.set(refByKey.get(key) ?? key, outcomeScoreToSalience(score, max));
            }
        }
    });
    return out;
}
/** Drop candidates whose file vanished since planning, with one aggregate event. */
async function dropRefsMissingOnDisk(sorted, options, eventsCtx, persist) {
    const actionableRefs = [];
    const missing = [];
    for (const candidate of sorted) {
        const filePath = candidate.filePath && fs.existsSync(candidate.filePath)
            ? candidate.filePath
            : await findAssetFilePath(candidate.ref, options.stashDir);
        if (filePath && fs.existsSync(filePath))
            actionableRefs.push(candidate);
        else
            missing.push(candidate.ref);
    }
    if (persist && missing.length > 0) {
        recordImproveSkip(eventsCtx, undefined, {
            reason: "asset_missing_on_disk",
            count: missing.length,
            refs: missing.slice(0, 50),
        });
    }
    return { actionableRefs, missing };
}