@phren/cli/dist/content/dedup.js

Knowledge layer for AI agents - CLI, MCP server, and data layer

import * as fs from "fs";
import * as path from "path";
import * as crypto from "crypto";
import { debugLog, runtimeFile, KNOWN_OBSERVATION_TAGS } from "../shared.js";
import { isFeatureEnabled, errorMessage } from "../utils.js";
import { storeAwareProjectPath } from "../store-routing.js";
import { UNIVERSAL_TECH_TERMS_RE, EXTRA_FRAGMENT_PATTERNS } from "../phren-core.js";
import { isInactiveFindingLine } from "../finding/lifecycle.js";
import { logger } from "../logger.js";
import { FINDINGS_FILENAME } from "../data/access.js";
// ── LLM provider abstraction ────────────────────────────────────────────────
const MAX_CACHE_ENTRIES = 500;
function loadCache(cachePath) {
    if (!fs.existsSync(cachePath))
        return {};
    const raw = JSON.parse(fs.readFileSync(cachePath, "utf8"));
    const now = Date.now();
    const normalized = {};
    for (const [key, entry] of Object.entries(raw)) {
        const ts = typeof entry.ts === "number" ? entry.ts : now;
        normalized[key] = { result: entry.result, ts };
    }
    return normalized;
}
function trimCache(cache) {
    const entries = Object.entries(cache);
    if (entries.length <= MAX_CACHE_ENTRIES)
        return;
    entries
        .sort(([, a], [, b]) => a.ts - b.ts)
        .slice(0, entries.length - MAX_CACHE_ENTRIES)
        .forEach(([key]) => {
        delete cache[key];
    });
}
function persistCache(cachePath, cache) {
    trimCache(cache);
    fs.writeFileSync(cachePath, JSON.stringify(cache));
}
/**
 * Generic cache-through helper: load cache → check TTL → touch timestamp → persist → return.
 * If the key is cached and within TTL, returns the cached result.
 * Otherwise, calls `compute()` to produce a fresh result, caches it, and returns it.
 */
async function withCache(cachePath, key, ttlMs, compute) {
    // Check cache
    try {
        const cache = loadCache(cachePath);
        if (cache[key] && Date.now() - cache[key].ts < ttlMs) {
            cache[key].ts = Date.now();
            persistCache(cachePath, cache);
            return cache[key].result;
        }
    }
    catch (err) {
        logger.debug("dedup", `withCache load (${path.basename(cachePath)}): ${errorMessage(err)}`);
    }
    const result = await compute();
    // Persist result
    try {
        const cache = loadCache(cachePath);
        cache[key] = { result, ts: Date.now() };
        persistCache(cachePath, cache);
    }
    catch (err) {
        logger.debug("dedup", `withCache persist (${path.basename(cachePath)}): ${errorMessage(err)}`);
    }
    return result;
}
function isAbortError(error) {
    return error instanceof Error && error.name === "AbortError";
}
const LLM_TIMEOUT_MS = 10_000;
/**
 * Per-call budget. Ten seconds suits a YES/NO dedup check against a hosted
 * model; a paragraph from a local 8B model on a laptop CPU takes longer, so
 * callers with real output to wait for pass their own, and PHREN_LLM_TIMEOUT_MS
 * raises the floor for everything.
 */
function llmTimeoutMs(requested) {
    const fromEnv = Number.parseInt(process.env.PHREN_LLM_TIMEOUT_MS ?? "", 10);
    const floor = Number.isFinite(fromEnv) && fromEnv > 0 ? fromEnv : LLM_TIMEOUT_MS;
    return Math.max(floor, requested ?? 0);
}
function parseOpenAiResponse(data) {
    const d = data;
    return d.choices?.[0]?.message?.content?.trim() ?? "";
}
/** POST to an LLM endpoint with a combined per-call timeout + parent abort relay. */
async function fetchLlm(url, init, signal, parseResponse, timeoutMs = LLM_TIMEOUT_MS) {
    const controller = new AbortController();
    const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
    if (signal)
        signal.addEventListener("abort", () => controller.abort(), { once: true });
    let response;
    try {
        response = await fetch(url, { ...init, signal: controller.signal });
    }
    finally {
        clearTimeout(timeoutId);
    }
    if (!response.ok)
        throw new Error(`LLM API error: ${response.status}`);
    return parseResponse(await response.json());
}
// Default maxTokens is 10 — callers that only need YES/NO or CONFLICT/OK responses
// need just 3-5 tokens. Callers expecting longer output pass an explicit override (e.g. 60).
export async function callLlm(prompt, signal, maxTokens = 10, timeoutMs) {
    const budget = llmTimeoutMs(timeoutMs);
    // Check abort before starting any work to avoid unnecessary API calls
    if (signal?.aborted)
        throw new DOMException("Aborted", "AbortError");
    const endpoint = (process.env.PHREN_LLM_ENDPOINT);
    const customKey = (process.env.PHREN_LLM_KEY);
    const anthropicKey = process.env.ANTHROPIC_API_KEY;
    const openaiKey = process.env.OPENAI_API_KEY;
    const model = (process.env.PHREN_LLM_MODEL);
    if (endpoint) {
        // Custom endpoint: use PHREN_LLM_KEY, fall back to any available key
        const key = customKey || openaiKey || anthropicKey || "";
        return fetchLlm(`${endpoint.replace(/\/$/, "")}/chat/completions`, {
            method: "POST",
            headers: { "Content-Type": "application/json", ...(key ? { Authorization: `Bearer ${key}` } : {}) },
            body: JSON.stringify({ model: model || "gpt-4o-mini", messages: [{ role: "user", content: prompt }], max_tokens: maxTokens, temperature: 0 }),
        }, signal, parseOpenAiResponse, budget);
    }
    else if (anthropicKey) {
        // Anthropic REST API fallback (no SDK required)
        return fetchLlm("https://api.anthropic.com/v1/messages", {
            method: "POST",
            headers: { "content-type": "application/json", "x-api-key": anthropicKey, "anthropic-version": "2023-06-01" },
            body: JSON.stringify({ model: model || "claude-haiku-4-5-20251001", max_tokens: maxTokens, messages: [{ role: "user", content: prompt }] }),
        }, signal, (data) => {
            const d = data;
            const block = d.content?.[0];
            return (block?.type === "text" ? block.text ?? "" : "").trim();
        }, budget);
    }
    else if (openaiKey) {
        // OpenAI REST API fallback
        return fetchLlm("https://api.openai.com/v1/chat/completions", {
            method: "POST",
            headers: { "Content-Type": "application/json", Authorization: `Bearer ${openaiKey}` },
            body: JSON.stringify({ model: model || "gpt-4o-mini", messages: [{ role: "user", content: prompt }], max_tokens: maxTokens, temperature: 0 }),
        }, signal, parseOpenAiResponse, budget);
    }
    else {
        // No LLM configured — return empty to signal "not duplicate" / "no conflict"
        return "";
    }
}
// ── Cache TTL constants ───────────────────────────────────────────────────────
const DEDUP_CACHE_TTL_MS = 86_400_000; // 1 day
const CONFLICT_CACHE_TTL_MS = 7 * 86_400_000; // 7 days
// ── Metadata stripping helpers ────────────────────────────────────────────────
/**
 * Strip HTML comments only (timestamp metadata, citations).
 * Use this when you only need to remove <!-- ... --> markers.
 */
function stripHtmlComments(s) {
    return s.replace(/<!--.*?-->/gs, "");
}
/**
 * Strip all common finding metadata:
 * - HTML comments: <!-- ... -->
 * - "migrated from" annotations: (migrated from ...)
 * - Leading bullet dash: "- " at the start of the string
 */
export function stripMetadata(s) {
    return s
        .replace(/<!--.*?-->/gs, "")
        .replace(/\(migrated from [^)]+\)/gi, "")
        .replace(/^-\s+/, "");
}
// Stop words for lightweight semantic overlap checks
const DEDUP_STOP_WORDS = new Set([
    "the", "a", "an", "is", "are", "was", "were", "in", "on", "at", "to", "for",
    "of", "and", "or", "but", "not", "with", "from", "by", "as", "it", "its",
    "this", "that", "be", "has", "have", "had", "will", "would", "can", "could", "should",
]);
export function jaccardTokenize(text) {
    return new Set(text.toLowerCase()
        .split(/[\s\W]+/)
        .filter(w => w.length > 0 && !DEDUP_STOP_WORDS.has(w)));
}
export function jaccardSimilarity(a, b) {
    if (a.size === 0 && b.size === 0)
        return 1;
    let intersection = 0;
    for (const w of a) {
        if (b.has(w))
            intersection++;
    }
    const union = a.size + b.size - intersection;
    return union === 0 ? 0 : intersection / union;
}
// ── Contradiction detection ───────────────────────────────────────────────────
// Use the shared universal starter set. Framework/tool specifics are learned
// dynamically per project via extractDynamicEntities().
const PROSE_ENTITY_RE = UNIVERSAL_TECH_TERMS_RE;
const POSITIVE_RE = /\b(always|prefer|should|must|works|recommend|enable)\b/i;
const NEGATIVE_RE = /\b(never|avoid|don't|do not|shouldn't|must not|broken|deprecated|disable)\b/i;
// ── Dynamic fragment extraction ────────────────────────────────────────────────
const ENTITY_CACHE_TTL_MS = 60 * 60 * 1000; // 1 hour
// Patterns that suggest a token is a proper noun / tool name:
//   - CamelCase word (at least one interior uppercase): PhotonMappingEngine, GameKit
//   - All-caps acronym of 2–8 letters: AWS, GPU, API
//   - Known suffix patterns: *.js, *Engine, *API, *SDK, *DB, *UI, *ML
const DYNAMIC_ENTITY_RE = /\b(?:[A-Z][a-z]+(?:[A-Z][a-z]*)+|[A-Z]{2,8}|[A-Z][a-z]+(?:Engine|API|SDK|DB|UI|ML|IO|OS|JS|TS|CLI|MCP|GL|VR|AR|AI|NN|GAN))\b/g;
/**
 * Scan existing findings for proper nouns / tool names that appear in 2+ bullets.
 * Results are cached in .runtime/project-entities-{project}.json (1h TTL or
 * invalidated when FINDINGS.md changes).
 */
export function extractDynamicEntities(phrenPath, project) {
    try {
        const findingsPath = path.join(phrenPath, project, FINDINGS_FILENAME);
        if (!fs.existsSync(findingsPath))
            return new Set();
        const findingsStat = fs.statSync(findingsPath);
        const findingsMtime = findingsStat.mtimeMs;
        const cachePath = runtimeFile(phrenPath, `project-entities-${project}.json`);
        // Try reading existing cache
        if (fs.existsSync(cachePath)) {
            try {
                const cached = JSON.parse(fs.readFileSync(cachePath, "utf8"));
                const age = Date.now() - (cached.builtAt ?? 0);
                if (age < ENTITY_CACHE_TTL_MS && cached.findingsMtimeMs === findingsMtime) {
                    return new Set(cached.entities);
                }
            }
            catch {
                // fall through to rebuild
            }
        }
        // Rebuild: scan bullets for candidate tokens
        const content = fs.readFileSync(findingsPath, "utf8");
        const bullets = content.split("\n").filter(l => l.startsWith("- ") && !isInactiveFindingLine(l));
        // Count occurrences of each candidate across bullets
        const counts = new Map();
        for (const bullet of bullets) {
            const stripped = bullet.replace(/<!--.*?-->/g, "").replace(/^-\s+/, "");
            const seen = new Set();
            let m;
            const re = new RegExp(DYNAMIC_ENTITY_RE.source, DYNAMIC_ENTITY_RE.flags);
            while ((m = re.exec(stripped)) !== null) {
                const token = m[0];
                if (!seen.has(token)) {
                    seen.add(token);
                    counts.set(token, (counts.get(token) ?? 0) + 1);
                }
            }
        }
        // Keep tokens that appear in 2+ distinct bullets
        const entities = [...counts.entries()]
            .filter(([, n]) => n >= 2)
            .map(([token]) => token.toLowerCase());
        // Write cache
        const cacheEntry = { entities, builtAt: Date.now(), findingsMtimeMs: findingsMtime };
        fs.writeFileSync(cachePath, JSON.stringify(cacheEntry));
        return new Set(entities);
    }
    catch {
        return new Set();
    }
}
function extractProseEntities(text, dynamicEntities) {
    const found = new Set();
    const re = new RegExp(PROSE_ENTITY_RE.source, PROSE_ENTITY_RE.flags);
    let m;
    while ((m = re.exec(text)) !== null)
        found.add(m[0].toLowerCase());
    // Match additional fragment patterns (versions, env keys, file paths, error codes, dates)
    for (const { re: pattern } of EXTRA_FRAGMENT_PATTERNS) {
        const pRe = new RegExp(pattern.source, pattern.flags);
        let pm;
        while ((pm = pRe.exec(text)) !== null)
            found.add(pm[0].toLowerCase());
    }
    if (dynamicEntities) {
        // Also check whether any dynamic fragment appears (case-insensitive word match)
        for (const entity of dynamicEntities) {
            const escaped = entity.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
            if (new RegExp(`\\b${escaped}\\b`, "i").test(text)) {
                found.add(entity);
            }
        }
    }
    return [...found];
}
function learningPolarity(text) {
    const hasPos = POSITIVE_RE.test(text);
    const hasNeg = NEGATIVE_RE.test(text);
    if (hasPos && !hasNeg)
        return "positive";
    if (hasNeg && !hasPos)
        return "negative";
    return "neutral";
}
// Minimum topical overlap (Jaccard over substantive tokens, with observation tags
// and metadata stripped) required before two findings that share an entity and have
// opposing polarity are treated as a contradiction. The bare "one shared entity +
// opposite polarity" rule fires on unrelated findings that merely mention the same
// tool or word (e.g. a git-workflow decision and a security decision that both say
// "GitHub"); requiring genuine subject-matter overlap removes that false-positive
// class while keeping real same-topic contradictions.
const CONFLICT_MIN_TOPIC_OVERLAP = 0.12;
/** Strip observation tags like [decision] so a shared tag is not counted as shared topic. */
function stripObservationTags(text) {
    return text.replace(/\[[a-zA-Z_-]+\]/g, " ");
}
/** Jaccard overlap of the substantive tokens of two findings (tags + metadata removed). */
function topicalOverlap(a, b) {
    const ta = jaccardTokenize(stripObservationTags(stripMetadata(a)));
    const tb = jaccardTokenize(stripObservationTags(stripMetadata(b)));
    return jaccardSimilarity(ta, tb);
}
/** Returns existing learning lines that appear to conflict with newFinding. */
export function detectConflicts(newFinding, existingLines, dynamicEntities) {
    const newEntities = extractProseEntities(newFinding, dynamicEntities);
    if (newEntities.length === 0)
        return [];
    const newPol = learningPolarity(newFinding);
    if (newPol === "neutral")
        return [];
    const conflicts = [];
    for (const line of existingLines) {
        if (!line.startsWith("- "))
            continue;
        const lineEntities = extractProseEntities(line, dynamicEntities);
        const shared = lineEntities.filter((e) => newEntities.includes(e));
        if (shared.length === 0)
            continue;
        const linePol = learningPolarity(line);
        if (linePol === "neutral" || linePol === newPol)
            continue;
        // Require genuine topical overlap, not just one incidental shared entity, before
        // flagging. Unrelated findings that happen to share a tool name fall below this bar.
        if (topicalOverlap(newFinding, line) < CONFLICT_MIN_TOPIC_OVERLAP)
            continue;
        conflicts.push(line);
    }
    return conflicts;
}
export function isDuplicateFinding(existingContent, newLearning, threshold = 0.6) {
    const normalize = (text) => {
        return stripHtmlComments(text).trim()
            .toLowerCase()
            .replace(/[^a-z0-9\s]/g, " ")
            .split(/\s+/)
            .filter(w => w.length > 2 && !DEDUP_STOP_WORDS.has(w));
    };
    const newWords = normalize(newLearning);
    if (newWords.length === 0)
        return false;
    const newSet = new Set(newWords);
    const bullets = existingContent.split("\n").filter(l => l.startsWith("- "));
    for (const bullet of bullets) {
        if (isInactiveFindingLine(bullet))
            continue;
        const existingWords = normalize(bullet);
        if (existingWords.length === 0)
            continue;
        const existingSet = new Set(existingWords);
        // Fast path: exact word overlap check
        let overlap = 0;
        for (const w of newSet) {
            if (existingSet.has(w))
                overlap++;
        }
        const smaller = Math.min(newSet.size, existingSet.size);
        if (smaller > 0 && overlap / smaller > threshold) {
            debugLog(`duplicate-detection: skipping learning, ${Math.round((overlap / smaller) * 100)}% overlap with existing: "${bullet.slice(0, 80)}"`);
            return true;
        }
        // Second pass: Jaccard similarity (strip metadata before comparing)
        // Threshold lowered from 0.55 to 0.40 to catch agent paraphrases —
        // swarm agents often report the same insight with different wording,
        // and 0.55 let too many through.
        const newTokens = jaccardTokenize(stripMetadata(newLearning));
        const existingTokens = jaccardTokenize(stripMetadata(bullet));
        if (newTokens.size < 3 || existingTokens.size < 3)
            continue; // too few tokens for reliable Jaccard
        const jaccard = jaccardSimilarity(newTokens, existingTokens);
        if (jaccard > 0.40) {
            debugLog(`duplicate-detection: Jaccard ${Math.round(jaccard * 100)}% with existing: "${bullet.slice(0, 80)}"`);
            return true;
        }
    }
    return false;
}
// ── Typed observation tags ────────────────────────────────────────────────────
/**
 * Normalize known observation tags in learning text to lowercase.
 * Returns the normalized text and a warning if unknown bracket tags are found.
 */
export function normalizeObservationTags(text) {
    // Normalize known tags to lowercase
    let normalized = text.replace(/\[([a-zA-Z_-]+)\]/g, (_match, tag) => {
        const lower = tag.toLowerCase();
        if (KNOWN_OBSERVATION_TAGS.has(lower))
            return `[${lower}]`;
        return _match; // keep unknown tags as-is
    });
    // Detect unknown bracket tags for warning
    const unknownTags = [];
    const tagPattern = /\[([a-zA-Z_-]+)\]/g;
    let m;
    while ((m = tagPattern.exec(normalized)) !== null) {
        const lower = m[1].toLowerCase();
        if (!KNOWN_OBSERVATION_TAGS.has(lower)) {
            unknownTags.push(m[0]);
        }
    }
    const warning = unknownTags.length > 0
        ? `Unknown tag(s) ${unknownTags.join(", ")} — known tags: ${[...KNOWN_OBSERVATION_TAGS].map(t => `[${t}]`).join(", ")}`
        : undefined;
    return { text: normalized, warning };
}
// Credential detection lives in a dependency-free leaf module so hooks.ts can
// use it without dragging in this file's import graph. Re-exported here
// because content/learning.ts, data/notes.ts and shared/content.ts all import
// scanForSecrets from dedup.
export { looksLikePlaceholderSecret, scanForSecrets, redactSecretsForLog } from "./secrets.js";
/**
 * Resolve coreferences in learning text by replacing vague pronouns with concrete names.
 */
export function resolveCoref(text, context) {
    if (!context.project && !context.file)
        return text;
    let result = text;
    if (context.project) {
        // Sentence-starting "It ", "This ", "That " followed by a verb-like word
        result = result.replace(/^(It|This|That)\s+(?=[a-z])/i, (match) => `[${context.project}] ${match}`);
        // " the project" -> " {project}"
        result = result.replace(/\bthe project\b/gi, context.project);
    }
    if (context.file) {
        const basename = path.basename(context.file);
        result = result.replace(/\b(this file|the file)\b/gi, basename);
    }
    // If text has no concrete nouns AND has vague pronouns, prepend context
    if (context.project && /\b(it|this|that|they|them)\b/i.test(result)) {
        const hasConcreteNoun = /[A-Z][a-z]+[A-Z]|[a-z]+\.[a-z]+|@[a-z]|https?:\/\//.test(result);
        if (!hasConcreteNoun && result === text) {
            result = `[context: ${context.project}] ${result}`;
        }
    }
    return result;
}
/**
 * LLM-based semantic dedup check. Only called when PHREN_FEATURE_SEMANTIC_DEDUP=1.
 * Must be called before addFindingToFile() since that function is sync.
 * Returns true if the new learning is a semantic duplicate of any existing bullet.
 */
export async function checkSemanticDedup(phrenPath, project, newLearning, signal) {
    if (!isFeatureEnabled("PHREN_FEATURE_SEMANTIC_DEDUP", false))
        return false;
    const resolvedDir = storeAwareProjectPath(phrenPath, project);
    if (!resolvedDir)
        return false;
    const findingsPath = path.join(resolvedDir, FINDINGS_FILENAME);
    if (!fs.existsSync(findingsPath))
        return false;
    const existingContent = fs.readFileSync(findingsPath, "utf8");
    const bullets = existingContent.split("\n").filter((l) => l.startsWith("- ") && !isInactiveFindingLine(l));
    for (const bullet of bullets) {
        const a = stripMetadata(newLearning).trim();
        const b = stripMetadata(bullet).trim();
        const tokA = jaccardTokenize(a);
        const tokB = jaccardTokenize(b);
        if (tokA.size < 3 || tokB.size < 3)
            continue;
        const jaccard = jaccardSimilarity(tokA, tokB);
        if (jaccard >= 0.40)
            continue; // already caught by sync isDuplicateFinding
        if (jaccard >= 0.3) {
            const isDup = await semanticDedup(a, b, phrenPath, signal);
            if (isDup)
                return true;
        }
    }
    return false;
}
async function semanticDedup(a, b, phrenPath, signal) {
    const key = crypto.createHash("sha256").update(a + "|||" + b).digest("hex");
    const cachePath = runtimeFile(phrenPath, "dedup-cache.json");
    try {
        return await withCache(cachePath, key, DEDUP_CACHE_TTL_MS, async () => {
            const answer = await callLlm(`Are these two findings semantically equivalent? Reply YES or NO only.\nA: ${a}\nB: ${b}`, signal);
            return answer.trim().toUpperCase().startsWith("YES");
        });
    }
    catch (error) {
        if (isAbortError(error))
            return false;
        return false; // fallback: not a duplicate
    }
}
const CONFLICT_CHECK_TOTAL_TIMEOUT_MS = 30_000;
/**
 * LLM-based conflict check. Only called when PHREN_FEATURE_SEMANTIC_CONFLICT=1.
 * Call after detectConflicts() in addFindingToFile flow.
 * Returns conflict annotations to append to the bullet.
 * Also scans global findings and other projects for cross-project contradictions.
 * Has a 30-second total timeout; returns partial results if the deadline is hit.
 */
export async function checkSemanticConflicts(phrenPath, project, newFinding, signal) {
    if (!isFeatureEnabled("PHREN_FEATURE_SEMANTIC_CONFLICT", false))
        return { annotations: [], checked: false };
    const resolvedDir = storeAwareProjectPath(phrenPath, project);
    if (!resolvedDir)
        return { annotations: [], checked: false };
    const newEntities = extractProseEntities(newFinding);
    if (newEntities.length === 0)
        return { annotations: [], checked: true };
    // Collect bullet sources: { bullets, sourceProject } pairs
    const sources = [];
    // Current project
    const findingsPath = path.join(resolvedDir, FINDINGS_FILENAME);
    if (fs.existsSync(findingsPath)) {
        const content = fs.readFileSync(findingsPath, "utf8");
        sources.push({ bullets: content.split("\n").filter((l) => l.startsWith("- ")), sourceProject: null });
    }
    // Global project findings
    const globalFindingsPath = path.join(phrenPath, "global", FINDINGS_FILENAME);
    if (fs.existsSync(globalFindingsPath)) {
        const content = fs.readFileSync(globalFindingsPath, "utf8");
        const bullets = content.split("\n").filter((l) => l.startsWith("- "));
        if (bullets.length > 0)
            sources.push({ bullets, sourceProject: "global" });
    }
    // Scan other projects by FINDINGS.md recency so we still check the hottest projects first,
    // but do not truncate the search set and miss older contradictions.
    try {
        const entries = fs.readdirSync(phrenPath, { withFileTypes: true });
        const otherProjects = entries
            .filter((e) => e.isDirectory() && e.name !== project && e.name !== "global" && !e.name.startsWith("."))
            .map((e) => {
            const fp = path.join(phrenPath, e.name, FINDINGS_FILENAME);
            if (!fs.existsSync(fp))
                return null;
            try {
                return { name: e.name, mtime: fs.statSync(fp).mtimeMs, fp };
            }
            catch (err) {
                logger.debug("dedup", `crossProjectScan stat: ${errorMessage(err)}`);
                return null;
            }
        })
            .filter((x) => x !== null)
            .sort((a, b) => b.mtime - a.mtime);
        for (const proj of otherProjects) {
            const content = fs.readFileSync(proj.fp, "utf8");
            const bullets = content.split("\n").filter((l) => l.startsWith("- "));
            if (bullets.length > 0)
                sources.push({ bullets, sourceProject: proj.name });
        }
    }
    catch (err) {
        logger.debug("dedup", `crossProjectScan: ${errorMessage(err)}`);
    }
    const annotations = [];
    const deadline = Date.now() + CONFLICT_CHECK_TOTAL_TIMEOUT_MS;
    outer: for (const { bullets, sourceProject } of sources) {
        for (const line of bullets) {
            // Respect the aggregate deadline — return partial results rather than hanging
            if (Date.now() >= deadline) {
                debugLog("checkSemanticConflicts: total timeout reached, returning partial results");
                break outer;
            }
            const lineEntities = extractProseEntities(line);
            const shared = lineEntities.filter((e) => newEntities.includes(e));
            if (shared.length === 0)
                continue;
            const result = await llmConflictCheck(line, newFinding, shared[0], phrenPath, signal);
            if (result === "CONFLICT") {
                const snippet = stripMetadata(line).trim().slice(0, 80);
                const sourceLabel = sourceProject ? ` (from project: ${sourceProject})` : "";
                annotations.push(`<!-- conflicts_with: "${snippet}"${sourceLabel} -->`);
            }
        }
    }
    return { annotations, checked: true };
}
async function llmConflictCheck(existing, newFinding, entity, phrenPath, signal) {
    const key = crypto.createHash("sha256").update(existing + "|||" + newFinding).digest("hex");
    const cachePath = runtimeFile(phrenPath, "conflict-cache.json");
    try {
        return await withCache(cachePath, key, CONFLICT_CACHE_TTL_MS, async () => {
            const answer = await callLlm(`Finding A: ${existing}. Finding B: ${newFinding}. Do these contradict each other about how to use ${entity}? Reply CONFLICT or OK only.`, signal);
            return answer.trim().toUpperCase().startsWith("CONFLICT")
                ? "CONFLICT"
                : "OK";
        });
    }
    catch (error) {
        if (isAbortError(error))
            return "OK";
        return "OK";
    }
}