UNPKG

@phren/cli

Version:

Knowledge layer for AI agents - CLI, MCP server, and data layer

515 lines (514 loc) • 28 kB
import * as fs from "fs"; import * as path from "path"; import * as crypto from "crypto"; import { debugLog, appendAuditLog, phrenOk, phrenErr, PhrenError } from "../shared.js"; import { normalizeMemoryScope } from "../shared.js"; import { withFileLock, recordLookupEvents } from "../shared/governance.js"; import { findingNodeIdForLine } from "../finding-graph-id.js"; import { isValidProjectName } from "../utils.js"; import { storeAwareProjectPath } from "../store-routing.js"; import { buildCitationComment, buildScopeComment, buildSourceComment, getHeadCommit, getRepoRoot, inferCitationLocation, } from "./citation.js"; import { isDuplicateFinding, scanForSecrets, normalizeObservationTags, resolveCoref } from "./dedup.js"; import { validateFindingsFormat, validateFinding } from "./validate.js"; import { countActiveFindings, autoArchiveToReference } from "./archive.js"; import { resolveAutoFindingTaskItem, resolveFindingTaskReference, } from "../finding/context.js"; import { buildLifecycleComments, extractFindingType, parseFindingLifecycle, stripLifecycleComments, } from "../finding/lifecycle.js"; import { METADATA_REGEX, } from "./metadata.js"; import { FINDINGS_FILENAME } from "../data/access.js"; /** Default cap for active findings before auto-archiving is triggered. */ const DEFAULT_FINDINGS_CAP = 20; const LIFECYCLE_ANNOTATION_RE = METADATA_REGEX.lifecycleAnnotation; /** Default finding-id source: 8 lowercase hex chars from a CSPRNG. Overridable * via `AddFindingOptions.idSource` for deterministic fixture generation * (the phren apps' generate-fixtures.mjs) — production callers never set it, * so this is the only thing that ever runs outside tests. */ function defaultFindingIdSource() { return crypto.randomBytes(4).toString("hex"); } function buildFindingCitation(citationInput, nowIso, inferredRepo, headCommit) { const citation = { created_at: nowIso ?? new Date().toISOString(), repo: citationInput?.repo || inferredRepo, file: citationInput?.file, line: citationInput?.line, commit: citationInput?.commit || (citationInput?.repo || inferredRepo ? headCommit ?? getHeadCommit(citationInput?.repo || inferredRepo || "") : undefined), symbol: citationInput?.symbol, symbol_unresolved: citationInput?.symbol_unresolved ? true : undefined, supersedes: citationInput?.supersedes, task_item: citationInput?.task_item, }; if (citation.repo && citation.commit && (!citation.file || !citation.line)) { const inferred = inferCitationLocation(citation.repo, citation.commit); citation.file = citation.file || inferred.file; citation.line = citation.line || inferred.line; } return citation; } function resolveInferredCitationRepo(citationInput) { if (citationInput?.repo) return citationInput.repo; if (citationInput?.file) { const fileDir = path.dirname(citationInput.file); return getRepoRoot(fileDir); } return undefined; } function buildFindingScope(scope) { return normalizeMemoryScope(scope) || undefined; } function resolveFindingCitationInput(phrenPath, project, citationInput) { const resolved = citationInput ? { ...citationInput } : {}; if (citationInput?.task_item) { const taskResolution = resolveFindingTaskReference(phrenPath, project, citationInput.task_item); if (taskResolution.error) { return phrenErr(taskResolution.error, PhrenError.VALIDATION_ERROR); } if (taskResolution.stableId) { resolved.task_item = taskResolution.stableId; } } else { const taskItem = resolveAutoFindingTaskItem(phrenPath, project); if (taskItem) { resolved.task_item = taskItem; } } return phrenOk(Object.keys(resolved).length > 0 ? resolved : undefined); } /** * Heuristically infer a finding's type tag from its own wording when the * caller didn't supply one. Return type is FindingTag (not FindingType) * because "workaround" and "context" are valid outputs here even though * they aren't part of the smaller offered/pickable FINDING_TYPES set — * both still have a FINDING_TYPE_DECAY row and are searchable via * search_knowledge's `tag` filter (FINDING_TAGS), so this function agrees * with both. */ export function autoDetectFindingType(text) { const lower = text.toLowerCase(); if (/\b(we decided|decision:|chose .+ over|went with)\b/.test(lower)) return 'decision'; if (/\b(bug:|bug in|found a bug|broken|crashes|fails when)\b/.test(lower)) return 'bug'; if (/\b(workaround:|work around|temporary fix|hack:)\b/.test(lower)) return 'workaround'; if (/\b(pattern:|always .+ before|never .+ without|best practice)\b/.test(lower)) return 'pattern'; if (/\b(pitfall:|gotcha:|watch out|careful with|trap:)\b/.test(lower)) return 'pitfall'; if (/\b(currently|as of|right now|at the moment|observation:)\b/.test(lower)) return 'context'; return null; } function prepareFinding(opts) { const { finding: learning, project, fullHistory, extraAnnotations, citationInput, scope, provenance, nowIso, inferredRepo, headCommit, idSource } = opts; const secretType = scanForSecrets(learning); if (secretType) { return { status: "rejected", reason: `Contains ${secretType}` }; } const today = (nowIso ?? new Date().toISOString()).slice(0, 10); const { text: tagNormalized, warning: tagWarning } = normalizeObservationTags(learning); let normalizedLearning = resolveCoref(tagNormalized, { project, file: citationInput?.file, }); const existingType = extractFindingType('- ' + normalizedLearning); if (!existingType) { const detected = autoDetectFindingType(normalizedLearning); if (detected) { normalizedLearning = `[${detected}] ${normalizedLearning}`; } } const fid = (idSource ?? defaultFindingIdSource)(); const fidComment = `<!-- fid:${fid} -->`; const createdComment = `<!-- created: ${today} -->`; const scopeComment = buildScopeComment(scope); let lifecycle = { status: "active", status_updated: today }; let bullet = `${normalizedLearning.startsWith("- ") ? normalizedLearning : `- ${normalizedLearning}`} ${fidComment} ${createdComment}`; if (scopeComment) bullet += ` ${scopeComment}`; const sourceComment = provenance ? buildSourceComment(provenance) : ""; if (sourceComment) bullet += ` ${sourceComment}`; if (isDuplicateFinding(fullHistory, bullet)) { return { status: "duplicate" }; } // NOTE: heuristic contradiction detection is intentionally NOT done here. The lexical // heuristic (shared entity + opposing polarity) produces false positives, so it must // never silently mutate the stored finding's status. Instead the MCP add_finding tool // surfaces heuristic conflict *candidates* (via findConflictCandidates) in its response // so the calling agent — an LLM already in the loop with full context — can judge them, // exactly as it already does for potential duplicates. A hard "contradicted" status is // reserved for the LLM-confirmed path (checkSemanticConflicts) and explicit user // resolution via resolve_contradiction. if (extraAnnotations && extraAnnotations.length > 0) { const lifecycleFromExtra = parseFindingLifecycle(`- lifecycle ${extraAnnotations.join(" ")}`); if (lifecycleFromExtra.status !== "active" || lifecycleFromExtra.status_reason || lifecycleFromExtra.status_ref || lifecycleFromExtra.status_updated) { lifecycle = { ...lifecycle, ...lifecycleFromExtra, status_updated: lifecycleFromExtra.status_updated ?? lifecycle.status_updated ?? today, }; } const existing = new Set([...bullet.matchAll(METADATA_REGEX.conflictsWithAll)].map((m) => m[0])); for (const annotation of extraAnnotations) { if (!annotation.startsWith("<!--")) continue; if (LIFECYCLE_ANNOTATION_RE.test(annotation)) continue; if (existing.has(annotation)) continue; bullet += ` ${annotation}`; existing.add(annotation); } } bullet += ` ${buildLifecycleComments(lifecycle, today)}`; const citation = buildFindingCitation(citationInput, nowIso, inferredRepo, headCommit); return { status: "added", finding: { original: learning, normalized: normalizedLearning, bullet, citationComment: ` ${buildCitationComment(citation)}`, tagWarning, }, }; } function insertFindingIntoContent(content, today, bullet, citationComment) { const todayHeader = `## ${today}`; // Use positional insertion (not String.replace) to avoid: (1) special $& replacement patterns // if bullet contains $ chars, and (2) inserting inside an archived <details> block when a // duplicate date header exists from a prior consolidation run. // Search for todayHeader only after the last </details> close tag so we never // insert into an archived block whose date happens to match today. const lastDetailsClose = content.lastIndexOf("</details>"); const searchFrom = lastDetailsClose >= 0 ? lastDetailsClose : 0; const idx = content.indexOf(todayHeader, searchFrom); if (idx !== -1) { const insertAt = idx + todayHeader.length; return content.slice(0, insertAt) + `\n\n${bullet}\n${citationComment}` + content.slice(insertAt); } const firstHeadingMatch = content.match(/^## \d{4}-\d{2}-\d{2}/m); if (firstHeadingMatch?.index != null) { return (content.slice(0, firstHeadingMatch.index) + `${todayHeader}\n\n${bullet}\n${citationComment}\n\n` + content.slice(firstHeadingMatch.index)); } return content.trimEnd() + `\n\n## ${today}\n\n${bullet}\n${citationComment}\n`; } export function upsertCanonical(phrenPath, project, memory) { if (!isValidProjectName(project)) return phrenErr(`Invalid project name: "${project}".`, PhrenError.INVALID_PROJECT_NAME); const resolvedDir = storeAwareProjectPath(phrenPath, project); if (!resolvedDir || !fs.existsSync(resolvedDir)) return phrenErr(`Project "${project}" not found in phren.`, PhrenError.PROJECT_NOT_FOUND); const canonicalPath = path.join(resolvedDir, "truths.md"); const today = new Date().toISOString().slice(0, 10); const bullet = memory.startsWith("- ") ? memory : `- ${memory}`; withFileLock(canonicalPath, () => { if (!fs.existsSync(canonicalPath)) { fs.writeFileSync(canonicalPath, `# ${project} Truths\n\n## Truths\n\n${bullet} _(added ${today})_\n`); } else { const existing = fs.readFileSync(canonicalPath, "utf8"); const line = `${bullet} _(added ${today})_`; if (!existing.includes(bullet)) { const updated = existing.includes("## Truths") ? existing.replace("## Truths", `## Truths\n\n${line}`) : `${existing.trimEnd()}\n\n## Truths\n\n${line}\n`; const content = updated.endsWith("\n") ? updated : updated + "\n"; const tmpPath = canonicalPath + `.tmp-${crypto.randomUUID()}`; fs.writeFileSync(tmpPath, content); fs.renameSync(tmpPath, canonicalPath); } } }); appendAuditLog(phrenPath, "pin_memory", `project=${project} memory=${JSON.stringify(memory)}`); return phrenOk(`Truth saved in ${project}.`); } export function addFindingToFile(phrenPath, project, learning, citationInput, opts) { const findingError = validateFinding(learning); if (findingError) return phrenErr(findingError, PhrenError.EMPTY_INPUT); if (!isValidProjectName(project)) return phrenErr(`Invalid project name: "${project}".`, PhrenError.INVALID_PROJECT_NAME); const resolvedDir = storeAwareProjectPath(phrenPath, project); if (!resolvedDir) return phrenErr(`Invalid project name: "${project}".`, PhrenError.INVALID_PROJECT_NAME); const learningsPath = path.join(resolvedDir, FINDINGS_FILENAME); // Secret/PII scan — reject before anything else (before existence check, before lock) const nowIso = (opts?.now ?? new Date()).toISOString(); const today = nowIso.slice(0, 10); const resolvedCitationInputResult = resolveFindingCitationInput(phrenPath, project, citationInput); if (!resolvedCitationInputResult.ok) return resolvedCitationInputResult; const resolvedCitationInput = resolvedCitationInputResult.data; const scope = buildFindingScope(opts?.scope); const inferredRepo = resolveInferredCitationRepo(resolvedCitationInput); const headCommit = inferredRepo ? getHeadCommit(inferredRepo) : undefined; const supersedesText = resolvedCitationInput?.supersedes; const normalizedForSupersedes = supersedesText ? resolveCoref(normalizeObservationTags(learning).text, { project, file: resolvedCitationInput?.file, }) : undefined; // Reject secrets before anything else — even if project doesn't exist yet const earlySecretType = scanForSecrets(learning); if (earlySecretType) { return phrenErr(`Rejected: finding appears to contain a secret (${earlySecretType}). Strip credentials before saving.`, PhrenError.VALIDATION_ERROR); } // Check project dir existence before withFileLock (which would create the dir via mkdirSync) if (!fs.existsSync(resolvedDir)) return phrenErr(`Project "${project}" does not exist.`, PhrenError.INVALID_PROJECT_NAME); const result = withFileLock(learningsPath, () => { const preparedForNewFile = prepareFinding({ finding: learning, project, fullHistory: "", extraAnnotations: opts?.extraAnnotations, citationInput: resolvedCitationInput, scope, provenance: opts?.provenance, nowIso, inferredRepo, headCommit, phrenPath, idSource: opts?.idSource, }); if (!fs.existsSync(learningsPath)) { if (preparedForNewFile.status === "rejected") { return phrenErr(`Rejected: finding appears to contain a secret (${preparedForNewFile.reason.replace(/^Contains /, "")}). Strip credentials before saving.`, PhrenError.VALIDATION_ERROR); } if (preparedForNewFile.status === "duplicate") { return phrenOk(`Skipped duplicate finding for "${project}": already exists with similar wording.`); } const newContent = `# ${project} Findings\n\n## ${today}\n\n${preparedForNewFile.finding.bullet}\n${preparedForNewFile.finding.citationComment}\n`; const tmpPath = learningsPath + `.tmp-${crypto.randomUUID()}`; fs.writeFileSync(tmpPath, newContent); fs.renameSync(tmpPath, learningsPath); return phrenOk({ content: newContent, citation: buildFindingCitation(resolvedCitationInput, nowIso, inferredRepo, headCommit), tagWarning: preparedForNewFile.finding.tagWarning, created: true, bullet: preparedForNewFile.finding.bullet, }); } const content = fs.readFileSync(learningsPath, "utf8"); // When superseding, strip the old finding from history so dedup doesn't block the intentionally similar replacement. // Skip the strip if new finding is identical to the superseded one (self-supersession should still be blocked by dedup). const isSelfSupersession = supersedesText && learning.trim().toLowerCase().slice(0, 60) === supersedesText.trim().toLowerCase().slice(0, 60); const historyForDedup = (supersedesText && !isSelfSupersession) ? content.split("\n") .filter(line => !line.startsWith("- ") || !line.toLowerCase().includes(supersedesText.slice(0, 40).toLowerCase())) .join("\n") : content; const prepared = prepareFinding({ finding: learning, project, fullHistory: historyForDedup, extraAnnotations: opts?.extraAnnotations, citationInput: resolvedCitationInput, scope, provenance: opts?.provenance, nowIso, inferredRepo, headCommit, phrenPath, idSource: opts?.idSource, }); if (prepared.status === "rejected") { return phrenErr(`Rejected: finding appears to contain a secret (${prepared.reason.replace(/^Contains /, "")}). Strip credentials before saving.`, PhrenError.VALIDATION_ERROR); } if (prepared.status === "duplicate") { debugLog(`add_finding: skipped duplicate for "${project}": ${learning.slice(0, 80)}`); return phrenOk(`Skipped duplicate finding for "${project}": already exists with similar wording.`); } const issues = validateFindingsFormat(content); if (issues.length > 0) { debugLog(`FINDINGS.md format warnings for "${project}": ${issues.join("; ")}`); } let updated = insertFindingIntoContent(content, today, prepared.finding.bullet, prepared.finding.citationComment); if (supersedesText && normalizedForSupersedes) { const lines = updated.split("\n"); const needle = supersedesText.slice(0, 60).toLowerCase().replace(/\s+/g, " ").trim(); for (let i = 0; i < lines.length; i++) { if (!lines[i].startsWith("- ")) continue; const lineText = lines[i].replace(/<!--.*?-->/g, "").replace(/^-\s+/, "").replace(/^\[[^\]]+\]\s+/, "").slice(0, 60).toLowerCase().replace(/\s+/g, " ").trim(); if (lineText === needle) { // Remove any legacy and normalized lifecycle supersession metadata before re-appending. lines[i] = lines[i].replace(METADATA_REGEX.stripSupersededByLegacy, ""); lines[i] = lines[i].replace(METADATA_REGEX.stripSupersededBy, ""); lines[i] = stripLifecycleComments(lines[i]); const newFirst60 = normalizedForSupersedes.replace(/^-\s+/, "").slice(0, 60); lines[i] = `${lines[i]} <!-- phren:superseded_by "${newFirst60}" ${today} --> ` + `${buildLifecycleComments({ status: "superseded", status_updated: today, status_reason: "superseded_by", status_ref: newFirst60 }, today)}`; updated = lines.join("\n"); break; } } // Also annotate the new finding bullet with phren:supersedes const newLines = updated.split("\n"); for (let i = 0; i < newLines.length; i++) { if (!newLines[i].startsWith("- ")) continue; if (newLines[i].includes(prepared.finding.bullet.slice(0, 40))) { if (!newLines[i].includes("phren:supersedes")) { const supersedesFirst60 = supersedesText.slice(0, 60); newLines[i] = `${newLines[i]} <!-- phren:supersedes "${supersedesFirst60}" -->`; } updated = newLines.join("\n"); break; } } } const tmpPath = learningsPath + `.tmp-${crypto.randomUUID()}`; fs.writeFileSync(tmpPath, updated); fs.renameSync(tmpPath, learningsPath); return phrenOk({ content: updated, citation: buildFindingCitation(resolvedCitationInput, nowIso, inferredRepo, headCommit), tagWarning: prepared.finding.tagWarning, created: false, bullet: prepared.finding.bullet, }); }); if (!result.ok) return result; if (typeof result.data === "string") return phrenOk({ message: result.data, status: "skipped" }); appendAuditLog(phrenPath, "add_finding", `project=${project}${result.data.created ? " created=true" : ""} citation_commit=${result.data.citation.commit ?? "none"} citation_file=${result.data.citation.file ?? "none"}`); // Live activity: a write lights up the same graph node a lookup would, so a // watching shell shows findings being saved as well as read. Best-effort. try { const bullet = result.data.bullet; if (bullet) { // Derive the node id the same way buildGraph does, so a watching graph // lights the node that was just written rather than a phantom id. const nodeId = findingNodeIdForLine(project, bullet.startsWith("-") ? bullet : `- ${bullet}`); recordLookupEvents(phrenPath, [{ query: "", project, filename: FINDINGS_FILENAME, type: "findings", snippet: bullet.replace(/^-\s+/, "").replace(/\s*<!--.*?-->/g, "").trim(), source: "write", ...(nodeId ? { nodeId } : {}), }]); } } catch (err) { debugLog(`addFindingToFile lookup-event: ${String(err)}`); } const cap = Number.parseInt((process.env.PHREN_FINDINGS_CAP) || "", 10) || DEFAULT_FINDINGS_CAP; const activeCount = countActiveFindings(result.data.content); if (activeCount > cap) { const archiveResult = autoArchiveToReference(phrenPath, project, cap); if (archiveResult.ok && archiveResult.data > 0) { debugLog(`Size cap: archived ${archiveResult.data} oldest entries for "${project}" (cap=${cap})`); } } if (result.data.created) { const createdMsg = `Created FINDINGS.md for "${project}" and added insight.`; const message = result.data.tagWarning ? `${createdMsg} Warning: ${result.data.tagWarning}` : createdMsg; return phrenOk({ message, status: "created" }); } const addedMsg = `Added finding to ${project}: ${result.data.bullet} (with citation metadata)`; const message = result.data.tagWarning ? `${addedMsg} Warning: ${result.data.tagWarning}` : addedMsg; return phrenOk({ message, status: "added" }); } export function addFindingsToFile(phrenPath, project, learnings, opts) { if (!isValidProjectName(project)) return phrenErr(`Invalid project name: "${project}".`, PhrenError.INVALID_PROJECT_NAME); const resolvedDir = storeAwareProjectPath(phrenPath, project); if (!resolvedDir) return phrenErr(`Invalid project name: "${project}".`, PhrenError.INVALID_PROJECT_NAME); const learningsPath = path.join(resolvedDir, FINDINGS_FILENAME); const nowIso = new Date().toISOString(); const today = nowIso.slice(0, 10); const resolvedCitationInputResult = resolveFindingCitationInput(phrenPath, project); if (!resolvedCitationInputResult.ok) return resolvedCitationInputResult; const resolvedCitationInput = resolvedCitationInputResult.data; const scope = buildFindingScope(opts?.scope); const inferredRepo = resolveInferredCitationRepo(resolvedCitationInput); const headCommit = inferredRepo ? getHeadCommit(inferredRepo) : undefined; const added = []; const skipped = []; const rejected = []; // Check project dir existence before withFileLock (which would create the dir via mkdirSync) if (!fs.existsSync(resolvedDir)) return phrenErr(`Project "${project}" not found in phren.`, PhrenError.PROJECT_NOT_FOUND); const contentResult = withFileLock(learningsPath, () => { if (!fs.existsSync(learningsPath)) { let content = `# ${project} Findings\n\n## ${today}\n`; for (const [index, learning] of learnings.entries()) { const extraAnnotations = opts?.extraAnnotationsByFinding?.[index]; const lengthError = validateFinding(learning); if (lengthError) { rejected.push({ text: learning, reason: lengthError }); continue; } const prepared = prepareFinding({ finding: learning, project, fullHistory: content, extraAnnotations, citationInput: resolvedCitationInput, scope, provenance: opts?.provenance, nowIso, inferredRepo, headCommit, phrenPath, }); if (prepared.status === "rejected") { rejected.push({ text: learning, reason: prepared.reason }); continue; } if (prepared.status === "duplicate") { skipped.push(learning); continue; } content = insertFindingIntoContent(content, today, prepared.finding.bullet, prepared.finding.citationComment); if (prepared.finding.tagWarning) debugLog(`add_findings: ${prepared.finding.tagWarning}`); added.push(learning); } if (added.length > 0) { const tmpPath = learningsPath + `.tmp-${crypto.randomUUID()}`; fs.writeFileSync(tmpPath, content.endsWith("\n") ? content : `${content}\n`); fs.renameSync(tmpPath, learningsPath); } return phrenOk({ content, wrote: added.length > 0 }); } let content = fs.readFileSync(learningsPath, "utf8"); const issues = validateFindingsFormat(content); if (issues.length > 0) debugLog(`FINDINGS.md format warnings for "${project}": ${issues.join("; ")}`); for (const [index, learning] of learnings.entries()) { const extraAnnotations = opts?.extraAnnotationsByFinding?.[index]; const lengthError = validateFinding(learning); if (lengthError) { rejected.push({ text: learning, reason: lengthError }); continue; } const prepared = prepareFinding({ finding: learning, project, fullHistory: content, extraAnnotations, citationInput: resolvedCitationInput, scope, nowIso, inferredRepo, headCommit, phrenPath, }); if (prepared.status === "rejected") { rejected.push({ text: learning, reason: prepared.reason }); continue; } if (prepared.status === "duplicate") { skipped.push(learning); continue; } content = insertFindingIntoContent(content, today, prepared.finding.bullet, prepared.finding.citationComment); if (prepared.finding.tagWarning) debugLog(`add_findings: ${prepared.finding.tagWarning}`); added.push(learning); } if (added.length > 0) { const tmpPath = learningsPath + `.tmp-${crypto.randomUUID()}`; fs.writeFileSync(tmpPath, content); fs.renameSync(tmpPath, learningsPath); } return phrenOk({ content, wrote: added.length > 0 }); }); if (!contentResult.ok) return contentResult; if (contentResult.data.wrote) { appendAuditLog(phrenPath, "add_finding", `project=${project} count=${added.length} batch=true`); const cap = Number.parseInt((process.env.PHREN_FINDINGS_CAP) || "", 10) || DEFAULT_FINDINGS_CAP; if (countActiveFindings(contentResult.data.content) > cap) { const archiveResult = autoArchiveToReference(phrenPath, project, cap); if (archiveResult.ok && archiveResult.data > 0) { debugLog(`Size cap: archived ${archiveResult.data} oldest entries for "${project}" (cap=${cap})`); } } } return phrenOk({ added, skipped, rejected }); }