@phren/cli/dist/content/archive.js

Knowledge layer for AI agents - CLI, MCP server, and data layer

import * as fs from "fs";
import * as path from "path";
import * as crypto from "crypto";
import { debugLog, runtimeFile, phrenOk, phrenErr, PhrenError, appendAuditLog, tryUnlink } from "../shared.js";
import { isValidProjectName, errorMessage } from "../utils.js";
import { storeAwareProjectPath } from "../store-routing.js";
import { withFileLock } from "../governance/locks.js";
import { walkDirectory } from "../shared/data-utils.js";
import { appendArchivedEntriesToTopicDoc, classifyTopicForText, readProjectTopics, topicReferencePath } from "../project-topics.js";
import { isCitationLine, isArchiveStart, isArchiveEnd, stripComments } from "./metadata.js";
import { logger } from "../logger.js";
import { FINDINGS_FILENAME } from "../filenames.js";
/**
 * Count active (non-archived) finding entries in FINDINGS.md content.
 * Entries inside archive blocks are considered archived.
 * Supports structured archive markers and HTML details blocks.
 */
export function countActiveFindings(content) {
    let inArchive = false;
    let count = 0;
    for (const line of content.split("\n")) {
        if (isArchiveStart(line)) {
            inArchive = true;
            continue;
        }
        if (isArchiveEnd(line)) {
            inArchive = false;
            continue;
        }
        if (!inArchive && line.startsWith("- "))
            count++;
    }
    return count;
}
/**
 * Parse active (non-archived) entries from FINDINGS.md, oldest first.
 */
function parseActiveEntries(content) {
    const lines = content.split("\n");
    const entries = [];
    let currentDate = "";
    let inArchive = false;
    for (let i = 0; i < lines.length; i++) {
        const line = lines[i];
        if (isArchiveStart(line)) {
            inArchive = true;
            continue;
        }
        if (isArchiveEnd(line)) {
            inArchive = false;
            continue;
        }
        if (inArchive)
            continue;
        const heading = line.match(/^## (\d{4}-\d{2}-\d{2})$/);
        if (heading) {
            currentDate = heading[1];
            continue;
        }
        if (line.startsWith("- ") && currentDate) {
            const next = lines[i + 1] || "";
            const hasCitation = isCitationLine(next);
            entries.push({
                date: currentDate,
                bullet: line,
                citation: hasCitation ? next : undefined,
                lineIndex: i,
            });
            if (hasCitation)
                i++;
        }
    }
    // Sort oldest first: earliest date first; within the same date, higher
    // lineIndex = earlier in the file (newest findings are prepended at top).
    // Q25: use descending lineIndex within the same day so that when we slice
    // `toArchive = entries.slice(0, N)` we archive the truly oldest entries
    // (lowest in the file = largest lineIndex for that date).
    entries.sort((a, b) => a.date.localeCompare(b.date) || b.lineIndex - a.lineIndex);
    return entries;
}
/**
 * Check whether a bullet already exists in a reference file (already archived).
 */
/** Build a Set of normalized bullet strings from all .md files in referenceDir. */
function buildArchivedBulletSet(referenceDir) {
    const bulletSet = new Set();
    try {
        for (const filePath of walkDirectory(referenceDir)) {
            const content = fs.readFileSync(filePath, "utf8");
            for (const line of content.split("\n")) {
                if (!line.startsWith("- "))
                    continue;
                const normalizedLine = stripComments(line).replace(/^-\s+/, "").trim().toLowerCase();
                if (normalizedLine)
                    bulletSet.add(normalizedLine);
            }
        }
    }
    catch (err) {
        logger.debug("archive", `buildArchivedBulletSet: ${errorMessage(err)}`);
    }
    return bulletSet;
}
function isAlreadyArchived(archivedSet, bullet) {
    const normalizedBullet = stripComments(bullet).replace(/^-\s+/, "").trim().toLowerCase();
    if (!normalizedBullet)
        return false;
    return archivedSet.has(normalizedBullet);
}
/**
 * Archive the oldest entries from FINDINGS.md into reference/{topic}.md files.
 * Keeps `keepCount` most recent entries, archives the rest grouped by topic.
 * Returns the number of entries archived.
 */
export function autoArchiveToReference(phrenPath, project, keepCount) {
    if (!isValidProjectName(project))
        return phrenErr(`Invalid project name: "${project}".`, PhrenError.INVALID_PROJECT_NAME);
    const resolvedDir = storeAwareProjectPath(phrenPath, project);
    if (!resolvedDir || !fs.existsSync(resolvedDir))
        return phrenErr(`Project "${project}" not found in phren.`, PhrenError.PROJECT_NOT_FOUND);
    const learningsPath = path.join(resolvedDir, FINDINGS_FILENAME);
    if (!fs.existsSync(learningsPath))
        return phrenOk(0);
    // Consolidation lock to prevent concurrent runs for the same project (atomic create via wx flag).
    // Use a project-specific lock so consolidating multiple projects in parallel is allowed.
    const STALE_LOCK_MS = 600_000; // 10 min
    const lockFile = runtimeFile(phrenPath, `consolidation-${project}.lock`);
    try {
        fs.writeFileSync(lockFile, String(Date.now()), { flag: "wx" });
    }
    catch (e) {
        if (e.code === "EEXIST") {
            try {
                const stat = fs.statSync(lockFile);
                if (Date.now() - stat.mtimeMs < STALE_LOCK_MS) {
                    return phrenErr("Consolidation already running", PhrenError.LOCK_TIMEOUT);
                }
                // Stale lock: delete then re-create atomically with wx.
                tryUnlink(lockFile);
                // Re-attempt atomic create. If EEXIST, another thread won the race.
                try {
                    fs.writeFileSync(lockFile, String(Date.now()), { flag: "wx" });
                }
                catch (wxErr) {
                    if (wxErr.code === "EEXIST")
                        return phrenErr("Consolidation already running", PhrenError.LOCK_TIMEOUT);
                    throw wxErr;
                }
            }
            catch (innerErr) {
                if (innerErr.code === "EEXIST" || innerErr.code === "ENOENT") {
                    return phrenErr("Consolidation already running", PhrenError.LOCK_TIMEOUT);
                }
                throw innerErr;
            }
        }
        else {
            throw e;
        }
    }
    // Q11: Hold the per-file lock on FINDINGS.md for the entire read-modify-write
    // cycle so finding writers and the archive pass see a consistent file.
    try {
        return withFileLock(learningsPath, () => {
            const content = fs.readFileSync(learningsPath, "utf8");
            const entries = parseActiveEntries(content);
            if (entries.length <= keepCount)
                return phrenOk(0);
            const toArchive = entries.slice(0, entries.length - keepCount);
            // Guard: skip entries already present in reference tier (prevent double-archive)
            const referenceDir = path.join(resolvedDir, "reference");
            const { topics } = readProjectTopics(phrenPath, project);
            const today = new Date().toISOString().slice(0, 10);
            const archivedSet = buildArchivedBulletSet(referenceDir);
            const actuallyArchived = [];
            for (const entry of toArchive) {
                if (isAlreadyArchived(archivedSet, entry.bullet)) {
                    debugLog(`auto_archive: skipping already-archived entry: "${entry.bullet.slice(0, 60)}"`);
                    continue;
                }
                actuallyArchived.push(entry);
            }
            // Group archived entries by topic
            const byTopic = new Map();
            for (const entry of actuallyArchived) {
                const topic = classifyTopicForText(entry.bullet, topics);
                const bucket = byTopic.get(topic.slug) ?? [];
                bucket.push({ date: today, bullet: entry.bullet, citation: entry.citation });
                byTopic.set(topic.slug, bucket);
            }
            // Write to reference/topics/{topic}.md (atomic rename per file)
            fs.mkdirSync(referenceDir, { recursive: true });
            const successfulTopics = new Set();
            for (const [topicSlug, topicEntries] of byTopic) {
                const filePath = topicReferencePath(phrenPath, project, topicSlug);
                const topic = topics.find((item) => item.slug === topicSlug) ?? topics.find((item) => item.slug === "general");
                if (!filePath || !topic)
                    continue;
                try {
                    appendArchivedEntriesToTopicDoc(filePath, project, topic, topicEntries);
                    successfulTopics.add(topicSlug);
                }
                catch (err) {
                    debugLog(`auto_archive: failed to write reference file for topic "${topicSlug}": ${errorMessage(err)}`);
                }
            }
            // Only remove entries whose topics were successfully written to reference files,
            // plus entries already present in reference (safe to remove since they're already archived).
            const alreadyArchivedEntries = toArchive.filter(entry => !actuallyArchived.includes(entry));
            const successfullyArchived = actuallyArchived.filter(entry => successfulTopics.has(classifyTopicForText(entry.bullet, topics).slug));
            const safeToRemove = [...successfullyArchived, ...alreadyArchivedEntries];
            // Remove archived entries from FINDINGS.md
            const lines = content.split("\n");
            const archiveLineSet = new Set();
            for (const entry of safeToRemove) {
                archiveLineSet.add(entry.lineIndex);
                if (entry.citation)
                    archiveLineSet.add(entry.lineIndex + 1);
            }
            const filtered = lines.filter((_, i) => !archiveLineSet.has(i));
            // Clean up empty date sections
            const cleaned = [];
            for (let i = 0; i < filtered.length; i++) {
                const line = filtered[i];
                const isDateHeading = /^## \d{4}-\d{2}-\d{2}$/.test(line);
                if (isDateHeading) {
                    // Check if next non-empty lines have any bullets
                    let hasBullets = false;
                    for (let j = i + 1; j < filtered.length; j++) {
                        const next = filtered[j].trim();
                        if (!next)
                            continue;
                        if (next.startsWith("## ") || next.startsWith("# "))
                            break;
                        if (next.startsWith("- ")) {
                            hasBullets = true;
                            break;
                        }
                        break;
                    }
                    if (!hasBullets)
                        continue;
                }
                cleaned.push(line);
            }
            // Write consolidation marker
            const marker = `<!-- consolidated: ${today} -->`;
            const markerIdx = cleaned.findIndex(l => l.includes("consolidated:"));
            if (markerIdx >= 0) {
                cleaned[markerIdx] = marker;
            }
            else {
                // Insert after title
                const titleIdx = cleaned.findIndex(l => l.startsWith("# "));
                if (titleIdx >= 0) {
                    cleaned.splice(titleIdx + 1, 0, "", marker);
                }
            }
            const tmpPath = learningsPath + `.tmp-${crypto.randomUUID()}`;
            fs.writeFileSync(tmpPath, cleaned.join("\n"));
            fs.renameSync(tmpPath, learningsPath);
            const skippedCount = alreadyArchivedEntries.length;
            const failedTopics = [...byTopic.keys()].filter(t => !successfulTopics.has(t));
            appendAuditLog(phrenPath, "auto_archive_reference", `project=${project} archived=${successfullyArchived.length} skipped_duplicates=${skippedCount}${failedTopics.length ? ` failed_topics=${failedTopics.join(",")}` : ""} topics=${[...successfulTopics].join(",")}`);
            return phrenOk(safeToRemove.length);
        });
    }
    finally {
        try {
            fs.unlinkSync(lockFile);
        }
        catch (err) {
            logger.debug("archive", `autoArchiveToReference unlockFile: ${errorMessage(err)}`);
        }
    }
}