@phren/cli/dist/shared/fragment-graph.js

Knowledge layer for AI agents - CLI, MCP server, and data layer

import { decodeStringRow } from "./index.js";
import * as fs from "fs";
import * as path from "path";
import { runtimeFile } from "../shared.js";
import { logger } from "../logger.js";
import { UNIVERSAL_TECH_TERMS_RE } from "../phren-core.js";
import { errorMessage } from "../utils.js";
// NOTE on naming: "entity" was renamed to "fragment" everywhere user-facing
// (CHANGELOG 0.0.5) — tool names, titles, and descriptions in tools/graph.ts
// all say "fragment" now. The underlying SQLite tables (`entities`,
// `entity_links`, `global_entities`, created in shared/index.ts) and the
// on-disk `.runtime/manual-links.json` format (`{entity, entityType, ...}`)
// were never renamed to match. Both are effectively rebuilt/local-runtime
// state rather than synced source-of-truth files, so a rename would not need
// a store migration in the git-history sense, but it does require touching
// shared/index.ts (which creates the tables and reads manual-links.json) —
// out of scope for a change confined to this file.
/** @internal Exported for tests. */
export function escapeRegex(s) { return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); }
/** Escape SQL LIKE wildcard characters so user input is treated literally.
 * @internal Exported for tests. */
export function escapeLike(s) { return s.replace(/[%_\\]/g, '\\$&'); }
/**
 * Log fragment resolution misses to .runtime/fragment-misses.jsonl.
 *
 * Judgment criteria — what's worth capturing vs noise:
 * - Worth capturing: repeated lookups for the same fragment name (indicates a gap
 *   in the fragment graph that the user keeps hitting), fragment names that look like
 *   real library/tool names (not random query fragments).
 * - Noise: single one-off lookups for short generic terms, lookups that fail
 *   because the query was malformed. We filter these by requiring name.length > 2.
 *
 * Gated by PHREN_DEBUG (or PHREN_DEBUG for compat) to avoid disk writes for
 * regular users. The miss log is append-only JSONL so downstream tooling can
 * detect repeated patterns (e.g. "fragment X was looked up 5 times but never
 * found" -> suggest adding it).
 */
export function logFragmentMiss(phrenPath, name, context, project) {
    if (!process.env.PHREN_DEBUG)
        return;
    if (!name || name.length <= 2)
        return;
    try {
        const entry = JSON.stringify({
            fragment: name,
            context,
            ts: Date.now(),
            project: project ?? null,
        });
        const missFile = runtimeFile(phrenPath, "fragment-misses.jsonl");
        fs.appendFileSync(missFile, entry + "\n");
    }
    catch {
        // Best-effort logging; don't let miss tracking break the caller.
    }
}
// Use the shared universal starter set. Framework/tool specifics are learned
// dynamically per project via extractDynamicFragments() in content-dedup.ts.
const PROSE_FRAGMENT_PATTERN = UNIVERSAL_TECH_TERMS_RE;
const FRAGMENT_PATTERNS = [
    // import/require patterns: import X from 'pkg' or require('pkg')
    /(?:import\s+.*?\s+from\s+['"])(@?[\w\-/]+)(?:['"])/g,
    /(?:require\s*\(\s*['"])(@?[\w\-/]+)(?:['"]\s*\))/g,
    // @scope/package patterns in text
    /@[\w-]+\/[\w-]+/g,
    // Known library/tool names mentioned in prose (case-insensitive word boundaries)
    PROSE_FRAGMENT_PATTERN,
    // Backtick-quoted identifiers: `word` or `word-with-dashes`
    /`([\w][\w\-\.\/]{1,48}[\w])`/g,
    // Double-quoted short identifiers (tool/package names, not full sentences)
    /"([\w][\w\-]{1,30}[\w])"/g,
];
function isAllowedFragmentName(name) {
    const trimmed = name.trim();
    if (!trimmed || trimmed.length <= 1 || trimmed.length >= 100)
        return false;
    // Skip version strings (e.g. "1.2.3", "v2.0.0-beta")
    if (/^v?\d+\.\d+/.test(trimmed))
        return false;
    // Skip file paths and file-like names (e.g. "src/utils.ts", "./config.json")
    if (/^\.?\//.test(trimmed) ||
        /\.(ts|js|json|md|yaml|yml|py|go|rs|java|tsx|jsx|css|html|txt|sh|toml|cfg|ini|env|lock)$/i.test(trimmed)) {
        return false;
    }
    if (!/\s/.test(trimmed)) {
        const normalized = trimmed.replace(/^[@#]/, "").toLowerCase();
        if (COMMON_SINGLE_WORD_FRAGMENTS.has(normalized))
            return false;
    }
    return true;
}
/**
 * Lightweight synchronous fragment extraction from text — regex only, no DB writes.
 * Used by add_finding to surface detected fragments in the MCP response without
 * requiring a DB reference in the write path. Full DB linking happens on the next
 * index rebuild, which is triggered automatically after every add_finding call via
 * updateFileInIndex -> extractAndLinkFragments.
 */
export function extractFragmentNames(content) {
    const found = new Set();
    for (const pattern of FRAGMENT_PATTERNS) {
        const regex = new RegExp(pattern.source, pattern.flags);
        let match;
        while ((match = regex.exec(content)) !== null) {
            const name = match[1] || match[0];
            if (!isAllowedFragmentName(name))
                continue;
            found.add(name.toLowerCase());
        }
    }
    // Extract explicit fragment annotations: <!-- fragment: Foo,Bar --> (also supports legacy <!-- entity: ... -->)
    const annotationRe = /<!--\s*(?:fragment|entity):\s*([^-]+)-->/gi;
    let m;
    while ((m = annotationRe.exec(content)) !== null) {
        for (const part of m[1].split(",")) {
            const name = part.trim();
            if (isAllowedFragmentName(name)) {
                found.add(name.toLowerCase());
            }
        }
    }
    return [...found];
}
function getOrCreateFragment(db, name, type) {
    try {
        db.run("INSERT OR IGNORE INTO entities (name, type, first_seen_at) VALUES (?, ?, ?)", [name, type, new Date().toISOString().slice(0, 10)]);
    }
    catch (err) {
        logger.debug("fragmentInsert", errorMessage(err));
    }
    const result = db.exec("SELECT id FROM entities WHERE name = ? AND type = ?", [name, type]);
    if (result?.length && result[0]?.values?.length) {
        return Number(result[0].values[0][0]);
    }
    return -1;
}
/**
 * Ensure the global_entities cross-project index table exists.
 * Called during buildIndex to enable cross-project fragment queries.
 */
export function ensureGlobalEntitiesTable(db) {
    try {
        db.run(`CREATE TABLE IF NOT EXISTS global_entities (
        entity TEXT NOT NULL,
        project TEXT NOT NULL,
        doc_key TEXT NOT NULL,
        PRIMARY KEY (entity, project, doc_key)
      )`);
    }
    catch (err) {
        logger.debug("ensureGlobalEntitiesTable", errorMessage(err));
    }
}
/**
 * Parse user-defined fragment names from AGENTS.md frontmatter.
 * Looks for: <!-- phren:fragments: Redis,MyService,InternalAPI -->
 * Also supports legacy: <!-- phren:entities: ... -->
 *
 * Results are cached per project+mtime to avoid repeated sync readFileSync calls
 * during a single index build that processes many docs for the same project.
 */
const _userFragmentCache = new Map();
const _buildUserFragmentCache = new Map();
let _activeBuildCacheKeyPrefix = null;
function readUserDefinedFragmentsFromDisk(claudeMdPath) {
    if (!fs.existsSync(claudeMdPath))
        return null;
    const stat = fs.statSync(claudeMdPath);
    const content = fs.readFileSync(claudeMdPath, "utf-8");
    // Support both new phren:fragments and legacy phren:entities annotations
    const match = content.match(/<!--\s*(?:phren:fragments|phren:entities):\s*(.+?)\s*-->/);
    const fragments = match
        ? match[1].split(",").map(s => s.trim()).filter(s => s.length > 0)
        : [];
    return { mtime: stat.mtimeMs, fragments };
}
/**
 * Prime AGENTS.md fragments per project for a single build pass.
 * During an active build, extractAndLinkFragments resolves user fragments from this
 * in-memory map and avoids per-file sync stat/read calls.
 */
export function beginUserFragmentBuildCache(phrenPath, projects) {
    _activeBuildCacheKeyPrefix = `${phrenPath}/`;
    for (const project of projects) {
        // The cache key stays a slash-joined string — it is only ever compared
        // against other keys built the same way. The file path is a real path, so
        // it goes through path.join: concatenating with "/" reads fine on Windows
        // but produces a mixed-separator string that no other code path matches.
        const cacheKey = `${phrenPath}/${project}`;
        const claudeMdPath = path.join(phrenPath, project, "AGENTS.md");
        try {
            const loaded = readUserDefinedFragmentsFromDisk(claudeMdPath);
            if (!loaded) {
                _buildUserFragmentCache.set(cacheKey, []);
                continue;
            }
            _userFragmentCache.set(cacheKey, loaded);
            _buildUserFragmentCache.set(cacheKey, loaded.fragments);
        }
        catch (err) {
            logger.debug("beginUserFragmentBuildCache", errorMessage(err));
            _buildUserFragmentCache.set(cacheKey, []);
        }
    }
}
/** End a build-scoped cache created by beginUserFragmentBuildCache(). */
export function endUserFragmentBuildCache(phrenPath) {
    const prefix = `${phrenPath}/`;
    for (const key of [..._buildUserFragmentCache.keys()]) {
        if (key.startsWith(prefix))
            _buildUserFragmentCache.delete(key);
    }
    if (_activeBuildCacheKeyPrefix === prefix)
        _activeBuildCacheKeyPrefix = null;
}
function parseUserDefinedFragments(phrenPath, project) {
    const claudeMdPath = path.join(phrenPath, project, "AGENTS.md");
    const cacheKey = `${phrenPath}/${project}`;
    try {
        // Active build path: no sync I/O in per-file extraction.
        if (_activeBuildCacheKeyPrefix === `${phrenPath}/`) {
            if (_buildUserFragmentCache.has(cacheKey))
                return _buildUserFragmentCache.get(cacheKey) ?? [];
            _buildUserFragmentCache.set(cacheKey, []);
            return [];
        }
        const cached = _userFragmentCache.get(cacheKey);
        if (cached) {
            try {
                if (fs.existsSync(claudeMdPath) && fs.statSync(claudeMdPath).mtimeMs === cached.mtime) {
                    return cached.fragments;
                }
            }
            catch (err) {
                logger.debug("parseUserDefinedFragments statCheck", errorMessage(err));
            }
        }
        const loaded = readUserDefinedFragmentsFromDisk(claudeMdPath);
        if (!loaded)
            return [];
        _userFragmentCache.set(cacheKey, loaded);
        return loaded.fragments;
    }
    catch (err) {
        logger.debug("parseUserDefinedFragments", errorMessage(err));
        return [];
    }
}
/** Clear the user fragment cache (call between index builds). */
function _clearUserFragmentCache() {
    _userFragmentCache.clear();
    _buildUserFragmentCache.clear();
    _activeBuildCacheKeyPrefix = null;
}
// Words that commonly start sentences or appear in titles — not fragment names
const SENTENCE_START_WORDS = new Set([
    "the", "this", "that", "these", "those", "when", "where", "which", "while",
    "what", "with", "will", "would", "should", "could", "have", "has", "had",
    "been", "being", "before", "after", "about", "above", "below", "between",
    "only", "also", "even", "just", "like", "make", "made", "many", "more",
    "most", "much", "must", "need", "never", "note", "once", "other", "over",
    "same", "some", "such", "sure", "take", "than", "them", "then", "they",
    "each", "every", "both", "either", "neither", "here", "there", "first",
    "second", "third", "next", "last", "new", "old", "good", "bad", "best",
    "however", "therefore", "because", "although", "since", "unless", "until",
    "instead", "rather", "already", "always", "never", "sometimes", "often",
]);
const COMMON_SINGLE_WORD_FRAGMENTS = new Set([
    ...SENTENCE_START_WORDS,
    "agent", "analysis", "app", "approach", "artifact", "branch", "build", "cache",
    "change", "changes", "check", "cli", "code", "command", "config", "context",
    "data", "debug", "detail", "doc", "docs", "document", "entity", "error",
    "example", "extract", "feature", "file", "files", "fix", "flow", "hook", "idea",
    "index", "info", "issue", "item", "key", "log", "memory", "message", "model",
    "note", "output", "path", "pattern", "policy", "process", "profile", "project",
    "query", "repo", "result", "rule", "search", "session", "setting", "state",
    "step", "summary", "system", "task", "tasks", "test", "tool", "tools", "type",
    "update", "user", "value", "version", "workflow", "write",
]);
// Patterns that look like version strings, file paths, or dates — not fragments
const FALSE_POSITIVE_PATTERNS = [
    /^v?\d+\.\d+/, // version strings: 1.2.3, v2.0
    /^[A-Z]:\\/, // Windows paths: C:\
    /^\//, // Unix paths: /usr/bin
    /^\d{4}-\d{2}/, // ISO dates: 2026-03
    /^[A-Z]{2,6}$/, // All-caps abbreviations shorter than 7 chars (OK, API, etc.)
    /^(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\b/i, // Month names
];
/**
 * Extract capitalized noun phrases (2+ words starting with uppercase) as candidate fragments.
 * e.g. "Auth Service", "Data Pipeline", "Internal API"
 *
 * Filters out common false positives: sentence-start capitalization, version strings,
 * file paths, and single-word abbreviations.
 */
function extractCapitalizedPhrases(content) {
    const found = new Set();
    const pattern = /\b([A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+)+)\b/g;
    let match;
    while ((match = pattern.exec(content)) !== null) {
        const phrase = match[1];
        if (phrase.length < 4 || phrase.length >= 60)
            continue;
        // Check if first word is a common sentence-start word
        const firstWord = phrase.split(/\s+/)[0].toLowerCase();
        if (SENTENCE_START_WORDS.has(firstWord))
            continue;
        // Check if it looks like a false positive pattern
        if (FALSE_POSITIVE_PATTERNS.some(p => p.test(phrase)))
            continue;
        found.add(phrase.toLowerCase());
    }
    return [...found];
}
export function extractAndLinkFragments(db, content, sourceDoc, phrenPath) {
    const fragmentNames = extractFragmentNames(content);
    // Extract capitalized noun phrases as candidate fragments
    const capitalizedPhrases = extractCapitalizedPhrases(content);
    for (const phrase of capitalizedPhrases) {
        fragmentNames.push(phrase);
    }
    // Add user-defined fragments from AGENTS.md frontmatter
    if (phrenPath) {
        const projectMatch = sourceDoc.match(/^([^/]+)\//);
        if (projectMatch) {
            const project = projectMatch[1];
            const userFragments = parseUserDefinedFragments(phrenPath, project);
            for (const uf of userFragments) {
                const lower = uf.toLowerCase();
                // Check if user-defined fragment appears in content (use escaped regex for safe matching)
                const safePattern = new RegExp(`\\b${escapeRegex(lower)}\\b`, "i");
                if (safePattern.test(content)) {
                    fragmentNames.push(lower);
                }
            }
        }
    }
    // Deduplicate
    const uniqueNames = [...new Set(fragmentNames)];
    if (uniqueNames.length === 0)
        return;
    const docFragmentId = getOrCreateFragment(db, sourceDoc, "document");
    if (docFragmentId === -1)
        return;
    // Ensure global_entities table exists
    ensureGlobalEntitiesTable(db);
    // Extract project from sourceDoc for global_entities
    const projectMatch = sourceDoc.match(/^([^/]+)\//);
    const project = projectMatch ? projectMatch[1] : null;
    for (const name of uniqueNames) {
        const fragmentType = name.includes(" ") ? "concept" : "library";
        const fragmentId = getOrCreateFragment(db, name, fragmentType);
        if (fragmentId === -1)
            continue;
        try {
            db.run("INSERT OR IGNORE INTO entity_links (source_id, target_id, rel_type, source_doc) VALUES (?, ?, ?, ?)", [docFragmentId, fragmentId, "mentions", sourceDoc]);
        }
        catch (err) {
            logger.debug("fragmentLinksInsert", errorMessage(err));
        }
        // Write to global_entities for cross-project queries
        if (project) {
            try {
                db.run("INSERT OR IGNORE INTO global_entities (entity, project, doc_key) VALUES (?, ?, ?)", [name, project, sourceDoc]);
            }
            catch (err) {
                logger.debug("globalFragmentsInsert", errorMessage(err));
            }
        }
    }
}
/**
 * Query related fragments for a given name.
 */
export function queryFragmentLinks(db, name) {
    const related = [];
    try {
        // Find the fragment
        const fragmentResult = db.exec("SELECT id FROM entities WHERE name = ?", [name.toLowerCase()]);
        if (!fragmentResult?.length || !fragmentResult[0]?.values?.length)
            return { related };
        const fragmentId = Number(fragmentResult[0].values[0][0]);
        // Find related fragments through links (both directions)
        const links = db.exec(`SELECT DISTINCT e.name FROM entity_links el JOIN entities e ON (el.target_id = e.id OR el.source_id = e.id)
       WHERE (el.source_id = ? OR el.target_id = ?) AND e.id != ?`, [fragmentId, fragmentId, fragmentId]);
        if (links?.length && links[0]?.values?.length) {
            for (const row of links[0].values) {
                related.push(decodeStringRow(row, 1, "queryFragmentLinks")[0]);
            }
        }
    }
    catch (err) {
        logger.debug("queryFragmentLinks", errorMessage(err));
    }
    return { related };
}
/**
 * Query cross-project fragment relationships.
 * Returns projects and docs that share fragments with the given query.
 */
export function queryCrossProjectFragments(db, fragmentName, excludeProject) {
    const results = [];
    try {
        ensureGlobalEntitiesTable(db);
        const pattern = `%${escapeLike(fragmentName.toLowerCase())}%`;
        let sql = "SELECT entity, project, doc_key FROM global_entities WHERE entity LIKE ? ESCAPE '\\'";
        const params = [pattern];
        if (excludeProject) {
            sql += " AND project != ?";
            params.push(excludeProject);
        }
        sql += " ORDER BY entity LIMIT 50";
        const rows = db.exec(sql, params);
        if (rows?.length && rows[0]?.values?.length) {
            for (const row of rows[0].values) {
                const [fragment, project, docKey] = decodeStringRow(row, 3, "queryCrossProjectFragments");
                results.push({
                    fragment,
                    project,
                    docKey,
                });
            }
        }
    }
    catch (err) {
        logger.debug("queryCrossProjectFragments", errorMessage(err));
    }
    return results;
}
export function getFragmentBoostDocs(db, query) {
    const normalizedQuery = query.toLowerCase();
    try {
        const rows = db.exec(`SELECT DISTINCT el.source_doc
       FROM entity_links el
       JOIN entities e ON el.target_id = e.id
       WHERE length(e.name) > 2
         AND ? LIKE '%' || lower(e.name) || '%'`, [normalizedQuery])[0]?.values ?? [];
        const boostDocs = new Set();
        for (const [doc] of rows) {
            if (typeof doc === "string")
                boostDocs.add(doc);
        }
        return boostDocs;
    }
    catch (err) {
        logger.debug("getFragmentBoostDocs", errorMessage(err));
        return new Set();
    }
}