socraticode/dist/services/graph-inputs.js

SocratiCode — MCP server for local codebase indexing, semantic search, and code dependency graphs. All private, all local via Docker.

import { constants as fsConstants } from "node:fs";
import fs from "node:fs/promises";
import path from "node:path";
import { EXTENSION_LANGUAGE_MAP, hashContent, indexExtensionlessEnabled, MAX_GRAPH_FILE_BYTES, SOCRATICODE_VERSION, toForwardSlash, } from "../constants.js";
import { detectionHeadWindow, readFileHead } from "./extensionless.js";
import { createIgnoreFilter, shouldIgnore } from "./ignore.js";
// ── What the graph was built from ────────────────────────────────────────
/**
 * The shape version of a persisted {@link GraphInputRecord}.
 *
 * A record carrying any other version is treated exactly as a missing one: one
 * full rebuild, which writes the current shape back. Nothing migrates, nothing
 * is re-indexed, and no operator action is involved.
 */
export const GRAPH_INPUTS_VERSION = 1;
/** Sorts record keys into one deterministic order, wherever they are written. */
const byKey = ([a], [b]) => a < b ? -1 : a > b ? 1 : 0;
const sortedRecord = (entries) => Object.fromEntries([...entries].sort(byKey));
/**
 * Hash of the inputs the build reads from configuration rather than from the
 * tree: which extra extensions count as source, which extensions are mapped to
 * another language's grammar, which configured Python roots are searched and
 * in what order, whether dotfiles and `.gitignore` are honoured, and how large
 * a file the graph will still read.
 */
/**
 * Which parsers were available to the build, as one hash.
 *
 * Names only, never the error text of a failed load: the text carries paths
 * and messages that differ between hosts for reasons that have nothing to do
 * with what the build could parse, and would make the fingerprint unstable in
 * exactly the situation it exists to detect.
 */
export function graphCapabilitiesHash(input) {
    return hashContent(JSON.stringify({
        loadedGrammars: [...input.loadedGrammars].sort(),
        gdscript: input.gdscript,
        elixirTemplates: input.elixirTemplates,
    }));
}
export function graphSettingsHash(extraExtensions, pythonRoots = []) {
    return hashContent(JSON.stringify({
        extraExtensions: [...extraExtensions].sort(),
        // Preserve the pre-pythonRoots fingerprint when the feature is unused,
        // so an upgrade alone does not rebuild every existing project's graph.
        // Order is significant when roots are configured because they are
        // searched in declared order.
        ...(pythonRoots.length > 0 ? { pythonRoots: [...pythonRoots] } : {}),
        // Decides whether an extensionless file can become a node at all, so
        // toggling it moves the node set with no file on disk changing.
        indexExtensionless: indexExtensionlessEnabled(),
        extensionLanguageMap: [...EXTENSION_LANGUAGE_MAP.entries()].sort(byKey),
        includeDotFiles: (process.env.INCLUDE_DOT_FILES ?? "false").toLowerCase() === "true",
        respectGitignore: (process.env.RESPECT_GITIGNORE ?? "true").toLowerCase() !== "false",
        maxGraphFileBytes: MAX_GRAPH_FILE_BYTES,
    }));
}
const hashEntryNames = (names) => hashContent(names.sort().join("\n"));
const entryName = (entry) => entry.isDirectory() ? `${entry.name}/` : entry.name;
/**
 * Apply the ignore filter to one directory's entries, and hash what it saw.
 *
 * The single place the filter is applied to a listing: the discovery walk
 * iterates the `kept` entries this returns rather than filtering again, and
 * the rebuild check re-reads the directory through this same function. Neither
 * can drift from the other about which entries counted, because neither has
 * its own opinion.
 */
export function scanDirectory(ig, projectPath, dir, entries) {
    const kept = [];
    for (const entry of entries) {
        const relPath = toForwardSlash(path.relative(projectPath, path.join(dir, entry.name)));
        if (shouldIgnore(ig, entry.isDirectory() ? `${relPath}/` : relPath))
            continue;
        kept.push(entry);
    }
    return {
        kept,
        listing: {
            all: hashEntryNames(entries.map(entryName)),
            kept: hashEntryNames(kept.map(entryName)),
        },
    };
}
/**
 * Raised by {@link GraphInputRecorder.finish} when the build observed one file
 * two irreconcilable ways.
 *
 * The build reads some sources twice — resolver setup builds its namespace,
 * FQCN, module and `class_name` maps before the main loop parses the same
 * files — so a file edited in between leaves resolver output describing one
 * version while nodes, imports and symbols describe another. That graph is
 * internally mixed, and a record of either version would let the next update
 * call it current. Neither half is worth persisting, so the build is abandoned
 * and retried from a fresh scan instead.
 */
export class GraphInputsMovedDuringBuild extends Error {
    /** Project-relative path → what disagreed about it. */
    conflicts;
    constructor(conflicts) {
        const [first] = [...conflicts];
        super(`${conflicts.size} graph input(s) changed while the graph was being built` +
            (first ? ` — ${first[0]}: ${first[1]}` : ""));
        this.name = "GraphInputsMovedDuringBuild";
        this.conflicts = conflicts;
    }
}
export function createGraphInputRecorder(projectRoot) {
    const root = path.resolve(projectRoot);
    const files = new Map();
    const heads = new Map();
    const presence = new Map();
    const realpaths = new Map();
    const unreadable = new Set();
    const unreadableDirectories = new Set();
    const directories = new Map();
    /**
     * Head-window hash of every file read whole, so a head-read arriving after
     * one can be checked against the same bytes rather than guessed at. Never
     * persisted — the content hash it belongs to already covers it.
     */
    const windowOfRead = new Map();
    const conflicts = new Map();
    const rel = (absolutePath) => toForwardSlash(path.relative(root, path.resolve(absolutePath))) || ".";
    /** First conflict per path wins, for the same reason first observations do. */
    const conflict = (key, detail) => {
        if (!conflicts.has(key))
            conflicts.set(key, detail);
    };
    return {
        read(absolutePath, content) {
            const key = rel(absolutePath);
            const hash = hashContent(content);
            const alreadyRead = files.get(key);
            if (alreadyRead !== undefined) {
                // Identical repeat: the resolver pass and the build loop saw one file.
                if (alreadyRead !== hash)
                    conflict(key, "read twice, with different content each time");
                return;
            }
            const headSeen = heads.get(key);
            if (headSeen !== undefined) {
                // Discovery head-read it, and now it has been read whole. Valid only if
                // the window detection scored is still the same bytes. The full hash is
                // what gets kept: it subsumes the window — a change inside the window
                // changes the whole-file hash too — where keeping the head would leave
                // the record blind to everything past it.
                if (headSeen !== hashContent(detectionHeadWindow(content))) {
                    conflict(key, "its detection head no longer matches the file read after it");
                    return;
                }
                heads.delete(key);
            }
            if (unreadable.has(key)) {
                conflict(key, "could not be read, then read");
                return;
            }
            files.set(key, hash);
            windowOfRead.set(key, hashContent(detectionHeadWindow(content)));
            // A whole-file read supersedes a sighting that never opened it.
            presence.delete(key);
        },
        head(absolutePath, headText) {
            const key = rel(absolutePath);
            const hash = hashContent(headText);
            const windowRead = windowOfRead.get(key);
            if (windowRead !== undefined) {
                if (windowRead !== hash) {
                    conflict(key, "its detection head no longer matches the file read before it");
                }
                return;
            }
            const headSeen = heads.get(key);
            if (headSeen !== undefined) {
                if (headSeen !== hash)
                    conflict(key, "head-read twice, with different content each time");
                return;
            }
            if (unreadable.has(key)) {
                conflict(key, "could not be read, then head-read");
                return;
            }
            heads.set(key, hash);
        },
        present(absolutePath, size) {
            const key = rel(absolutePath);
            if (!files.has(key) && !heads.has(key))
                presence.set(key, size ?? -1);
        },
        realpath(absolutePath, canonicalPath) {
            const key = rel(absolutePath);
            const hash = canonicalPath === null ? null : hashContent(toForwardSlash(canonicalPath));
            if (realpaths.has(key) && realpaths.get(key) !== hash) {
                conflict(key, "resolved twice, with different targets each time");
                return;
            }
            realpaths.set(key, hash);
        },
        unreadable(absolutePath) {
            const key = rel(absolutePath);
            // A file the build already read and then could not read is the same mixed
            // snapshot the other way round: whatever was built from those bytes
            // stands on content this record would no longer describe.
            if (files.has(key) || heads.has(key)) {
                conflict(key, "read, then could not be read");
                return;
            }
            unreadable.add(key);
        },
        directory(absolutePath, listing) {
            const key = rel(absolutePath);
            directories.set(key, listing);
            unreadableDirectories.delete(key);
        },
        unreadableDirectory(absolutePath) {
            const key = rel(absolutePath);
            if (!directories.has(key))
                unreadableDirectories.add(key);
        },
        finish(extraExtensions, capabilities, pythonRoots = []) {
            if (conflicts.size > 0)
                throw new GraphInputsMovedDuringBuild(new Map(conflicts));
            return {
                version: GRAPH_INPUTS_VERSION,
                builtByVersion: SOCRATICODE_VERSION,
                capabilities,
                files: sortedRecord(files),
                heads: sortedRecord(heads),
                presence: sortedRecord(presence),
                ...(realpaths.size > 0 ? { realpaths: sortedRecord(realpaths) } : {}),
                unreadable: [...unreadable].sort(),
                unreadableDirectories: [...unreadableDirectories].sort(),
                directories: sortedRecord(directories),
                settings: graphSettingsHash(extraExtensions, pythonRoots),
            };
        },
    };
}
const isStringMap = (value) => value !== null &&
    typeof value === "object" &&
    !Array.isArray(value) &&
    Object.values(value).every((v) => typeof v === "string");
/**
 * A record whose shape this build understands, or null.
 *
 * Persisted as a JSON string beside the graph, so the string form is what
 * comes back and is accepted here; anything that fails to parse, or parses to
 * something that is not this shape, is null — which the caller reads the same
 * way as no record at all.
 */
export function parseGraphInputRecord(stored) {
    let candidate = stored;
    if (typeof candidate === "string") {
        try {
            candidate = JSON.parse(candidate);
        }
        catch {
            return null;
        }
    }
    if (candidate === null || typeof candidate !== "object")
        return null;
    const record = candidate;
    if (record.version !== GRAPH_INPUTS_VERSION)
        return null;
    if (typeof record.builtByVersion !== "string")
        return null;
    if (typeof record.settings !== "string")
        return null;
    if (typeof record.capabilities !== "string")
        return null;
    if (!isStringMap(record.files))
        return null;
    if (!isStringMap(record.heads))
        return null;
    if (record.realpaths !== undefined && (record.realpaths === null || typeof record.realpaths !== "object" || Array.isArray(record.realpaths)
        || Object.values(record.realpaths).some((target) => target !== null && typeof target !== "string")))
        return null;
    if (record.directories === null ||
        typeof record.directories !== "object" ||
        Array.isArray(record.directories) ||
        Object.values(record.directories).some((listing) => listing === null ||
            typeof listing !== "object" ||
            typeof listing.all !== "string" ||
            typeof listing.kept !== "string")) {
        return null;
    }
    if (!Array.isArray(record.unreadable) || record.unreadable.some((p) => typeof p !== "string")) {
        return null;
    }
    if (!Array.isArray(record.unreadableDirectories) ||
        record.unreadableDirectories.some((p) => typeof p !== "string")) {
        return null;
    }
    if (record.presence === null ||
        typeof record.presence !== "object" ||
        Array.isArray(record.presence) ||
        Object.values(record.presence).some((size) => typeof size !== "number")) {
        return null;
    }
    return record;
}
/** Files hashed per round when the record has inputs the index does not cover. */
const HASH_BATCH = 32;
/** Presence-only inputs stat'd per round. */
const STAT_BATCH = 64;
/** Directories listed per round. */
const LISTING_BATCH = 64;
const rebuildFor = (reason) => ({
    rebuild: true,
    reason,
    graphExists: true,
});
async function inBatches(items, size, run, inspect) {
    for (let i = 0; i < items.length; i += size) {
        const batch = items.slice(i, i + size);
        const results = await Promise.all(batch.map(run));
        for (let j = 0; j < batch.length; j++) {
            const decision = inspect(batch[j], results[j]);
            if (decision)
                return decision;
        }
    }
    return null;
}
/**
 * Whether the graph has to be rebuilt for this change.
 *
 * Conservative in every direction that is not proven: an unreadable or
 * unrecognised record, a version or settings change, any addition, and any
 * recorded input that changed, vanished, or cannot be read all rebuild. The
 * skip is reached only by checking every recorded input and finding all of
 * them unchanged.
 *
 * Ordered cheapest first — the checks that need no I/O, then stats, then head
 * reads, then whole-file hashes, and last the directory listings, which are
 * the only check that walks anything.
 */
export async function decideGraphRebuild(projectRoot, stored, change, extraExtensions, capabilities, pythonRoots = []) {
    const record = parseGraphInputRecord(stored);
    if (!record) {
        // A graph is there; what is missing is any account of what it was built
        // from. That is the legacy case, and it rebuilds once and writes one
        // however quiet the update was — `graphExists` is what says so.
        return rebuildFor("no usable record of what the graph was built from");
    }
    if (record.capabilities !== capabilities) {
        // Same version, different parsers: the optional grammars and the GDScript
        // addon ship platform-specific prebuilds, so a graph built where one was
        // missing holds none of the edges it would have found.
        return rebuildFor("the parsers available to the build changed");
    }
    if (record.builtByVersion !== SOCRATICODE_VERSION) {
        // A persisted graph is served unchanged across upgrades, so one cut before
        // a resolver shipped keeps answering as if that resolver did not exist —
        // the reasoning already recorded in `builtByVersion` (issue #120). One
        // rebuild on the first update after an upgrade settles it, and covers
        // every compile-time constant the build reads in the same stroke.
        return rebuildFor(`built by SocratiCode ${record.builtByVersion}, running ${SOCRATICODE_VERSION}`);
    }
    if (record.settings !== graphSettingsHash(extraExtensions, pythonRoots)) {
        return rebuildFor("graph configuration changed since the last build");
    }
    if (change.hasAdditions) {
        return rebuildFor("files were added");
    }
    const root = path.resolve(projectRoot);
    // Set lookups first: they settle the common case without touching the disk.
    const needHashing = [];
    for (const [relativePath, recordedHash] of Object.entries(record.files)) {
        if (change.removed.has(relativePath))
            return rebuildFor(`graph input removed: ${relativePath}`);
        if (change.unreadable.has(relativePath)) {
            return rebuildFor(`graph input no longer readable: ${relativePath}`);
        }
        const indexedHash = change.changed.get(relativePath) ?? change.knownHash(relativePath);
        if (indexedHash === undefined) {
            needHashing.push(relativePath);
            continue;
        }
        if (indexedHash !== recordedHash)
            return rebuildFor(`graph input changed: ${relativePath}`);
    }
    for (const relativePath of Object.keys(record.heads)) {
        if (change.removed.has(relativePath))
            return rebuildFor(`graph input removed: ${relativePath}`);
        if (change.unreadable.has(relativePath)) {
            return rebuildFor(`graph input no longer readable: ${relativePath}`);
        }
    }
    // A presence-only input is one the build already could not read, so an index
    // that still cannot read it says nothing new — only a change or a removal
    // does, which keeps an unreadable file from forcing a rebuild every run.
    const needStatting = Object.entries(record.presence);
    for (const [relativePath] of needStatting) {
        if (change.removed.has(relativePath) || change.changed.has(relativePath)) {
            return rebuildFor(`graph input changed: ${relativePath}`);
        }
    }
    const rebound = await inBatches(Object.entries(record.realpaths ?? {}), STAT_BATCH, async ([relativePath]) => {
        try {
            return hashContent(toForwardSlash(await fs.realpath(path.resolve(root, relativePath))));
        }
        catch (error) {
            const code = error.code;
            if (code !== "ENOENT" && code !== "ENOTDIR")
                throw error;
            return null;
        }
    }, ([relativePath, recordedTarget], target) => target !== recordedTarget
        ? rebuildFor(`module lookup binding changed: ${relativePath}`) : null);
    if (rebound)
        return rebound;
    const sized = await inBatches(needStatting, STAT_BATCH, async ([relativePath]) => {
        try {
            return (await fs.stat(path.resolve(root, relativePath))).size;
        }
        catch {
            return null;
        }
    }, ([relativePath, recordedSize], size) => {
        if (size === null)
            return rebuildFor(`graph input gone: ${relativePath}`);
        // Size is what carried the file across the limit, in either direction.
        if (recordedSize >= 0 && size !== recordedSize) {
            return rebuildFor(`graph input changed size: ${relativePath}`);
        }
        return null;
    });
    if (sized)
        return sized;
    // A directory the walk could not list is watched the same way, but with the
    // read it actually failed at: a subtree that becomes listable can add nodes
    // while its own entry in the parent never changes.
    const relisted = await inBatches(record.unreadableDirectories, LISTING_BATCH, async (relativePath) => {
        try {
            await fs.readdir(path.resolve(root, relativePath));
            return true;
        }
        catch {
            return false;
        }
    }, (relativePath, listable) => listable ? rebuildFor(`graph input directory became listable: ${relativePath}`) : null);
    if (relisted)
        return relisted;
    // A file the build could not open is watched by whether that is still true:
    // one that starts being readable starts contributing, and nothing else in
    // the record would move when it does.
    const recovered = await inBatches(record.unreadable, HASH_BATCH, async (relativePath) => {
        try {
            await fs.access(path.resolve(root, relativePath), fsConstants.R_OK);
            return true;
        }
        catch {
            return false;
        }
    }, (relativePath, readable) => readable ? rebuildFor(`graph input became readable: ${relativePath}`) : null);
    if (recovered)
        return recovered;
    const headed = await inBatches(Object.keys(record.heads), HASH_BATCH, async (relativePath) => {
        try {
            // The same window discovery scored the file on, so the two answers
            // stay comparable — see `ExtensionlessDetection.head`.
            return await readFileHead(path.resolve(root, relativePath));
        }
        catch {
            return null;
        }
    }, (relativePath, head) => {
        if (head === null)
            return rebuildFor(`graph input unreadable: ${relativePath}`);
        if (hashContent(head) !== record.heads[relativePath]) {
            return rebuildFor(`graph input changed: ${relativePath}`);
        }
        return null;
    });
    if (headed)
        return headed;
    const hashed = await inBatches(needHashing, HASH_BATCH, async (relativePath) => {
        try {
            return await fs.readFile(path.resolve(root, relativePath), "utf-8");
        }
        catch {
            return null;
        }
    }, (relativePath, content) => {
        if (content === null)
            return rebuildFor(`graph input unreadable: ${relativePath}`);
        if (hashContent(content) !== record.files[relativePath]) {
            return rebuildFor(`graph input changed: ${relativePath}`);
        }
        return null;
    });
    if (hashed)
        return hashed;
    // Last: whether anything appeared or vanished where the walk looked. This is
    // what catches a file the index never holds — a new nested `go.mod`, a new
    // `project.godot`, a `.uid` sidecar — none of which any other bucket can
    // speak for, since none of them existed when the record was written.
    //
    // Two tiers, because the entry names are free and the ignore filter is not.
    const moved = [];
    const listed = await inBatches(Object.keys(record.directories), LISTING_BATCH, async (relativePath) => {
        try {
            return await fs.readdir(path.resolve(root, relativePath), { withFileTypes: true });
        }
        catch {
            return null;
        }
    }, (relativePath, entries) => {
        if (entries === null)
            return rebuildFor(`graph input directory gone: ${relativePath}`);
        if (hashEntryNames(entries.map(entryName)) !== record.directories[relativePath].all) {
            moved.push([relativePath, entries]);
        }
        return null;
    });
    if (listed)
        return listed;
    if (moved.length > 0) {
        // Something is different in at least one directory. Only now is the filter
        // worth building, and only these directories are worth asking about.
        //
        // Nothing is written back. An ignored entry that appeared after the last
        // build leaves this cheap hash stale, so every later decision pays for one
        // ignore-filter build (~0.6s on a 3,941-file repository) to reach this
        // same answer, until the next real rebuild refreshes the record. That is
        // the accepted cost of keeping the record immutable except when the graph
        // itself is saved: the alternative is a second writer for a field the
        // graph save owns, and a write that lands after a concurrent rebuild would
        // replace a newer graph's inputs with an older graph's.
        const ig = createIgnoreFilter(root);
        for (const [relativePath, entries] of moved) {
            const { listing } = scanDirectory(ig, root, path.resolve(root, relativePath), entries);
            if (listing.kept !== record.directories[relativePath].kept) {
                return rebuildFor(`files appeared or vanished in: ${relativePath}`);
            }
        }
    }
    return { rebuild: false, reason: "no graph input changed", graphExists: true };
}
//# sourceMappingURL=graph-inputs.js.map